Requests kept and replayed: keys requests per project with retention and a share, ctl request --body and --replay
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
841ee21c12
commit
32ff1dbd6f
22 files changed
+757
-10
No files matched your search
@@ -10,18 +10,34 @@ import (
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
var (
|
||||
body bool
|
||||
replay string
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "request ID",
|
||||
Short: "What one request went through: replica, queue, first token, tokens, guardrails",
|
||||
Long: "Every response carries its id in X-Request-Id, and a client may send its\n" +
|
||||
"own. Given one, this says which replica answered, how long admission held\n" +
|
||||
"it, how long the first token took and what it cost — the answer to \"why\n" +
|
||||
"was this slow\". The access log has the same fields under \"request\".",
|
||||
Args: cobra.ExactArgs(1),
|
||||
"was this slow\". The access log has the same fields under \"request\".\n\n" +
|
||||
"For a project that keeps its requests (`llmgw keys requests`), --body\n" +
|
||||
"shows what the model was sent and what came back, and --replay MODEL\n" +
|
||||
"sends the same request to another model, past the limits and the\n" +
|
||||
"guardrails, and shows its answer beside the first.",
|
||||
Example: " llmgw ctl request req_3f2a…\n llmgw ctl request req_3f2a… --body\n llmgw ctl request req_3f2a… --replay qwen2.5-14b-instruct",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
if replay != "" {
|
||||
return replayRequest(cmd, client, args[0], replay, *asJSON)
|
||||
}
|
||||
if body {
|
||||
return showRequestBody(cmd, client, args[0], *asJSON)
|
||||
}
|
||||
var v adminapi.RequestView
|
||||
if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(args[0]), &v); err != nil {
|
||||
return err
|
||||
@@ -65,4 +81,70 @@ func newRequestCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().BoolVar(&body, "body", false, "show the request and its answer, for a project that keeps them")
|
||||
c.Flags().StringVar(&replay, "replay", "", "send the kept request to this model and show both answers")
|
||||
return c
|
||||
}
|
||||
|
||||
func showRequestBody(cmd *cobra.Command, client func() *ctlClient, id string, asJSON bool) error {
|
||||
var b store.RequestBody
|
||||
if err := client().get(cmd.Context(), "/api/v1/requests/"+url.PathEscape(id)+"/body", &b); err != nil {
|
||||
return err
|
||||
}
|
||||
if asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(b)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "request %s at %s, %s to %s (served by %s), status %d\n", b.RequestID,
|
||||
b.At.UTC().Format("2006-01-02 15:04:05Z"), b.Path, b.Model, b.ServedModel, b.Status)
|
||||
fmt.Fprintf(out, "kept until %s\n\n", b.ExpiresAt.UTC().Format("2006-01-02 15:04Z"))
|
||||
fmt.Fprintln(out, "--- sent")
|
||||
fmt.Fprintln(out, indentJSON(b.Request))
|
||||
fmt.Fprintln(out, "--- answered")
|
||||
fmt.Fprintln(out, or(b.Response, "(no text: an error, or not a completion)"))
|
||||
if b.Truncated {
|
||||
fmt.Fprintln(out, "(longer than what is kept; cut here)")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func replayRequest(cmd *cobra.Command, client func() *ctlClient, id, model string, asJSON bool) error {
|
||||
var v adminapi.ReplayView
|
||||
body, _ := json.Marshal(map[string]string{"model": model})
|
||||
if err := client().post(cmd.Context(), "/api/v1/requests/"+url.PathEscape(id)+"/replay", body, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if asJSON {
|
||||
return json.NewEncoder(cmd.OutOrStdout()).Encode(v)
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "--- %s answered (status %d)\n%s\n\n", v.Original.ServedModel, v.Original.Status,
|
||||
or(v.Original.Response, "(no text)"))
|
||||
fmt.Fprintf(out, "--- %s answers (status %d, %d ms)\n%s\n", v.Model, v.Status, v.TookMs, or(replyText(v.Body), string(v.Body)))
|
||||
return nil
|
||||
}
|
||||
|
||||
// replyText is a completion's text, or "" when it is not one.
|
||||
func replyText(body json.RawMessage) string {
|
||||
var c struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
Text string `json:"text"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if json.Unmarshal(body, &c) != nil || len(c.Choices) == 0 {
|
||||
return ""
|
||||
}
|
||||
return c.Choices[0].Message.Content + c.Choices[0].Text
|
||||
}
|
||||
|
||||
func indentJSON(s string) string {
|
||||
var v any
|
||||
if json.Unmarshal([]byte(s), &v) != nil {
|
||||
return s
|
||||
}
|
||||
b, _ := json.MarshalIndent(v, "", " ")
|
||||
return string(b)
|
||||
}
|
||||
@@ -42,6 +42,7 @@ func newTenancyCmd() *cobra.Command {
|
||||
newKeyTTLCmd(open),
|
||||
newLimitsCmd(open),
|
||||
newThreadsCmd(open),
|
||||
newRequestLogCmd(open),
|
||||
newMemoryCmd(open),
|
||||
newMoveStoreCmd(),
|
||||
)
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func newRequestLogCmd(open opener) *cobra.Command {
|
||||
var (
|
||||
project string
|
||||
hours int
|
||||
sample float64
|
||||
off bool
|
||||
)
|
||||
cmd := &cobra.Command{
|
||||
Use: "requests",
|
||||
Short: "Keep a project's requests and answers, to follow one up or replay it",
|
||||
Long: "With this on, what each request of the project sent the model — after the\n" +
|
||||
"guardrails' redaction, with any stored conversation in it — and the answer\n" +
|
||||
"the client got are kept for --retain-hours. `llmgw ctl request ID --body`\n" +
|
||||
"shows one, and `--replay MODEL` sends it to another model.\n\n" +
|
||||
"Off for every project until this is run, like conversations: llmgw\n" +
|
||||
"otherwise keeps no text. --sample keeps a share of the requests rather\n" +
|
||||
"than all of them. A project under a guardrails policy with no_store keeps\n" +
|
||||
"nothing whatever this says.",
|
||||
Example: " llmgw keys requests --project Acme/ml/train --retain-hours 72\n" +
|
||||
" llmgw keys requests --project Acme/ml/train --retain-hours 168 --sample 0.05\n" +
|
||||
" llmgw keys requests --project Acme/ml/train --off",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
if !off && hours <= 0 {
|
||||
return fmt.Errorf("--retain-hours is required: say how long requests are kept, or pass --off")
|
||||
}
|
||||
if sample <= 0 || sample > 1 {
|
||||
return fmt.Errorf("--sample is a share: above 0, at most 1")
|
||||
}
|
||||
s, err := open(cmd.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer s.Close()
|
||||
p, err := resolveProject(cmd.Context(), s, project)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if off {
|
||||
hours = 0
|
||||
}
|
||||
if err := s.SetRequestLog(cmd.Context(), p.ID, hours, sample); err != nil {
|
||||
return err
|
||||
}
|
||||
if hours == 0 {
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s keeps no requests. Those kept are removed at the next prune.\n", p.Path)
|
||||
return nil
|
||||
}
|
||||
fmt.Fprintf(cmd.OutOrStdout(), "%s keeps %s of its requests and answers for %d hours.\n",
|
||||
p.Path, share(sample), hours)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.Flags().StringVar(&project, "project", "", "project: id, path or unique name")
|
||||
cmd.Flags().IntVar(&hours, "retain-hours", 0, "how long a request and its answer are kept")
|
||||
cmd.Flags().Float64Var(&sample, "sample", 1, "the share of requests kept, above 0 and at most 1")
|
||||
cmd.Flags().BoolVar(&off, "off", false, "stop keeping requests for this project")
|
||||
return cmd
|
||||
}
|
||||
|
||||
func share(f float64) string {
|
||||
if f >= 1 {
|
||||
return "every one"
|
||||
}
|
||||
return fmt.Sprintf("%g%%", f*100)
|
||||
}
|
||||
@@ -292,6 +292,7 @@ func newServerCmd() *cobra.Command {
|
||||
adminAPI = api
|
||||
api.SetReleases(cfg.Releases)
|
||||
api.SetNotifier(notifier)
|
||||
api.SetReplayer(rt.Replay)
|
||||
if st := rt.Store(); st != nil && gate != nil {
|
||||
api.SetGuard(&adminapi.GuardAdmin{Library: cfg.Guard, Store: st, Invalidate: gate.Invalidate})
|
||||
}
|
||||
|
||||
@@ -70,6 +70,8 @@ type peopleData struct {
|
||||
limits map[string]store.Limits
|
||||
threads map[string]int // project id → hours kept, conversations
|
||||
notes map[string]int // project id → hours kept, agent notes
|
||||
// requests is how each project keeps its requests and answers.
|
||||
requests map[string]store.RequestLog
|
||||
// guardPolicies are server.yaml's guardrail policies, guardDefault those
|
||||
// applied everywhere, and guardAt which are attached where.
|
||||
guardPolicies []string
|
||||
@@ -122,7 +124,7 @@ func (m consoleModel) fetchPeople() tea.Cmd {
|
||||
d.guardDefault = cfg.Guard.Default
|
||||
d.guardAt, _ = s.GuardAttachments(ctx)
|
||||
}
|
||||
d.threads, d.notes = map[string]int{}, map[string]int{}
|
||||
d.threads, d.notes, d.requests = map[string]int{}, map[string]int{}, map[string]store.RequestLog{}
|
||||
for _, r := range d.orgs {
|
||||
if r.level != "project" {
|
||||
continue
|
||||
@@ -133,6 +135,9 @@ func (m consoleModel) fetchPeople() tea.Cmd {
|
||||
if t, err := s.MemoryRetention(ctx, r.id); err == nil {
|
||||
d.notes[r.id] = int(t.Hours())
|
||||
}
|
||||
if l, err := s.RequestLogOf(ctx, r.id); err == nil {
|
||||
d.requests[r.id] = l
|
||||
}
|
||||
}
|
||||
return consolePeopleMsg(d)
|
||||
}
|
||||
@@ -502,7 +507,33 @@ func (m consoleModel) orgMenu(r orgRow) *menuDialog {
|
||||
}
|
||||
items = append(items,
|
||||
retention("Keep conversations", "threads", "conversations", m.people.threads[r.id]),
|
||||
retention("Keep agent notes", "memory", "notes", m.people.notes[r.id]))
|
||||
retention("Keep agent notes", "memory", "notes", m.people.notes[r.id]),
|
||||
menuItem{"Keep requests…", func(m *consoleModel) tea.Cmd {
|
||||
l := m.people.requests[r.id]
|
||||
sample := "1"
|
||||
if l.Sample > 0 {
|
||||
sample = strconv.FormatFloat(l.Sample, 'f', -1, 64)
|
||||
}
|
||||
m.openForm(&formDialog{title: "Keep requests · " + r.path,
|
||||
note: "Off by default: llmgw keeps no requests unless a project asks for it.\n" +
|
||||
"Kept, `llmgw ctl request ID --body` shows one and --replay sends it to another model.",
|
||||
fields: []formField{
|
||||
textF("hours", strconv.Itoa(int(l.Retain.Hours())), "how long they are kept; 0 keeps none"),
|
||||
textF("share", sample, "of requests kept, above 0 and at most 1"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
h, err := strconv.Atoi(v[0])
|
||||
if err != nil || h < 0 {
|
||||
return nil, fmt.Errorf("hours is a whole number")
|
||||
}
|
||||
args := []string{"keys", "requests", "--project=" + r.id}
|
||||
if h == 0 {
|
||||
return m.runCLI("Keep requests", append(args, "--off")...), nil
|
||||
}
|
||||
return m.runCLI("Keep requests", append(args, "--retain-hours="+v[0], "--sample="+v[1])...), nil
|
||||
}})
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
items = append(items, menuItem{"Remove…", func(m *consoleModel) tea.Cmd {
|
||||
m.confirmD = &confirmDialog{title: "Remove " + r.path,
|
||||
|
||||
Reference in new issue
Block a user