A suite is a set of cases in YAML or JSON lines, each scored by exact, contains, regex, a number within a tolerance, a JSON Schema subset, or a judge model with a rubric; weighed into one score against a threshold. internal/bench/eval runs one on the control plane through Router.Replay, as the gateway's own requests, and keeps every run and its cases in the store (migration 0028: eval_runs, eval_cases). - llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits non-zero below the score. Suites by name from control.eval_dir, or the two built in from deploy/eval (arithmetic, json-format). - model update and promote take --eval SUITE --min-score N, or a model's standing gate (ctl model gate, eval: in the catalog): a promote asks the canary first and leaves it below the score; an update tries the new weights as a canary given no share, then promotes or removes it. - The console's model menu: ask a suite with its progress, its runs, its gate. - The audit middleware no longer hands a handler a body cut off at 64 KiB. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
159 lines
4.1 KiB
Go
159 lines
4.1 KiB
Go
// Package audit records who changed what: every change made through the
|
|
// admin API and the administration pages, written to the store with the
|
|
// request's fields — anything secret taken out — and how it ended.
|
|
package audit
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// Sink is where events go: the store.
|
|
type Sink interface {
|
|
WriteAudit(ctx context.Context, e store.AuditEvent) error
|
|
}
|
|
|
|
// secretName is a field whose value is never recorded.
|
|
// The password form's fields are current and new.
|
|
var secretName = regexp.MustCompile(`(?i)(pass|secret|token|api_?key|^key$|dsn|private|credential|csrf|cookie|^current$|^new$|^confirm)`)
|
|
|
|
// Redact is a request's fields with secrets replaced, as JSON.
|
|
func Redact(fields map[string]any) string {
|
|
clean := redactValue(fields)
|
|
b, _ := json.Marshal(clean)
|
|
if len(b) > 8192 {
|
|
b = append(b[:8192], []byte("…")...)
|
|
}
|
|
return string(b)
|
|
}
|
|
|
|
func redactValue(v any) any {
|
|
switch t := v.(type) {
|
|
case map[string]any:
|
|
out := make(map[string]any, len(t))
|
|
for k, val := range t {
|
|
if secretName.MatchString(k) {
|
|
out[k] = "***"
|
|
continue
|
|
}
|
|
out[k] = redactValue(val)
|
|
}
|
|
return out
|
|
case []any:
|
|
out := make([]any, len(t))
|
|
for i, val := range t {
|
|
out[i] = redactValue(val)
|
|
}
|
|
return out
|
|
case string:
|
|
// A secret pasted where a name was expected.
|
|
if strings.HasPrefix(t, "sk-") || strings.Contains(t, "PRIVATE KEY") {
|
|
return "***"
|
|
}
|
|
return t
|
|
}
|
|
return v
|
|
}
|
|
|
|
// FormFields is a form as fields, for Redact.
|
|
func FormFields(v url.Values) map[string]any {
|
|
out := map[string]any{}
|
|
keys := make([]string, 0, len(v))
|
|
for k := range v {
|
|
keys = append(keys, k)
|
|
}
|
|
sort.Strings(keys)
|
|
for _, k := range keys {
|
|
if len(v[k]) == 1 {
|
|
out[k] = v[k][0]
|
|
} else {
|
|
out[k] = v[k]
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// statusWriter keeps the status a handler answered with.
|
|
type statusWriter struct {
|
|
http.ResponseWriter
|
|
status int
|
|
}
|
|
|
|
func (w *statusWriter) WriteHeader(code int) {
|
|
if w.status == 0 {
|
|
w.status = code
|
|
}
|
|
w.ResponseWriter.WriteHeader(code)
|
|
}
|
|
|
|
func (w *statusWriter) Write(b []byte) (int, error) {
|
|
if w.status == 0 {
|
|
w.status = http.StatusOK
|
|
}
|
|
return w.ResponseWriter.Write(b)
|
|
}
|
|
|
|
func (w *statusWriter) Flush() {
|
|
if f, ok := w.ResponseWriter.(http.Flusher); ok {
|
|
f.Flush()
|
|
}
|
|
}
|
|
|
|
// API records every change to the admin API: a request that is not a GET or
|
|
// HEAD, under /api/. actor says who made it.
|
|
func API(sink Sink, actor func(*http.Request) string, log *slog.Logger, next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.Method == http.MethodGet || r.Method == http.MethodHead || !strings.HasPrefix(r.URL.Path, "/api/") {
|
|
next.ServeHTTP(w, r)
|
|
return
|
|
}
|
|
var body []byte
|
|
if r.Body != nil {
|
|
// The first 64 KiB are what is recorded; the handler is given the
|
|
// whole body, or a larger one — a quality suite sent whole — would
|
|
// reach it cut off.
|
|
body, _ = io.ReadAll(io.LimitReader(r.Body, 64<<10))
|
|
r.Body = struct {
|
|
io.Reader
|
|
io.Closer
|
|
}{io.MultiReader(bytes.NewReader(body), r.Body), r.Body}
|
|
}
|
|
sw := &statusWriter{ResponseWriter: w}
|
|
next.ServeHTTP(sw, r)
|
|
e := store.AuditEvent{At: time.Now(), Actor: actor(r), Source: "api",
|
|
Action: r.Method + " " + r.URL.Path, Status: sw.status}
|
|
var fields map[string]any
|
|
if json.Unmarshal(body, &fields) == nil {
|
|
e.Detail = Redact(fields)
|
|
}
|
|
for _, k := range []string{"name", "target", "scope", "model"} {
|
|
if s, ok := fields[k].(string); ok && e.Target == "" {
|
|
e.Target = s
|
|
}
|
|
}
|
|
Write(r.Context(), sink, log, e)
|
|
})
|
|
}
|
|
|
|
// Write records an event, saying so in the log when the store will not take
|
|
// it: a change that happened and is not in the audit is worth a warning.
|
|
func Write(ctx context.Context, sink Sink, log *slog.Logger, e store.AuditEvent) {
|
|
if sink == nil {
|
|
return
|
|
}
|
|
if err := sink.WriteAudit(context.WithoutCancel(ctx), e); err != nil && log != nil {
|
|
log.Warn("a change could not be written to the audit log", "action", e.Action, "actor", e.Actor, "err", err)
|
|
}
|
|
}
|