Ilker ManapandClaude Opus 5.5
f0477f140d
A suite is a set of cases in YAML or JSON lines, each scored by exact, contains, regex, a number within a tolerance, a JSON Schema subset, or a judge model with a rubric; weighed into one score against a threshold. internal/bench/eval runs one on the control plane through Router.Replay, as the gateway's own requests, and keeps every run and its cases in the store (migration 0028: eval_runs, eval_cases). - llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits non-zero below the score. Suites by name from control.eval_dir, or the two built in from deploy/eval (arithmetic, json-format). - model update and promote take --eval SUITE --min-score N, or a model's standing gate (ctl model gate, eval: in the catalog): a promote asks the canary first and leaves it below the score; an update tries the new weights as a canary given no share, then promotes or removes it. - The console's model menu: ask a suite with its progress, its runs, its gate. - The audit middleware no longer hands a handler a body cut off at 64 KiB. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
75 lines
2.4 KiB
Go
75 lines
2.4 KiB
Go
package audit
|
|
|
|
import (
|
|
"context"
|
|
"io"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"net/url"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func TestNothingSecretIsRecorded(t *testing.T) {
|
|
got := Redact(FormFields(url.Values{
|
|
"current": {"old-pw"}, "new": {"new-pw"}, "password": {"pw"}, "csrf": {"tok"},
|
|
"name": {"ok-name"}, "note": {"sk-proj-abcdefgh"},
|
|
}))
|
|
for _, secret := range []string{"old-pw", "new-pw", `"pw"`, "tok", "sk-proj"} {
|
|
if strings.Contains(got, secret) {
|
|
t.Errorf("%s recorded: %s", secret, got)
|
|
}
|
|
}
|
|
if !strings.Contains(got, "ok-name") {
|
|
t.Errorf("an ordinary field was dropped: %s", got)
|
|
}
|
|
nested := Redact(map[string]any{"external": map[string]any{"api_key": "x1", "url": "https://p"}})
|
|
if strings.Contains(nested, "x1") || !strings.Contains(nested, "https://p") {
|
|
t.Errorf("nested: %s", nested)
|
|
}
|
|
}
|
|
|
|
type memSink struct{ got []store.AuditEvent }
|
|
|
|
func (m *memSink) WriteAudit(_ context.Context, e store.AuditEvent) error {
|
|
m.got = append(m.got, e)
|
|
return nil
|
|
}
|
|
|
|
func TestEveryChangeToTheAdminAPIIsRecorded(t *testing.T) {
|
|
sink := &memSink{}
|
|
h := API(sink, func(*http.Request) string { return "ilker" }, nil, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
w.WriteHeader(http.StatusCreated)
|
|
}))
|
|
for _, req := range []*http.Request{
|
|
httptest.NewRequest("GET", "/api/v1/models", nil),
|
|
httptest.NewRequest("POST", "/api/v1/models", strings.NewReader(`{"name":"m1","external":{"api_key":"sk-1"}}`)),
|
|
httptest.NewRequest("DELETE", "/api/v1/nodes/n1", nil),
|
|
} {
|
|
h.ServeHTTP(httptest.NewRecorder(), req)
|
|
}
|
|
if len(sink.got) != 2 {
|
|
t.Fatalf("recorded %d: %+v", len(sink.got), sink.got)
|
|
}
|
|
e := sink.got[0]
|
|
if e.Actor != "ilker" || e.Action != "POST /api/v1/models" || e.Target != "m1" || e.Status != 201 ||
|
|
strings.Contains(e.Detail, "sk-1") || e.Source != "api" {
|
|
t.Errorf("%+v", e)
|
|
}
|
|
}
|
|
|
|
func TestABodyLargerThanWhatIsRecordedReachesTheHandlerWhole(t *testing.T) {
|
|
big := `{"content":"` + strings.Repeat("x", 200<<10) + `"}`
|
|
var got int
|
|
h := API(&memSink{}, func(*http.Request) string { return "ilker" }, nil, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
b, _ := io.ReadAll(r.Body)
|
|
got = len(b)
|
|
}))
|
|
h.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("POST", "/api/v1/models/m/eval", strings.NewReader(big)))
|
|
if got != len(big) {
|
|
t.Errorf("the handler was given %d of %d bytes", got, len(big))
|
|
}
|
|
}
|