Ilker ManapandClaude Opus 5.5
f0477f140d
A suite is a set of cases in YAML or JSON lines, each scored by exact, contains, regex, a number within a tolerance, a JSON Schema subset, or a judge model with a rubric; weighed into one score against a threshold. internal/bench/eval runs one on the control plane through Router.Replay, as the gateway's own requests, and keeps every run and its cases in the store (migration 0028: eval_runs, eval_cases). - llmgw ctl eval MODEL --suite FILE|NAME, --history, --run, --suites; exits non-zero below the score. Suites by name from control.eval_dir, or the two built in from deploy/eval (arithmetic, json-format). - model update and promote take --eval SUITE --min-score N, or a model's standing gate (ctl model gate, eval: in the catalog): a promote asks the canary first and leaves it below the score; an update tries the new weights as a canary given no share, then promotes or removes it. - The console's model menu: ask a suite with its progress, its runs, its gate. - The audit middleware no longer hands a handler a body cut off at 64 KiB. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
60 lines
2.4 KiB
Go
60 lines
2.4 KiB
Go
package store
|
|
|
|
import (
|
|
"errors"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestEvalRunsAreKeptWithTheirCases(t *testing.T) {
|
|
checkEvalRuns(t, capacityStore(t))
|
|
}
|
|
|
|
func TestPostgresEvalRunsAreKeptWithTheirCases(t *testing.T) {
|
|
checkEvalRuns(t, openPG(t))
|
|
}
|
|
|
|
func checkEvalRuns(t *testing.T, db *DB) {
|
|
ctx := t.Context()
|
|
now := time.Now().UTC().Truncate(time.Millisecond)
|
|
first := EvalRun{ID: "eval-1", Model: "qwen", Repo: "o/r", Revision: "abc", Suite: "arith", SuiteHash: "h1",
|
|
At: now.Add(-time.Hour), TookMs: 1200, Score: 0.9, Threshold: 0.8, Passed: true, Cases: 2, Below: 0,
|
|
By: "ilker", Results: []EvalCase{
|
|
{ID: "add", Kind: "number", Weight: 1, Score: 1, Passed: true, Answer: "42", Detail: "read 42", TookMs: 300},
|
|
{ID: "json", Kind: "json_schema", Weight: 2, Score: 0.85, Passed: true, Answer: "{}", TookMs: 900},
|
|
}}
|
|
second := EvalRun{ID: "eval-2", Model: "qwen", Target: "qwen-canary", Revision: "def", Suite: "arith",
|
|
SuiteHash: "h1", At: now, Score: 0.5, Threshold: 0.8, Cases: 1, Below: 1, Errors: 1, Job: "promote-1",
|
|
Results: []EvalCase{{ID: "add", Kind: "number", Weight: 1, Error: "qwen-canary answered 503"}}}
|
|
other := EvalRun{ID: "eval-3", Model: "llama", Suite: "fmt", SuiteHash: "h2", At: now, Score: 1, Threshold: 0.8,
|
|
Passed: true, Cases: 0}
|
|
for _, r := range []EvalRun{first, second, other} {
|
|
if err := db.PutEvalRun(ctx, r); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
runs, err := db.EvalRuns(ctx, "qwen", 10)
|
|
if err != nil || len(runs) != 2 || runs[0].ID != "eval-2" || runs[1].ID != "eval-1" {
|
|
t.Fatalf("%+v %v", runs, err)
|
|
}
|
|
if r := runs[0]; r.Target != "qwen-canary" || r.Passed || r.Job != "promote-1" || r.Errors != 1 ||
|
|
!r.At.Equal(now) || r.Results != nil {
|
|
t.Errorf("%+v", r)
|
|
}
|
|
// Asked by the canary's name, its runs are found too.
|
|
if got, _ := db.EvalRuns(ctx, "qwen-canary", 10); len(got) != 1 || got[0].ID != "eval-2" {
|
|
t.Errorf("by target: %+v", got)
|
|
}
|
|
if all, _ := db.EvalRuns(ctx, "", 2); len(all) != 2 {
|
|
t.Errorf("limit: %d", len(all))
|
|
}
|
|
got, err := db.EvalRunByID(ctx, "eval-1")
|
|
if err != nil || got.Score != 0.9 || !got.Passed || got.Repo != "o/r" || got.By != "ilker" || len(got.Results) != 2 ||
|
|
got.Results[1].ID != "json" || got.Results[1].Weight != 2 || got.Results[0].Answer != "42" || !got.Results[0].Passed {
|
|
t.Fatalf("%+v %v", got, err)
|
|
}
|
|
if _, err := db.EvalRunByID(ctx, "eval-9"); !errors.Is(err, ErrNotFound) {
|
|
t.Errorf("%v", err)
|
|
}
|
|
}
|