Ilker ManapandClaude Opus 5.5
a8bb417618
llmgw ctl guard judge status lists each judge model the policies ask: in the catalog, as a judge, replicas ready, its last failure to answer, and what is wrong in words. enable serves the missing one through model serve - Llama Guard 3 1B on CPUs, role guard, under the name the policies give it - with --replace for a model of that name that is not a judge; the console has it under Do. GET /api/v1/guard/judges, and the summary's judges, put the problem on ctl status and the console's Overview. A Llama Guard file served from the mirror page is added as a judge, under the name the policies ask, rather than as a model clients use the policies never find. ctl says plainly when the API answers with a page. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
67 lines
2.5 KiB
Go
67 lines
2.5 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
|
)
|
|
|
|
func TestEachJudgeSaysWhatIsWrongWithIt(t *testing.T) {
|
|
models := append(testModels(),
|
|
// Served from the mirror page before it knew better: a model
|
|
// clients use, under the judge's name.
|
|
catalog.Model{Name: "shield", Source: "/s", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
|
|
catalog.Model{Name: "llama-guard-3-1b", Source: "/g", Role: catalog.RoleGuard,
|
|
Guard: &catalog.GuardModel{Format: "llamaguard3"}, Scaling: catalog.Scaling{Min: 2, Max: 2},
|
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
|
|
a, _ := setup(t, models)
|
|
lib := guard.Config{Default: []string{"baseline"}, Policies: map[string]guard.Policy{
|
|
"baseline": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}},
|
|
"strict": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "missing-judge", Input: "block"}},
|
|
"shield": {Fail: "open", Judge: &guard.JudgePolicy{Model: "shield", Input: "flag"}},
|
|
}}
|
|
a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib }})
|
|
|
|
var js []JudgeView
|
|
if code := call(t, a, "GET", "/api/v1/guard/judges", "", &js); code != 200 || len(js) != 3 {
|
|
t.Fatalf("%d %+v", code, js)
|
|
}
|
|
by := map[string]JudgeView{}
|
|
for _, j := range js {
|
|
by[j.Model] = j
|
|
}
|
|
if j := by["missing-judge"]; j.InCatalog || !strings.Contains(j.Problem, "not in the catalog") ||
|
|
!strings.Contains(j.Problem, "refused") {
|
|
t.Errorf("missing: %+v", j)
|
|
}
|
|
if j := by["shield"]; !j.InCatalog || j.Judge || !strings.Contains(j.Problem, "--replace") ||
|
|
!strings.Contains(j.Problem, "unjudged") {
|
|
t.Errorf("not a judge: %+v", j)
|
|
}
|
|
if j := by["llama-guard-3-1b"]; !j.Judge || j.Wanted != 2 || !strings.Contains(j.Problem, "no replica ready") {
|
|
t.Errorf("not running: %+v", j)
|
|
}
|
|
|
|
var s Summary
|
|
call(t, a, "GET", "/api/v1/summary", "", &s)
|
|
if len(s.Judges) != 3 {
|
|
t.Errorf("the summary does not carry the judges' problems: %+v", s.Judges)
|
|
}
|
|
|
|
// A judge that failed lately says so, once it is otherwise fine.
|
|
at := time.Now()
|
|
v := JudgeView{Model: "x", InCatalog: true, Judge: true, Ready: 1, LastError: "deadline exceeded", LastErrorAt: &at}
|
|
if p := judgeProblem(v); !strings.Contains(p, "deadline exceeded") {
|
|
t.Errorf("a recent failure: %q", p)
|
|
}
|
|
old := at.Add(-time.Hour)
|
|
v.LastErrorAt = &old
|
|
if p := judgeProblem(v); p != "" {
|
|
t.Errorf("an old failure still counts: %q", p)
|
|
}
|
|
}
|