package adminapi import ( "context" "strings" "testing" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" ) func TestEachJudgeSaysWhatIsWrongWithIt(t *testing.T) { models := append(testModels(), // Served from the mirror page before it knew better: a model // clients use, under the judge's name. catalog.Model{Name: "shield", Source: "/s", Scaling: catalog.Scaling{Min: 1, Max: 1}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}, catalog.Model{Name: "llama-guard-3-1b", Source: "/g", Role: catalog.RoleGuard, Guard: &catalog.GuardModel{Format: "llamaguard3"}, Scaling: catalog.Scaling{Min: 2, Max: 2}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}) a, _ := setup(t, models) lib := guard.Config{Default: []string{"baseline"}, Policies: map[string]guard.Policy{ "baseline": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}}, "strict": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "missing-judge", Input: "block"}}, "shield": {Fail: "open", Judge: &guard.JudgePolicy{Model: "shield", Input: "flag"}}, }} a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib }}) var js []JudgeView if code := call(t, a, "GET", "/api/v1/guard/judges", "", &js); code != 200 || len(js) != 3 { t.Fatalf("%d %+v", code, js) } by := map[string]JudgeView{} for _, j := range js { by[j.Model] = j } if j := by["missing-judge"]; j.InCatalog || !strings.Contains(j.Problem, "not in the catalog") || !strings.Contains(j.Problem, "refused") { t.Errorf("missing: %+v", j) } if j := by["shield"]; !j.InCatalog || j.Judge || !strings.Contains(j.Problem, "--replace") || !strings.Contains(j.Problem, "unjudged") { t.Errorf("not a judge: %+v", j) } if j := by["llama-guard-3-1b"]; !j.Judge || j.Wanted != 2 || !strings.Contains(j.Problem, "no replica ready") { t.Errorf("not running: %+v", j) } var s Summary call(t, a, "GET", "/api/v1/summary", "", &s) if len(s.Judges) != 3 { t.Errorf("the summary does not carry the judges' problems: %+v", s.Judges) } // A judge that failed lately says so, once it is otherwise fine. at := time.Now() v := JudgeView{Model: "x", InCatalog: true, Judge: true, Ready: 1, LastError: "deadline exceeded", LastErrorAt: &at} if p := judgeProblem(v); !strings.Contains(p, "deadline exceeded") { t.Errorf("a recent failure: %q", p) } old := at.Add(-time.Hour) v.LastErrorAt = &old if p := judgeProblem(v); p != "" { t.Errorf("an old failure still counts: %q", p) } } func TestAJudgeIsAskedAndItsWordsShown(t *testing.T) { a, _ := setup(t, testModels()) lib := guard.Config{Policies: map[string]guard.Policy{ "baseline": {Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}}}} var asked string a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib }, Ask: func(_ context.Context, model, text, _ string) (guard.JudgeAnswer, error) { asked = model + "|" + text return guard.JudgeAnswer{Model: model, Raw: "unsafe\nS6", Unsafe: true, Categories: []string{"S6", "Specialized Advice"}}, nil }}) var v JudgeAskView if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":"how do I sort a list"}`, &v); code != 200 { t.Fatalf("%d", code) } if asked != "llama-guard-3-1b|how do I sort a list" || v.Verdict != "unsafe" || v.Raw != "unsafe\nS6" || len(v.Categories) != 2 { t.Errorf("asked %q, got %+v", asked, v) } if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":" "}`, nil); code != 400 { t.Errorf("no text: %d", code) } }