The guardrails' judge has its own command, and its trouble is shown

llmgw ctl guard judge status lists each judge model the policies ask:
in the catalog, as a judge, replicas ready, its last failure to answer, and
what is wrong in words. enable serves the missing one through model serve -
Llama Guard 3 1B on CPUs, role guard, under the name the policies give it -
with --replace for a model of that name that is not a judge; the console has
it under Do. GET /api/v1/guard/judges, and the summary's judges, put the
problem on ctl status and the console's Overview.

A Llama Guard file served from the mirror page is added as a judge, under
the name the policies ask, rather than as a model clients use the policies
never find. ctl says plainly when the API answers with a page.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/control/adminapi/judges_test.go

@@ -0,0 +1,66 @@
+package adminapi
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
+)
+
+func TestEachJudgeSaysWhatIsWrongWithIt(t *testing.T) {
+	models := append(testModels(),
+		// Served from the mirror page before it knew better: a model
+		// clients use, under the judge's name.
+		catalog.Model{Name: "shield", Source: "/s", Scaling: catalog.Scaling{Min: 1, Max: 1},
+			Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
+		catalog.Model{Name: "llama-guard-3-1b", Source: "/g", Role: catalog.RoleGuard,
+			Guard: &catalog.GuardModel{Format: "llamaguard3"}, Scaling: catalog.Scaling{Min: 2, Max: 2},
+			Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
+	a, _ := setup(t, models)
+	lib := guard.Config{Default: []string{"baseline"}, Policies: map[string]guard.Policy{
+		"baseline": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}},
+		"strict":   {Fail: "closed", Judge: &guard.JudgePolicy{Model: "missing-judge", Input: "block"}},
+		"shield":   {Fail: "open", Judge: &guard.JudgePolicy{Model: "shield", Input: "flag"}},
+	}}
+	a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib }})
+
+	var js []JudgeView
+	if code := call(t, a, "GET", "/api/v1/guard/judges", "", &js); code != 200 || len(js) != 3 {
+		t.Fatalf("%d %+v", code, js)
+	}
+	by := map[string]JudgeView{}
+	for _, j := range js {
+		by[j.Model] = j
+	}
+	if j := by["missing-judge"]; j.InCatalog || !strings.Contains(j.Problem, "not in the catalog") ||
+		!strings.Contains(j.Problem, "refused") {
+		t.Errorf("missing: %+v", j)
+	}
+	if j := by["shield"]; !j.InCatalog || j.Judge || !strings.Contains(j.Problem, "--replace") ||
+		!strings.Contains(j.Problem, "unjudged") {
+		t.Errorf("not a judge: %+v", j)
+	}
+	if j := by["llama-guard-3-1b"]; !j.Judge || j.Wanted != 2 || !strings.Contains(j.Problem, "no replica ready") {
+		t.Errorf("not running: %+v", j)
+	}
+
+	var s Summary
+	call(t, a, "GET", "/api/v1/summary", "", &s)
+	if len(s.Judges) != 3 {
+		t.Errorf("the summary does not carry the judges' problems: %+v", s.Judges)
+	}
+
+	// A judge that failed lately says so, once it is otherwise fine.
+	at := time.Now()
+	v := JudgeView{Model: "x", InCatalog: true, Judge: true, Ready: 1, LastError: "deadline exceeded", LastErrorAt: &at}
+	if p := judgeProblem(v); !strings.Contains(p, "deadline exceeded") {
+		t.Errorf("a recent failure: %q", p)
+	}
+	old := at.Add(-time.Hour)
+	v.LastErrorAt = &old
+	if p := judgeProblem(v); p != "" {
+		t.Errorf("an old failure still counts: %q", p)
+	}
+}