The guardrails' judge has its own command, and its trouble is shown

llmgw ctl guard judge status lists each judge model the policies ask:
in the catalog, as a judge, replicas ready, its last failure to answer, and
what is wrong in words. enable serves the missing one through model serve -
Llama Guard 3 1B on CPUs, role guard, under the name the policies give it -
with --replace for a model of that name that is not a judge; the console has
it under Do. GET /api/v1/guard/judges, and the summary's judges, put the
problem on ctl status and the console's Overview.

A Llama Guard file served from the mirror page is added as a judge, under
the name the policies ask, rather than as a model clients use the policies
never find. ctl says plainly when the API answers with a page.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/control/adminapi/guard.go

@@ -28,6 +28,8 @@ type GuardAdmin struct {
 	Library    func() guard.Config
 	Store      GuardStore
 	Invalidate func()
+	// JudgeErrors are each judge's last failure to give a verdict.
+	JudgeErrors func() map[string]guard.JudgeError
 }
 
 // SetGuard enables the guard endpoints.
@@ -198,6 +200,7 @@ type GuardAttachmentRequest struct {
 }
 
 func (a *API) registerGuard(mux Mux) {
+	mux.HandleFunc("GET /api/v1/guard/judges", a.listJudges)
 	mux.HandleFunc("GET /api/v1/guard", func(w http.ResponseWriter, r *http.Request) {
 		if a.guard == nil {
 			writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")

internal/control/adminapi/judges.go

@@ -0,0 +1,124 @@
+package adminapi
+
+import (
+	"fmt"
+	"net/http"
+	"slices"
+	"sort"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
+)
+
+// JudgeView is one judge model the guardrail policies ask, and whether it
+// can answer them.
+type JudgeView struct {
+	Model string `json:"model"`
+	// Policies are those that ask it; Closed those that refuse what it
+	// cannot read (fail: closed).
+	Policies []string `json:"policies"`
+	Closed   []string `json:"closed,omitempty"`
+	// InCatalog is whether a model of that name is in the catalog, and
+	// Judge whether it is there as a judge (role: guard).
+	InCatalog bool `json:"in_catalog"`
+	Judge     bool `json:"judge"`
+	Ready     int  `json:"ready_replicas"`
+	Wanted    int  `json:"wanted_replicas"`
+	// LastError is its last failure to give a verdict, if any.
+	LastError   string     `json:"last_error,omitempty"`
+	LastErrorAt *time.Time `json:"last_error_at,omitempty"`
+	// Problem says, in words, what is wrong and what to do; empty when
+	// nothing is.
+	Problem string `json:"problem,omitempty"`
+}
+
+// judgeErrorFresh is how recent a judge's failure must be to count as a
+// problem now.
+const judgeErrorFresh = 5 * time.Minute
+
+// Judges are the judge models the policies in force ask, each with what is
+// wrong with it, if anything.
+func (a *API) Judges() []JudgeView {
+	if a.guard == nil || a.guard.Library == nil {
+		return nil
+	}
+	lib := a.guard.Library()
+	byModel := map[string]*JudgeView{}
+	for name, p := range lib.Policies {
+		if p.Judge == nil || p.Judge.Model == "" {
+			continue
+		}
+		v := byModel[p.Judge.Model]
+		if v == nil {
+			v = &JudgeView{Model: p.Judge.Model}
+			byModel[p.Judge.Model] = v
+		}
+		v.Policies = append(v.Policies, name)
+		if p.Fail == "closed" && p.Mode != "shadow" {
+			v.Closed = append(v.Closed, name)
+		}
+	}
+	ready := map[string]int{}
+	for _, rep := range a.ctl.Reconciler().Replicas() {
+		if rep.State == reconciler.Ready {
+			ready[rep.Model]++
+		}
+	}
+	var errs map[string]guard.JudgeError
+	if a.guard.JudgeErrors != nil {
+		errs = a.guard.JudgeErrors()
+	}
+	out := make([]JudgeView, 0, len(byModel))
+	for _, v := range byModel {
+		sort.Strings(v.Policies)
+		sort.Strings(v.Closed)
+		if m, ok := a.ctl.Catalog().Resolve(v.Model); ok && m.Name == v.Model {
+			v.InCatalog, v.Judge = true, m.IsGuard()
+			v.Wanted, v.Ready = m.Scaling.Min, ready[m.Name]
+		}
+		if e, ok := errs[v.Model]; ok {
+			at := e.At
+			v.LastError, v.LastErrorAt = e.Err, &at
+		}
+		v.Problem = judgeProblem(*v)
+		out = append(out, *v)
+	}
+	slices.SortFunc(out, func(x, y JudgeView) int {
+		switch {
+		case x.Model < y.Model:
+			return -1
+		case x.Model > y.Model:
+			return 1
+		}
+		return 0
+	})
+	return out
+}
+
+func judgeProblem(v JudgeView) string {
+	consequence := "requests its policies judge are served unjudged"
+	if len(v.Closed) > 0 {
+		consequence = "requests its policies judge are refused (guard_unavailable)"
+	}
+	switch {
+	case !v.InCatalog:
+		return fmt.Sprintf("not in the catalog, so %s: llmgw ctl guard judge enable", consequence)
+	case !v.Judge:
+		return fmt.Sprintf("in the catalog as a model clients use, not as a judge (role: guard), so %s: "+
+			"llmgw ctl guard judge enable --replace", consequence)
+	case v.Ready == 0:
+		return fmt.Sprintf("no replica ready, so %s: llmgw ctl replicas --model %s", consequence, v.Model)
+	case v.LastErrorAt != nil && time.Since(*v.LastErrorAt) < judgeErrorFresh:
+		return fmt.Sprintf("gave no verdict %s ago (%s)", time.Since(*v.LastErrorAt).Round(time.Second), v.LastError)
+	}
+	return ""
+}
+
+func (a *API) listJudges(w http.ResponseWriter, _ *http.Request) {
+	if a.guard == nil {
+		writeError(w, http.StatusNotImplemented, "no guard policies: define them under guard in the cluster's settings (llmgw ctl config edit)")
+		return
+	}
+	writeJSON(w, http.StatusOK, a.Judges())
+}

internal/control/adminapi/judges_test.go

@@ -0,0 +1,66 @@
+package adminapi
+
+import (
+	"strings"
+	"testing"
+	"time"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
+)
+
+func TestEachJudgeSaysWhatIsWrongWithIt(t *testing.T) {
+	models := append(testModels(),
+		// Served from the mirror page before it knew better: a model
+		// clients use, under the judge's name.
+		catalog.Model{Name: "shield", Source: "/s", Scaling: catalog.Scaling{Min: 1, Max: 1},
+			Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
+		catalog.Model{Name: "llama-guard-3-1b", Source: "/g", Role: catalog.RoleGuard,
+			Guard: &catalog.GuardModel{Format: "llamaguard3"}, Scaling: catalog.Scaling{Min: 2, Max: 2},
+			Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
+	a, _ := setup(t, models)
+	lib := guard.Config{Default: []string{"baseline"}, Policies: map[string]guard.Policy{
+		"baseline": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}},
+		"strict":   {Fail: "closed", Judge: &guard.JudgePolicy{Model: "missing-judge", Input: "block"}},
+		"shield":   {Fail: "open", Judge: &guard.JudgePolicy{Model: "shield", Input: "flag"}},
+	}}
+	a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib }})
+
+	var js []JudgeView
+	if code := call(t, a, "GET", "/api/v1/guard/judges", "", &js); code != 200 || len(js) != 3 {
+		t.Fatalf("%d %+v", code, js)
+	}
+	by := map[string]JudgeView{}
+	for _, j := range js {
+		by[j.Model] = j
+	}
+	if j := by["missing-judge"]; j.InCatalog || !strings.Contains(j.Problem, "not in the catalog") ||
+		!strings.Contains(j.Problem, "refused") {
+		t.Errorf("missing: %+v", j)
+	}
+	if j := by["shield"]; !j.InCatalog || j.Judge || !strings.Contains(j.Problem, "--replace") ||
+		!strings.Contains(j.Problem, "unjudged") {
+		t.Errorf("not a judge: %+v", j)
+	}
+	if j := by["llama-guard-3-1b"]; !j.Judge || j.Wanted != 2 || !strings.Contains(j.Problem, "no replica ready") {
+		t.Errorf("not running: %+v", j)
+	}
+
+	var s Summary
+	call(t, a, "GET", "/api/v1/summary", "", &s)
+	if len(s.Judges) != 3 {
+		t.Errorf("the summary does not carry the judges' problems: %+v", s.Judges)
+	}
+
+	// A judge that failed lately says so, once it is otherwise fine.
+	at := time.Now()
+	v := JudgeView{Model: "x", InCatalog: true, Judge: true, Ready: 1, LastError: "deadline exceeded", LastErrorAt: &at}
+	if p := judgeProblem(v); !strings.Contains(p, "deadline exceeded") {
+		t.Errorf("a recent failure: %q", p)
+	}
+	old := at.Add(-time.Hour)
+	v.LastErrorAt = &old
+	if p := judgeProblem(v); p != "" {
+		t.Errorf("an old failure still counts: %q", p)
+	}
+}

internal/control/adminapi/routes.go

@@ -254,6 +254,8 @@ var Routes = []Route{
 			{Name: "since", Description: "How far back: 90m, 24h, 7d; 24h when absent."},
 		},
 		Response: GuardEventsView{}},
+	{Method: "GET", Path: "/api/v1/guard/judges", Summary: "The judge models the policies ask, and what is wrong with each",
+		Response: []JudgeView{}},
 	{Method: "POST", Path: "/api/v1/guard/attachments", Summary: "Attach a guardrail policy",
 		Request: GuardAttachmentRequest{}, Response: guard.Attachment{}},
 	{Method: "DELETE", Path: "/api/v1/guard/attachments", Summary: "Detach a guardrail policy",

internal/control/adminapi/summary.go

@@ -42,6 +42,8 @@ type Summary struct {
 	// ControlPlanes is the cluster's control planes when there is more than
 	// one: which leads, which stand by, and the floating address in front.
 	ControlPlanes ControlPlanes `json:"control_planes,omitzero"`
+	// Judges are the guardrails' judge models with something wrong.
+	Judges []JudgeView `json:"judges,omitempty"`
 }
 
 // StoreMember is one member of a replicated store, as its Patroni reports it.
@@ -149,6 +151,11 @@ const idleThreshold = 10 * time.Minute
 // Summary builds the fleet summary.
 func (a *API) Summary() Summary {
 	s := Summary{Version: version.Version}
+	for _, j := range a.Judges() {
+		if j.Problem != "" {
+			s.Judges = append(s.Judges, j)
+		}
+	}
 	if a.storeStatus != nil {
 		members, err := a.storeStatus()
 		s.Store = members