The guardrails' judge has its own command, and its trouble is shown

llmgw ctl guard judge status lists each judge model the policies ask:
in the catalog, as a judge, replicas ready, its last failure to answer, and
what is wrong in words. enable serves the missing one through model serve -
Llama Guard 3 1B on CPUs, role guard, under the name the policies give it -
with --replace for a model of that name that is not a judge; the console has
it under Do. GET /api/v1/guard/judges, and the summary's judges, put the
problem on ctl status and the console's Overview.

A Llama Guard file served from the mirror page is added as a judge, under
the name the policies ask, rather than as a model clients use the policies
never find. ctl says plainly when the API answers with a page.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/cli/ctl.go

@@ -116,6 +116,13 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
 	if out == nil {
 		return nil
 	}
+	// An older control plane answers a path it does not know with its
+	// dashboard: a page, which read as JSON said only "invalid character '<'".
+	if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
+		return fmt.Errorf("%s %s answered with a page, not the API: the control plane answering "+
+			"(the leader, through a standby) does not know it — is it on an older release? "+
+			"llmgw ctl status says which it runs", method, path)
+	}
 	return json.NewDecoder(resp.Body).Decode(out)
 }
 

internal/cli/ctl_config_test.go

@@ -0,0 +1,22 @@
+package cli
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+)
+
+func TestAPageInsteadOfTheAPIIsSaidPlainly(t *testing.T) {
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+		_, _ = w.Write([]byte("<!doctype html><title>llmgw</title>"))
+	}))
+	defer srv.Close()
+	c := newCtlClient(strings.TrimPrefix(srv.URL, "http://"), "")
+	var out map[string]any
+	err := c.get(t.Context(), "/api/v1/settings", &out)
+	if err == nil || !strings.Contains(err.Error(), "older release") || strings.Contains(err.Error(), "invalid character") {
+		t.Errorf("%v", err)
+	}
+}

internal/cli/guard.go

@@ -24,6 +24,7 @@ func newGuardCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 			"Targets are written org:Acme, team:Acme/ml, project:Acme/ml/training (or\n" +
 			"with IDs), key:KEY-ID or user:USER-ID. An attachment applies at once.",
 	}
+	c.AddCommand(newGuardJudgeCmd(client, asJSON))
 	c.AddCommand(&cobra.Command{
 		Use:   "list",
 		Short: "The policies, and where each is attached",

internal/cli/guard_judge.go

@@ -0,0 +1,189 @@
+package cli
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/spf13/cobra"
+
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
+)
+
+// The judge the guardrails are installed with: Meta's Llama Guard 3 1B,
+// from the repository server init fetches it from.
+const (
+	guardModelRepo = "QuantFactory/Llama-Guard-3-1B-GGUF"
+	guardModelFile = "Llama-Guard-3-1B.Q4_K_M.gguf"
+)
+
+func newGuardJudgeCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	c := &cobra.Command{
+		Use:   "judge",
+		Short: "The judge models the policies ask: whether each can answer, and serving one",
+		Long: "A policy with a judge has a model read what patterns cannot. The judge must\n" +
+			"be in the catalog under the name the policy gives, as a judge (role:\n" +
+			"guard), with a replica ready; otherwise what the policy judges is refused\n" +
+			"(fail: closed, guard_unavailable) or served unjudged (fail: open).",
+	}
+	c.AddCommand(newGuardJudgeStatusCmd(client, asJSON), newGuardJudgeEnableCmd(client))
+	return c
+}
+
+func newGuardJudgeStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
+	return &cobra.Command{
+		Use:   "status",
+		Short: "Each judge the policies ask, and what is wrong with it",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			var js []adminapi.JudgeView
+			if err := client().get(cmd.Context(), "/api/v1/guard/judges", &js); err != nil {
+				return err
+			}
+			if *asJSON {
+				return emitJSON(js)
+			}
+			out := cmd.OutOrStdout()
+			if len(js) == 0 {
+				fmt.Fprintln(out, "no policy asks a judge")
+				return nil
+			}
+			w := tabTo(out)
+			fmt.Fprintln(w, "JUDGE\tPOLICIES\tIN CATALOG\tAS JUDGE\tREADY\tSTATE")
+			for _, j := range js {
+				state := "answering"
+				if j.Problem != "" {
+					state = j.Problem
+				}
+				fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%d/%d\t%s\n", j.Model, strings.Join(j.Policies, ","),
+					yesNo(j.InCatalog), yesNo(j.Judge), j.Ready, j.Wanted, state)
+			}
+			if err := w.Flush(); err != nil {
+				return err
+			}
+			strays, _ := strayJudges(cmd, client, js)
+			for _, s := range strays {
+				fmt.Fprintf(out, "\n%s looks like a judge's weights but is served as a model clients use, and no "+
+					"policy asks it: llmgw ctl model remove %s\n", s, s)
+			}
+			return nil
+		},
+	}
+}
+
+// strayJudges are models whose weights are a judge's but which are in the
+// catalog as models clients use, under a name no policy asks: what serving
+// Llama Guard from the mirror page did before it knew better.
+func strayJudges(cmd *cobra.Command, client func() *ctlClient, js []adminapi.JudgeView) ([]string, error) {
+	var models []adminapi.ModelView
+	if err := client().get(cmd.Context(), "/api/v1/models", &models); err != nil {
+		return nil, err
+	}
+	asked := map[string]bool{}
+	for _, j := range js {
+		asked[j.Model] = true
+	}
+	var out []string
+	for _, m := range models {
+		if !asked[m.Name] && strings.Contains(strings.ToLower(m.Source), "llama-guard") {
+			out = append(out, m.Name)
+		}
+	}
+	return out, nil
+}
+
+func newGuardJudgeEnableCmd(client func() *ctlClient) *cobra.Command {
+	var (
+		model, repo, file string
+		replicas, cores   int
+		replace, detach   bool
+	)
+	c := &cobra.Command{
+		Use:   "enable",
+		Short: "Serve the judge the policies ask, as a judge: Llama Guard 3 1B on CPUs",
+		Long: "Fetches Llama Guard 3 1B into the mirror (once), adds it to the catalog under\n" +
+			"the name the policies give its judge, as a judge (role: guard,\n" +
+			"format llamaguard3), on CPUs so it takes no card, and waits for a replica\n" +
+			"to answer. With no --model, the judge the policies ask that is missing.\n\n" +
+			"A model of that name already in the catalog as a model clients use is\n" +
+			"refused; --replace removes it first. An offline cluster gets its judge\n" +
+			"from its bundle with server init.",
+		Example: "  llmgw ctl guard judge enable\n" +
+			"  llmgw ctl guard judge enable --replace",
+		Args: cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, _ []string) error {
+			ctx := cmd.Context()
+			out := cmd.OutOrStdout()
+			var js []adminapi.JudgeView
+			if err := client().get(ctx, "/api/v1/guard/judges", &js); err != nil {
+				return err
+			}
+			var target *adminapi.JudgeView
+			for i, j := range js {
+				if (model != "" && j.Model == model) || (model == "" && (!j.InCatalog || !j.Judge)) {
+					target = &js[i]
+					break
+				}
+			}
+			switch {
+			case target == nil && model != "":
+				target = &adminapi.JudgeView{Model: model}
+				fmt.Fprintf(out, "no policy asks %s yet; it is served, and a policy's judge.model names it\n", model)
+			case target == nil && len(js) == 0:
+				return errors.New("no policy asks a judge: give one a judge (llmgw ctl config edit, guard.policies.<name>.judge.model)")
+			case target == nil:
+				fmt.Fprintln(out, "every judge the policies ask is in the catalog, as a judge")
+				return nil
+			}
+			if target.InCatalog && target.Judge {
+				fmt.Fprintf(out, "%s is in the catalog as a judge already; llmgw ctl guard judge status says how it is\n", target.Model)
+				return nil
+			}
+			if target.InCatalog && !target.Judge {
+				if !replace {
+					return fmt.Errorf("%s is in the catalog as a model clients use, not as a judge: "+
+						"--replace removes it and serves it again as a judge", target.Model)
+				}
+				var res map[string]string
+				if err := client().delete(ctx, "/api/v1/models/"+target.Model, &res); err != nil {
+					return fmt.Errorf("removing %s: %w", target.Model, err)
+				}
+				fmt.Fprintf(out, "%s removed\n", target.Model)
+			}
+			req := modelserve.Request{
+				Repo: repo, File: file, Name: target.Model, GuardFormat: "llamaguard3",
+				Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
+				CPU:         true, Replicas: replicas, CPUCores: cores,
+				CtxPool: 8192, ContextLength: 4096,
+			}
+			body, _ := json.Marshal(req)
+			var job modelserve.Job
+			if err := client().patient(30*time.Second).post(ctx, "/api/v1/models/serve", body, &job); err != nil {
+				return err
+			}
+			if detach {
+				return detached(cmd, job)
+			}
+			if err := followJob(cmd, client, job, "is the judge"); err != nil {
+				return err
+			}
+			if strays, _ := strayJudges(cmd, client, js); len(strays) > 0 {
+				fmt.Fprintf(out, "\nno policy asks %s, served from a judge's weights as a model clients use: "+
+					"llmgw ctl model remove %s\n", strings.Join(strays, ", "), strings.Join(strays, " "))
+			}
+			return nil
+		},
+	}
+	f := c.Flags()
+	f.StringVar(&model, "model", "", "the judge to serve (default: the one the policies ask that is missing)")
+	f.StringVar(&repo, "repo", guardModelRepo, "the registry repository its weights come from")
+	f.StringVar(&file, "file", guardModelFile, "the file in it")
+	f.IntVar(&replicas, "replicas", 2, "replicas: every judged request waits on one")
+	f.IntVar(&cores, "cores", 4, "CPU cores a replica computes on")
+	f.BoolVar(&replace, "replace", false, "remove a model of that name that is not a judge first")
+	f.BoolVar(&detach, "detach", false, "start it and return; llmgw ctl jobs follows it")
+	return c
+}

internal/cli/server.go

@@ -361,7 +361,8 @@ func newServerCmd() *cobra.Command {
 				evals = newEvalService(cfg, rt, admin, log)
 				api.SetEval(evals)
 				if st := rt.Store(); st != nil && gate != nil {
-					api.SetGuard(&adminapi.GuardAdmin{Library: gate.Library, Store: st, Invalidate: gate.Invalidate})
+					api.SetGuard(&adminapi.GuardAdmin{Library: gate.Library, Store: st, Invalidate: gate.Invalidate,
+						JudgeErrors: gate.JudgeErrors})
 				}
 				if st := rt.Store(); st != nil {
 					// Removing a node revokes its certificate.

internal/cli/status.go

@@ -153,6 +153,13 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
 		fmt.Fprintf(out, "STORE     %s\n", strings.Join(parts, ", "))
 	}
 
+	if len(s.Judges) > 0 {
+		fmt.Fprintln(out, "\nGUARDRAILS' JUDGE")
+		for _, j := range s.Judges {
+			fmt.Fprintf(out, "  %s: %s\n", j.Model, j.Problem)
+		}
+	}
+
 	if len(s.Parked) > 0 {
 		fmt.Fprintln(out, "\nPARKED — failed every time they started; not retried until fixed")
 		for _, p := range s.Parked {

internal/cli/tui_actions.go

@@ -679,6 +679,8 @@ func (m *consoleModel) doEntry() tea.Cmd {
 			}}
 	case cBackup:
 		return m.runAfter("server", "backup", "--config="+m.file)
+	case cJudge:
+		return m.runAfter("ctl", server, "guard", "judge", "enable", "--replace")
 	}
 	return nil
 }

internal/cli/tui_console.go

@@ -59,6 +59,7 @@ const (
 	cAddControl
 	cReplicate
 	cServe
+	cJudge
 	cUpgrade
 	cRestartAll
 	cBackup
@@ -68,7 +69,7 @@ const (
 )
 
 var consoleLabels = [cCount]string{"Overview", "Monitor", "Nodes", "GPUs", "Models", "Replicas", "Usage", "Quotas", "Guardrails", "Federation", "Mirror", "Jobs", "Logs", "Accounts", "API keys", "Organizations", "Audit", "Manual",
-	"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Upgrade the cluster",
+	"server.yaml", "Add a node", "Add a control plane", "Replicate the store", "Serve a model", "Serve the guardrails' judge", "Upgrade the cluster",
 	"Restart all replicas", "Back up now", "Wipe this machine", "Leave"}
 
 // consoleAction is what is done once the screen closes: each runs its own
@@ -496,7 +497,7 @@ func (m consoleModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
 				return m.leave()
 			case cWipe:
 				return m.askWipe()
-			case cUpgrade, cRestartAll, cBackup:
+			case cUpgrade, cRestartAll, cBackup, cJudge:
 				return m, m.doEntry()
 			case cOverview:
 				return m, nil
@@ -1299,6 +1300,9 @@ func (m consoleModel) panel() (title, body, keys string) {
 		item = "\n  Moves the store onto two machines under Patroni, replicated,\n  so losing one loses nothing.\n\n  " +
 			sDim.Render("On the command line:") +
 			"\n  sudo --preserve-env=SSH_AUTH_SOCK llmgw server init \\\n    --patroni <user>@<a> --patroni <user>@<b> --no-mirror --no-node --no-model"
+	case cJudge:
+		item = "\n  Serves the model the guardrail policies ask as their judge — Llama\n  Guard 3 1B on CPUs, under the name the policies give it — so what they\n  judge is not refused (guard_unavailable). A model of that name that is\n  not a judge is replaced. Overview says when a judge is missing.\n\n  " +
+			sDim.Render("On the command line:") + "\n  llmgw ctl guard judge status\n  llmgw ctl guard judge enable --replace"
 	case cUpgrade:
 		item = "\n  Moves the cluster to a release: this machine first, then each node\n  in turn. Engines keep serving; a node that does not come back stops it.\n\n  " +
 			sDim.Render("On the command line:") + "\n  sudo llmgw ctl upgrade VERSION"