A judge the leader serves by itself is sized as init's is

A judge a policy names and the catalog lacked was served with 3 GB, no
cores and llama.cpp's prompt cache; on a busy node its judgments missed
their budget and fail: closed refused the requests. It now gets init's
2 GB and cores, and every judge modelserve serves runs with
--cache-ram 0.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

BACKLOG.md

@@ -31,7 +31,8 @@ What is left, most needed first. The detail is in the item each names.
    Still to do: try it against a real domain controller (a Samba AD DC
    on Proxmox): LDAPS, nested groups, a disabled user suspended by the
    sync.
-6. **J2** — a judge the leader adds by itself is sized as init's is. (S)
+6. ~~**J2**~~ — done 2026-10-19: a judge the leader adds by itself is
+   sized as init's is (2 GB, init's cores, no prompt cache).
 7. **J3** — HA setup puts the judges on nodes of their own. (S)
 8. **J8** — a judge budget that fits the machine, measured at install. (S)
 9. **F5** — large imports limited by the embedding engine's slots. (M)

CHANGELOG.md

@@ -24,6 +24,16 @@ is only released once its **exit criterion** is met, not when its code lands.
 Until v1.0.0 the minor version is the breaking one: configuration formats and Go
 APIs change between minors without a deprecation period.
 
+## Unreleased
+
+### Fixed
+
+- **A judge the leader serves by itself is sized as init's is.** When a guard policy names a judge the catalog lacks,
+  the leader serves it. That judge reserved 3 GB and no cores, and ran llama.cpp's prompt cache, which grew one to
+  3.2 GB. Without cores of its own, a judgment on a busy node missed its 5 s budget, and `fail: closed` refused the
+  request. It now gets what `server init` gives its judges: 2 GB, two cores each (fewer on a small node), and no
+  prompt cache. Every judge served through `modelserve` runs with `--cache-ram 0`.
+
 ## v0.14.9 — 2026-10-19
 
 Sign-in set up from the console, with Active Directory tried before it is

docs/admin-api.openapi.json

@@ -10769,6 +10769,10 @@
           "cpu_cores": {
             "type": "integer",
             "description": "CPUCores is what a replica computes on; zero takes half the smallest node it fits on."
+          },
+          "memory_gb": {
+            "type": "integer",
+            "description": "MemoryGB is what a replica on CPUs reserves, in place of the figure worked out from its weights and context: a judge's, whose own sizing is known."
           }
         }
       },

internal/cli/judgekeeper.go

@@ -110,6 +110,15 @@ func (k *judgeKeeper) check() {
 		}
 		k.tried[name] = now()
 		var opts judgeServeOptions
+		if !kj.GPU {
+			// The cores init would give the same number of judges.
+			var cpus []int
+			if k.nodeCPUs != nil {
+				cpus = k.nodeCPUs()
+			}
+			opts.Replicas = 2
+			opts.Cores = refitCores(opts.Replicas, cpus)
+		}
 		if kj.GPU {
 			if opts.GPUClass = roomiestGPUClass(k.gpus()); opts.GPUClass == "" {
 				k.log.Warn("a policy names a judge that runs on a GPU, and this cluster has none",

internal/cli/judgekeeper_test.go

@@ -91,8 +91,12 @@ func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) {
 	lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}}
 	inCatalog = nil
 	k.gpus = func() []adminapi.ClassUse { return nil }
+	k.nodeCPUs = func() []int { return []int{8, 4} }
 	k.check()
-	if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 {
+	// Sized as init sizes its two judges: 2 GB, and two cores each on a
+	// cluster whose smallest node has four.
+	if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 ||
+		last.MemoryGB != judgeMemoryGB || last.CPUCores != refitCores(2, []int{8, 4}) || last.CPUCores == 0 {
 		t.Errorf("CPU judge: %+v", last)
 	}
 	fs.err = errors.New("the mirror is down")

internal/cli/judges_known.go

@@ -142,7 +142,9 @@ func (k knownJudge) serveRequest(o judgeServeOptions) modelserve.Request {
 		}
 		return req
 	}
-	req.CPU, req.CPUCores = true, o.Cores
+	// Sized as init sizes its judge: the memory it measured to need with
+	// no prompt cache, and the cores its rule gives.
+	req.CPU, req.CPUCores, req.MemoryGB = true, o.Cores, judgeMemoryGB
 	if req.Replicas == 0 {
 		req.Replicas = 2
 	}

internal/control/modelserve/modelserve.go

@@ -163,6 +163,10 @@ type Request struct {
 	// CPUCores is what a replica computes on; zero takes half the smallest
 	// node it fits on.
 	CPUCores int `json:"cpu_cores,omitempty"`
+	// MemoryGB is what a replica on CPUs reserves, in place of the figure
+	// worked out from its weights and context: a judge's, whose own sizing
+	// is known.
+	MemoryGB int `json:"memory_gb,omitempty"`
 	// By is who asked, for the catalog's record.
 	By string `json:"-"`
 }
@@ -518,6 +522,9 @@ func (s *Server) run(j *Job) {
 		memGB = int(gb(set.Bytes)*1.5) + 2
 		basis = "weights plus half again plus two; the header gave no shape"
 	}
+	if req.MemoryGB > 0 {
+		memGB, basis = req.MemoryGB, "as asked, in place of "+basis
+	}
 	s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis))
 
 	cores := req.CPUCores
@@ -631,6 +638,12 @@ func llamaArgs(req Request) []string {
 		n := strconv.Itoa(req.ContextLength)
 		args = append([]string{"--embedding"}, append(args, "--batch-size", n, "--ubatch-size", n)...)
 	}
+	if req.GuardFormat != "" {
+		// No prompt cache: a judge's prompts, each a different text, are
+		// not asked again, and llama.cpp's default of up to 8 GB grew one
+		// judge to 3.2 GB over a day.
+		args = append(args, "--cache-ram", "0")
+	}
 	return args
 }
 

internal/control/modelserve/modelserve_test.go

@@ -428,3 +428,27 @@ func TestAPoolTheCardCannotHoldIsShrunkNotStarted(t *testing.T) {
 		t.Errorf("a pool that fits was changed: %+v %q", got, why)
 	}
 }
+
+// A judge is sized as asked, not from its weights, and runs with no prompt
+// cache: what `server init` gives its own.
+func TestAJudgeIsServedWithTheMemoryAskedAndNoPromptCache(t *testing.T) {
+	files := []hf.File{{Path: "Qwen3Guard-Gen-0.6B-Q8_0.gguf", Bytes: 600 << 20, SHA256: "aa"}}
+	s, _, c, _ := setup(t, files, nil)
+	job, err := s.Start(Request{Repo: "example/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B-Q8_0.gguf",
+		GuardFormat: "qwen3guard", CPU: true, CPUCores: 2, MemoryGB: 2, Replicas: 2, CtxPool: 8192, ContextLength: 4096})
+	if err != nil {
+		t.Fatal(err)
+	}
+	wait(t, s, job.ID)
+	if len(c.added) != 1 {
+		t.Fatalf("added %+v", c.added)
+	}
+	p := c.added[0].Profiles[0]
+	if p.MinMemoryGB != 2 || p.CPUCores != 2 || !strings.Contains(strings.Join(p.Args, " "), "--cache-ram 0") {
+		t.Errorf("profile %+v", p)
+	}
+	// Any other model keeps llama.cpp's cache.
+	if strings.Contains(strings.Join(llamaArgs(Request{CtxPool: 8192, ContextLength: 4096}), " "), "cache-ram") {
+		t.Error("a model that is not a judge lost its prompt cache")
+	}
+}