From 8b122aae098fe3ffc1cc88bea7fcd93f80c7998d Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 19 Oct 2026 16:32:04 +0200 Subject: [PATCH] A judge the leader serves by itself is sized as init's is A judge a policy names and the catalog lacked was served with 3 GB, no cores and llama.cpp's prompt cache; on a busy node its judgments missed their budget and fail: closed refused the requests. It now gets init's 2 GB and cores, and every judge modelserve serves runs with --cache-ram 0. Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 3 ++- CHANGELOG.md | 10 ++++++++ docs/admin-api.openapi.json | 4 ++++ internal/cli/judgekeeper.go | 9 +++++++ internal/cli/judgekeeper_test.go | 6 ++++- internal/cli/judges_known.go | 4 +++- internal/control/modelserve/modelserve.go | 13 ++++++++++ .../control/modelserve/modelserve_test.go | 24 +++++++++++++++++++ 8 files changed, 70 insertions(+), 3 deletions(-) diff --git a/BACKLOG.md b/BACKLOG.md index 406b239..1b3dec9 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -31,7 +31,8 @@ What is left, most needed first. The detail is in the item each names. Still to do: try it against a real domain controller (a Samba AD DC on Proxmox): LDAPS, nested groups, a disabled user suspended by the sync. -6. **J2** — a judge the leader adds by itself is sized as init's is. (S) +6. ~~**J2**~~ — done 2026-10-19: a judge the leader adds by itself is + sized as init's is (2 GB, init's cores, no prompt cache). 7. **J3** — HA setup puts the judges on nodes of their own. (S) 8. **J8** — a judge budget that fits the machine, measured at install. (S) 9. **F5** — large imports limited by the embedding engine's slots. (M) diff --git a/CHANGELOG.md b/CHANGELOG.md index 486abe4..f80b21f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,16 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. +## Unreleased + +### Fixed + +- **A judge the leader serves by itself is sized as init's is.** When a guard policy names a judge the catalog lacks, + the leader serves it. That judge reserved 3 GB and no cores, and ran llama.cpp's prompt cache, which grew one to + 3.2 GB. Without cores of its own, a judgment on a busy node missed its 5 s budget, and `fail: closed` refused the + request. It now gets what `server init` gives its judges: 2 GB, two cores each (fewer on a small node), and no + prompt cache. Every judge served through `modelserve` runs with `--cache-ram 0`. + ## v0.14.9 — 2026-10-19 Sign-in set up from the console, with Active Directory tried before it is diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 2b44d80..a6057f4 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -10769,6 +10769,10 @@ "cpu_cores": { "type": "integer", "description": "CPUCores is what a replica computes on; zero takes half the smallest node it fits on." + }, + "memory_gb": { + "type": "integer", + "description": "MemoryGB is what a replica on CPUs reserves, in place of the figure worked out from its weights and context: a judge's, whose own sizing is known." } } }, diff --git a/internal/cli/judgekeeper.go b/internal/cli/judgekeeper.go index 0eb4a45..648a70b 100644 --- a/internal/cli/judgekeeper.go +++ b/internal/cli/judgekeeper.go @@ -110,6 +110,15 @@ func (k *judgeKeeper) check() { } k.tried[name] = now() var opts judgeServeOptions + if !kj.GPU { + // The cores init would give the same number of judges. + var cpus []int + if k.nodeCPUs != nil { + cpus = k.nodeCPUs() + } + opts.Replicas = 2 + opts.Cores = refitCores(opts.Replicas, cpus) + } if kj.GPU { if opts.GPUClass = roomiestGPUClass(k.gpus()); opts.GPUClass == "" { k.log.Warn("a policy names a judge that runs on a GPU, and this cluster has none", diff --git a/internal/cli/judgekeeper_test.go b/internal/cli/judgekeeper_test.go index 9f4d0c5..256a03d 100644 --- a/internal/cli/judgekeeper_test.go +++ b/internal/cli/judgekeeper_test.go @@ -91,8 +91,12 @@ func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) { lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}} inCatalog = nil k.gpus = func() []adminapi.ClassUse { return nil } + k.nodeCPUs = func() []int { return []int{8, 4} } k.check() - if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 { + // Sized as init sizes its two judges: 2 GB, and two cores each on a + // cluster whose smallest node has four. + if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 || + last.MemoryGB != judgeMemoryGB || last.CPUCores != refitCores(2, []int{8, 4}) || last.CPUCores == 0 { t.Errorf("CPU judge: %+v", last) } fs.err = errors.New("the mirror is down") diff --git a/internal/cli/judges_known.go b/internal/cli/judges_known.go index d8354c5..74ccb62 100644 --- a/internal/cli/judges_known.go +++ b/internal/cli/judges_known.go @@ -142,7 +142,9 @@ func (k knownJudge) serveRequest(o judgeServeOptions) modelserve.Request { } return req } - req.CPU, req.CPUCores = true, o.Cores + // Sized as init sizes its judge: the memory it measured to need with + // no prompt cache, and the cores its rule gives. + req.CPU, req.CPUCores, req.MemoryGB = true, o.Cores, judgeMemoryGB if req.Replicas == 0 { req.Replicas = 2 } diff --git a/internal/control/modelserve/modelserve.go b/internal/control/modelserve/modelserve.go index fc04af4..87fc2a8 100644 --- a/internal/control/modelserve/modelserve.go +++ b/internal/control/modelserve/modelserve.go @@ -163,6 +163,10 @@ type Request struct { // CPUCores is what a replica computes on; zero takes half the smallest // node it fits on. CPUCores int `json:"cpu_cores,omitempty"` + // MemoryGB is what a replica on CPUs reserves, in place of the figure + // worked out from its weights and context: a judge's, whose own sizing + // is known. + MemoryGB int `json:"memory_gb,omitempty"` // By is who asked, for the catalog's record. By string `json:"-"` } @@ -518,6 +522,9 @@ func (s *Server) run(j *Job) { memGB = int(gb(set.Bytes)*1.5) + 2 basis = "weights plus half again plus two; the header gave no shape" } + if req.MemoryGB > 0 { + memGB, basis = req.MemoryGB, "as asked, in place of "+basis + } s.step(j, "check", Done, fmt.Sprintf("%s, needs about %d GB (%s)", orUnknown(header.Architecture), memGB, basis)) cores := req.CPUCores @@ -631,6 +638,12 @@ func llamaArgs(req Request) []string { n := strconv.Itoa(req.ContextLength) args = append([]string{"--embedding"}, append(args, "--batch-size", n, "--ubatch-size", n)...) } + if req.GuardFormat != "" { + // No prompt cache: a judge's prompts, each a different text, are + // not asked again, and llama.cpp's default of up to 8 GB grew one + // judge to 3.2 GB over a day. + args = append(args, "--cache-ram", "0") + } return args } diff --git a/internal/control/modelserve/modelserve_test.go b/internal/control/modelserve/modelserve_test.go index dd58c56..1bd9311 100644 --- a/internal/control/modelserve/modelserve_test.go +++ b/internal/control/modelserve/modelserve_test.go @@ -428,3 +428,27 @@ func TestAPoolTheCardCannotHoldIsShrunkNotStarted(t *testing.T) { t.Errorf("a pool that fits was changed: %+v %q", got, why) } } + +// A judge is sized as asked, not from its weights, and runs with no prompt +// cache: what `server init` gives its own. +func TestAJudgeIsServedWithTheMemoryAskedAndNoPromptCache(t *testing.T) { + files := []hf.File{{Path: "Qwen3Guard-Gen-0.6B-Q8_0.gguf", Bytes: 600 << 20, SHA256: "aa"}} + s, _, c, _ := setup(t, files, nil) + job, err := s.Start(Request{Repo: "example/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B-Q8_0.gguf", + GuardFormat: "qwen3guard", CPU: true, CPUCores: 2, MemoryGB: 2, Replicas: 2, CtxPool: 8192, ContextLength: 4096}) + if err != nil { + t.Fatal(err) + } + wait(t, s, job.ID) + if len(c.added) != 1 { + t.Fatalf("added %+v", c.added) + } + p := c.added[0].Profiles[0] + if p.MinMemoryGB != 2 || p.CPUCores != 2 || !strings.Contains(strings.Join(p.Args, " "), "--cache-ram 0") { + t.Errorf("profile %+v", p) + } + // Any other model keeps llama.cpp's cache. + if strings.Contains(strings.Join(llamaArgs(Request{CtxPool: 8192, ContextLength: 4096}), " "), "cache-ram") { + t.Error("a model that is not a judge lost its prompt cache") + } +}