A judge the leader serves by itself is sized as init's is

A judge a policy names and the catalog lacked was served with 3 GB, no
cores and llama.cpp's prompt cache; on a busy node its judgments missed
their budget and fail: closed refused the requests. It now gets init's
2 GB and cores, and every judge modelserve serves runs with
--cache-ram 0.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 16:32:04 +02:00
1 parent 99c34fd68f
commit 8b122aae09
8 files changed
+70 -3

No files matched your search

@@ -428,3 +428,27 @@ func TestAPoolTheCardCannotHoldIsShrunkNotStarted(t *testing.T) {
t.Errorf("a pool that fits was changed: %+v %q", got, why)
}
}
// A judge is sized as asked, not from its weights, and runs with no prompt
// cache: what `server init` gives its own.
func TestAJudgeIsServedWithTheMemoryAskedAndNoPromptCache(t *testing.T) {
files := []hf.File{{Path: "Qwen3Guard-Gen-0.6B-Q8_0.gguf", Bytes: 600 << 20, SHA256: "aa"}}
s, _, c, _ := setup(t, files, nil)
job, err := s.Start(Request{Repo: "example/Qwen3Guard-Gen-0.6B-GGUF", File: "Qwen3Guard-Gen-0.6B-Q8_0.gguf",
GuardFormat: "qwen3guard", CPU: true, CPUCores: 2, MemoryGB: 2, Replicas: 2, CtxPool: 8192, ContextLength: 4096})
if err != nil {
t.Fatal(err)
}
wait(t, s, job.ID)
if len(c.added) != 1 {
t.Fatalf("added %+v", c.added)
}
p := c.added[0].Profiles[0]
if p.MinMemoryGB != 2 || p.CPUCores != 2 || !strings.Contains(strings.Join(p.Args, " "), "--cache-ram 0") {
t.Errorf("profile %+v", p)
}
// Any other model keeps llama.cpp's cache.
if strings.Contains(strings.Join(llamaArgs(Request{CtxPool: 8192, ContextLength: 4096}), " "), "cache-ram") {
t.Error("a model that is not a judge lost its prompt cache")
}
}