A judge the leader serves by itself is sized as init's is
CI / test (push) Successful in 5m13s

A judge a policy names and the catalog lacked was served with 3 GB, no
cores and llama.cpp's prompt cache; on a busy node its judgments missed
their budget and fail: closed refused the requests. It now gets init's
2 GB and cores, and every judge modelserve serves runs with
--cache-ram 0.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 16:32:04 +02:00
1 parent 99c34fd68f
commit 8b122aae09
8 files changed
+70 -3

No files matched your search

+9
View File
@@ -110,6 +110,15 @@ func (k *judgeKeeper) check() {
}
k.tried[name] = now()
var opts judgeServeOptions
if !kj.GPU {
// The cores init would give the same number of judges.
var cpus []int
if k.nodeCPUs != nil {
cpus = k.nodeCPUs()
}
opts.Replicas = 2
opts.Cores = refitCores(opts.Replicas, cpus)
}
if kj.GPU {
if opts.GPUClass = roomiestGPUClass(k.gpus()); opts.GPUClass == "" {
k.log.Warn("a policy names a judge that runs on a GPU, and this cluster has none",
+5 -1
View File
@@ -91,8 +91,12 @@ func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) {
lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}}
inCatalog = nil
k.gpus = func() []adminapi.ClassUse { return nil }
k.nodeCPUs = func() []int { return []int{8, 4} }
k.check()
if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 {
// Sized as init sizes its two judges: 2 GB, and two cores each on a
// cluster whose smallest node has four.
if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 ||
last.MemoryGB != judgeMemoryGB || last.CPUCores != refitCores(2, []int{8, 4}) || last.CPUCores == 0 {
t.Errorf("CPU judge: %+v", last)
}
fs.err = errors.New("the mirror is down")
+3 -1
View File
@@ -142,7 +142,9 @@ func (k knownJudge) serveRequest(o judgeServeOptions) modelserve.Request {
}
return req
}
req.CPU, req.CPUCores = true, o.Cores
// Sized as init sizes its judge: the memory it measured to need with
// no prompt cache, and the cores its rule gives.
req.CPU, req.CPUCores, req.MemoryGB = true, o.Cores, judgeMemoryGB
if req.Replicas == 0 {
req.Replicas = 2
}