A judge a policy names and the catalog lacked was served with 3 GB, no cores and llama.cpp's prompt cache; on a busy node its judgments missed their budget and fail: closed refused the requests. It now gets init's 2 GB and cores, and every judge modelserve serves runs with --cache-ram 0. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
99c34fd68f
commit
8b122aae09
8 files changed
+70
-3
No files matched your search
@@ -91,8 +91,12 @@ func TestAJudgeAPolicyNamesIsFetchedWhenTheCatalogLacksIt(t *testing.T) {
|
||||
lib = guard.Config{Policies: map[string]guard.Policy{"b": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}}}
|
||||
inCatalog = nil
|
||||
k.gpus = func() []adminapi.ClassUse { return nil }
|
||||
k.nodeCPUs = func() []int { return []int{8, 4} }
|
||||
k.check()
|
||||
if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 {
|
||||
// Sized as init sizes its two judges: 2 GB, and two cores each on a
|
||||
// cluster whose smallest node has four.
|
||||
if last := fs.started[len(fs.started)-1]; last.Name != "qwen3guard-0.6b" || !last.CPU || last.Replicas != 2 ||
|
||||
last.MemoryGB != judgeMemoryGB || last.CPUCores != refitCores(2, []int{8, 4}) || last.CPUCores == 0 {
|
||||
t.Errorf("CPU judge: %+v", last)
|
||||
}
|
||||
fs.err = errors.New("the mirror is down")
|
||||
|
||||
Reference in new issue
Block a user