Checked by a single-machine install of v0.14.6 on a 4-core, 8 GB VM. The second judge fitted nowhere: v0.14.6 reserved 4 GB a judge, a figure from one replica that grew to 3.2 GB through llama.cpp's prompt cache (up to 8 GB by default), which a judge never reuses. Once it fitted, every judgment timed out. v0.14.6 gave a judge one core below 8 CPUs, and a judgment of about 280 tokens takes 8.0 s on one core, 4.2 s on two and 2.3 s on four, against a 5 s budget: 0 of 20 requests went through. A judge now runs with --cache-ram 0, uses about 1.6 GB and reserves 2, and reserves two cores (one only below four CPUs). On the same VM, 19 of 20 went through. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0def6eaba1
commit
f8206c3540
4 files changed
+48
-23
No files matched your search
@@ -183,12 +183,12 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T)
|
||||
in serverInit
|
||||
judge, starter int
|
||||
}{
|
||||
{serverInit{cpus: 4}, 1, 2},
|
||||
{serverInit{cpus: 4}, 2, 0}, // two a judge, and the starter shares what is left
|
||||
{serverInit{cpus: 8}, 2, 4},
|
||||
{serverInit{cpus: 32}, 2, 8},
|
||||
{serverInit{cpus: 2}, 1, 0}, // nothing left: the starter shares rather than fits nowhere
|
||||
{serverInit{cpus: 2}, 1, 0}, // below four, one each, so both fit
|
||||
{serverInit{cpus: 1}, 0, 0},
|
||||
{serverInit{cpus: 4, noGuardModel: true}, 1, 4},
|
||||
{serverInit{cpus: 4, noGuardModel: true}, 2, 4},
|
||||
{serverInit{cpus: 8, judge: judgeSpec{Replicas: 3}}, 2, 2},
|
||||
{serverInit{cpus: 4, noNode: true}, 2, 0}, // on nodes not known yet
|
||||
} {
|
||||
@@ -198,8 +198,8 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T)
|
||||
}
|
||||
}
|
||||
m := guardModel("http://x/g.gguf", 2, 1)
|
||||
if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 4 {
|
||||
t.Errorf("judge profile: %d cores, %d GB", p.CPUCores, p.MinMemoryGB)
|
||||
if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 2 || !slices.Contains(p.Args, "--cache-ram") {
|
||||
t.Errorf("judge profile: %d cores, %d GB, args %v", p.CPUCores, p.MinMemoryGB, p.Args)
|
||||
}
|
||||
s := starterModel(starterChoices[0], defaultContext, 2, "http://x/q.gguf", []string{"6000-24g"}, true)
|
||||
if cpu := s.Profiles[len(s.Profiles)-1]; cpu.Name != "cpu" || cpu.CPUCores != 2 || s.Profiles[0].CPUCores != 0 {
|
||||
|
||||
Reference in new issue
Block a user