diff --git a/CHANGELOG.md b/CHANGELOG.md index a23998e..13d806e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,21 @@ is only released once its **exit criterion** is met, not when its code lands. Until v1.0.0 the minor version is the breaking one: configuration formats and Go APIs change between minors without a deprecation period. +## Unreleased + +### Fixed + +- **Both judges fit an 8 GB machine again.** v0.14.6 had `server init` reserve 4 GB a judge replica. That figure came + from one replica at 3.2 GB after a day of traffic. The growth was llama.cpp's prompt cache, up to 8 GB by default, + which a judge never reuses: each text it reads is different. On a 4-core, 8 GB machine the second judge then fitted + nowhere ("has 7 GB of memory with 1 free, needs 4"). The judge now runs with `--cache-ram 0`, uses about 1.6 GB and + stays there, and reserves 2. +- **A judge gets the two cores it needs.** v0.14.6 gave each judge replica one core on machines under eight. A judgment + of about 280 tokens takes 8.0 s on one core, 4.2 s on two and 2.3 s on four, against the 5 s budget, so on a 4-core + machine every judgment timed out and every request was refused. A judge replica now reserves two cores, one only + below four CPUs. On a 4-core machine the starter reserves none and shares what is left. A judge on two cores + stays close to its budget: give it more cores or a GPU, or raise `latency_budget`, where requests must not wait. + ## v0.14.6 — 2026-10-16 Setup that says what each part of an install is and lets every one be diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index 2c025ec..3175776 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -183,12 +183,12 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T) in serverInit judge, starter int }{ - {serverInit{cpus: 4}, 1, 2}, + {serverInit{cpus: 4}, 2, 0}, // two a judge, and the starter shares what is left {serverInit{cpus: 8}, 2, 4}, {serverInit{cpus: 32}, 2, 8}, - {serverInit{cpus: 2}, 1, 0}, // nothing left: the starter shares rather than fits nowhere + {serverInit{cpus: 2}, 1, 0}, // below four, one each, so both fit {serverInit{cpus: 1}, 0, 0}, - {serverInit{cpus: 4, noGuardModel: true}, 1, 4}, + {serverInit{cpus: 4, noGuardModel: true}, 2, 4}, {serverInit{cpus: 8, judge: judgeSpec{Replicas: 3}}, 2, 2}, {serverInit{cpus: 4, noNode: true}, 2, 0}, // on nodes not known yet } { @@ -198,8 +198,8 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T) } } m := guardModel("http://x/g.gguf", 2, 1) - if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 4 { - t.Errorf("judge profile: %d cores, %d GB", p.CPUCores, p.MinMemoryGB) + if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 2 || !slices.Contains(p.Args, "--cache-ram") { + t.Errorf("judge profile: %d cores, %d GB, args %v", p.CPUCores, p.MinMemoryGB, p.Args) } s := starterModel(starterChoices[0], defaultContext, 2, "http://x/q.gguf", []string{"6000-24g"}, true) if cpu := s.Profiles[len(s.Profiles)-1]; cpu.Name != "cpu" || cpu.CPUCores != 2 || s.Profiles[0].CPUCores != 0 { diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index 98c847c..6d5d181 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -496,10 +496,13 @@ func guardModel(source string, replicas, cores int) catalog.Model { // judge that is not there refuses everything. Scaling: catalog.Scaling{Min: replicas, Max: replicas}, Profiles: []catalog.Profile{{ - // It uses about 3.2 GB with its context: reserving 2 filled nodes - // past their memory. - Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 4, CPUCores: cores, - Args: []string{"--ctx-size", "8192"}, + // About 1.6 GB with its context, which stays so with no prompt + // cache: llama.cpp keeps up to 8 GB of prompts by default, and a + // judge's, each a different text, are not asked again. With the + // cache one grew to 3.2 GB over a day of traffic; reserving 4 for + // that left a 7 GB machine room for only one of the two. + Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: judgeMemoryGB, CPUCores: cores, + Args: []string{"--ctx-size", "8192", "--cache-ram", "0"}, }}, } } @@ -716,14 +719,16 @@ type cpuSplit struct{ judge, starter int } // the starter on one 4-core control plane all ran on all four: under load // most judgments missed their budget and the requests were refused. // -// When this machine is the node, by its own CPUs: a core a judge under 8, -// two from 8, and the starter what is left, up to 8, or no reservation at -// all when nothing is, rather than a starter that fits nowhere. Otherwise -// two a judge, on whichever node takes it, and the starter, on a node not -// known yet, none. +// A judge needs two cores: a judgment of about 280 tokens took 8.0 s on +// one, 4.2 s on two and 2.3 s on four, against a 5 s budget. When this +// machine is the node, each judge replica gets two (one each only below four +// CPUs, where two would not fit both), and the starter what is left, up to +// 8, or no reservation at all when nothing is, rather than a starter that +// fits nowhere. Otherwise two a judge, on whichever node takes it, and the +// starter, on a node not known yet, none. func (in *serverInit) cpuPlan() cpuSplit { if in.noNode { - return cpuSplit{judge: 2} + return cpuSplit{judge: judgeCores} } n := in.cpus if n <= 0 { @@ -736,13 +741,18 @@ func (in *serverInit) cpuPlan() cpuSplit { if !in.noGuardModel { judges = in.judgeFor().Replicas } - judge := 1 - if n >= 8 { - judge = 2 - } - left := n - judges*judge - if left < 1 { - left = 0 + judge := judgeCores + if judges > 0 && judges*judge > n { + judge = max(1, n/judges) } + left := max(n-judges*judge, 0) return cpuSplit{judge: judge, starter: min(left, 8)} } + +// judgeCores and judgeMemoryGB are what a judge replica needs: two cores +// for a judgment within its budget, and 2 GB for its 1.6 with no prompt +// cache. +const ( + judgeCores = 2 + judgeMemoryGB = 2 +) diff --git a/internal/cli/tui_choices.go b/internal/cli/tui_choices.go index a520aaa..ab3376f 100644 --- a/internal/cli/tui_choices.go +++ b/internal/cli/tui_choices.go @@ -282,7 +282,7 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta "The request is served unjudged and marked so in the logs. Nothing is refused for the judge's sake.", } f.judgeCopies = textSetupField("Copies", strconv.Itoa(j.Replicas), "2", - "Each takes about 1 GB and two CPU threads on whichever node has room.") + "Each reserves 2 GB and two CPU cores on whichever node has room: on one core a judgment takes longer than its budget.") f.judgeBudget = textSetupField("Waits at most", durationYAML(j.Budget), "5s", "A short text is judged in about two seconds on CPUs.") judgeFields := []*setupField{f.judgeFail, f.judgeCopies, f.judgeBudget, headingField("Refuses")}