From 63b2c6d9a460da9685fc9d701bf7dea870bb63d6 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Fri, 16 Oct 2026 13:00:39 +0200 Subject: [PATCH] The judges and the starter divide a machine's CPUs instead of each taking all Measured on the three-VM cluster with ha1's CPUs loaded: 38 of 60 requests refused with both judges and the starter on ha1, each engine on all four cores. With the judges moved to a 4-core, 32 GB node at 2 cores each, 2 of 60 were refused, both the first after the judges started. init now reserves cores by this machine's count: one per judge replica under 8 CPUs and two from 8, and the rest, up to 8, for the starter on CPU. The starter reserves nothing when nothing is left, so it still fits. Judges bound for nodes not yet known get two each. A judge reserves 4 GB; it uses about 3.2. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 7 +++ internal/cli/serverinit.go | 3 ++ internal/cli/serverinit_choices_test.go | 33 +++++++++++++- internal/cli/serverinit_cluster.go | 59 ++++++++++++++++++++++--- internal/cli/serverinit_cluster_test.go | 8 ++-- 5 files changed, 98 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 28c0405..7a1a5b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,13 @@ APIs change between minors without a deprecation period. ### Fixed +- **The judges and the starter model divide a machine's CPUs instead of each taking all of them.** `server init` gave the + judge and the starter no cores of their own, so every llama.cpp engine ran on every core: two judges and the starter + on a 4-core control plane competed, and under load 38 of 60 requests were refused for want of a judgment. Moved to a + node of their own with 2 cores each, the same test refused 2. When this machine is the node, init now gives each judge + replica 1 core (2 from 8 cores) and the starter what is left (at most 8); judges bound for other nodes get 2 each. A + judge also reserves 4 GB, not 2: it uses about 3.2. + - **A lost store machine stops costing requests most of a minute.** The store's connections try each member for 3 seconds, not until the kernel gives up, and a connection to a machine that is gone is closed within about 10 seconds (TCP keepalive, and `TCP_USER_TIMEOUT` on Linux). Patroni replaces a lost primary within 20 seconds (`ttl` 20, `loop_wait` diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index c0e679f..e051f79 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -120,6 +120,9 @@ type serverInit struct { // whichever leads. standbys bool vip string + // cpus is this machine's CPU count for dividing it between init's + // models; zero asks the machine. A seam for tests. + cpus int // pgSetup prepares PostgreSQL and returns the store address; a seam for // tests. Nil does it for real. pgSetup func(context.Context) (string, error) diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index 6407ef3..2c025ec 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -81,13 +81,13 @@ func TestAStarterIsChosenByNameOrByItsFile(t *testing.T) { if _, err := starterFrom("gpt-5"); err == nil { t.Error("a model nobody offers was taken") } - m := starterModel(s, 8192, "http://x/p.gguf", []string{"6000-24g"}, true) + m := starterModel(s, 8192, 0, "http://x/p.gguf", []string{"6000-24g"}, true) if m.Name != "phi-4-q4_k_m" || m.ContextLength != 8192 || len(m.Aliases) != 0 || !slices.Equal(m.Profiles[0].Args, []string{"--ctx-size", "32768"}) { t.Errorf("entry %+v", m) } big, _ := starterFrom("qwen3-14b") - if m := starterModel(big, 16384, "http://x/q.gguf", nil, true); m.Profiles[0].MinMemoryGB != 11 || + if m := starterModel(big, 16384, 0, "http://x/q.gguf", nil, true); m.Profiles[0].MinMemoryGB != 11 || !slices.Equal(m.Profiles[0].Args, []string{"--ctx-size", "65536"}) { t.Errorf("a 9 GB starter asks for %d GB, %v", m.Profiles[0].MinMemoryGB, m.Profiles[0].Args) } @@ -177,3 +177,32 @@ func TestTwoClustersOnOneNetworkKeepApartFloatingAddresses(t *testing.T) { t.Errorf("conf:\n%s", conf) } } + +func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T) { + for _, c := range []struct { + in serverInit + judge, starter int + }{ + {serverInit{cpus: 4}, 1, 2}, + {serverInit{cpus: 8}, 2, 4}, + {serverInit{cpus: 32}, 2, 8}, + {serverInit{cpus: 2}, 1, 0}, // nothing left: the starter shares rather than fits nowhere + {serverInit{cpus: 1}, 0, 0}, + {serverInit{cpus: 4, noGuardModel: true}, 1, 4}, + {serverInit{cpus: 8, judge: judgeSpec{Replicas: 3}}, 2, 2}, + {serverInit{cpus: 4, noNode: true}, 2, 0}, // on nodes not known yet + } { + got := c.in.cpuPlan() + if got.judge != c.judge || got.starter != c.starter { + t.Errorf("%+v: judge %d starter %d, want %d %d", c.in, got.judge, got.starter, c.judge, c.starter) + } + } + m := guardModel("http://x/g.gguf", 2, 1) + if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 4 { + t.Errorf("judge profile: %d cores, %d GB", p.CPUCores, p.MinMemoryGB) + } + s := starterModel(starterChoices[0], defaultContext, 2, "http://x/q.gguf", []string{"6000-24g"}, true) + if cpu := s.Profiles[len(s.Profiles)-1]; cpu.Name != "cpu" || cpu.CPUCores != 2 || s.Profiles[0].CPUCores != 0 { + t.Errorf("starter: GPU profile %d cores, CPU %+v", s.Profiles[0].CPUCores, cpu) + } +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index f047d85..98c847c 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -448,7 +449,7 @@ func hasNVIDIADevice() bool { // starterModel is the catalog entry: a GPU profile first when there is a card, // and a CPU profile always, both pinned to this machine. ctx is the context a // client may use; each engine holds four times that, for four at once. -func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, pinned bool) catalog.Model { +func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []string, pinned bool) catalog.Model { var pin map[string]string if pinned { pin = map[string]string{localLabelKey: localLabelValue} @@ -462,7 +463,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, }) } profiles = append(profiles, catalog.Profile{ - Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), NodeSelector: pin, + Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), CPUCores: cores, NodeSelector: pin, Args: []string{"--ctx-size", strconv.Itoa(max(16384, 4*ctx))}, }) var aliases []string @@ -484,7 +485,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, // guardModel is the judge's catalog entry: CPU only, so it takes no card from // the models people use, on whichever node has room; replicas copies of it. -func guardModel(source string, replicas int) catalog.Model { +func guardModel(source string, replicas, cores int) catalog.Model { return catalog.Model{ Name: guardModelName, Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.", @@ -495,7 +496,9 @@ func guardModel(source string, replicas int) catalog.Model { // judge that is not there refuses everything. Scaling: catalog.Scaling{Min: replicas, Max: replicas}, Profiles: []catalog.Profile{{ - Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2, + // It uses about 3.2 GB with its context: reserving 2 filled nodes + // past their memory. + Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 4, CPUCores: cores, Args: []string{"--ctx-size", "8192"}, }}, } @@ -508,14 +511,18 @@ func (in *serverInit) ensureStarter(ctx context.Context) error { pinned := !in.noNode spec, n := in.starterModelSpec(), in.starterContext() return in.ensureServed(ctx, spec.Name, spec.Source, starterVersion, pinned, - func(url string, classes []string) catalog.Model { return starterModel(spec, n, url, classes, pinned) }) + func(url string, classes []string) catalog.Model { + return starterModel(spec, n, in.cpuPlan().starter, url, classes, pinned) + }) } // ensureGuardModel fetches the guardrails' judge model into the mirror and // adds it to the catalog, to run on any node with room. func (in *serverInit) ensureGuardModel(ctx context.Context) error { return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false, - func(url string, _ []string) catalog.Model { return guardModel(url, in.judgeFor().Replicas) }) + func(url string, _ []string) catalog.Model { + return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge) + }) } // ensureServed fetches a model into the mirror, adds it to the catalog, and @@ -699,3 +706,43 @@ func waitFor(ctx context.Context, limit time.Duration, what string, done func() } } } + +// cpuSplit is the cores init's models reserve: each judge replica's, and +// the starter's on CPU. +type cpuSplit struct{ judge, starter int } + +// cpuPlan divides the CPUs between the judges and the starter, so that the +// engines do not each take every core. Without it, two judge replicas and +// the starter on one 4-core control plane all ran on all four: under load +// most judgments missed their budget and the requests were refused. +// +// When this machine is the node, by its own CPUs: a core a judge under 8, +// two from 8, and the starter what is left, up to 8, or no reservation at +// all when nothing is, rather than a starter that fits nowhere. Otherwise +// two a judge, on whichever node takes it, and the starter, on a node not +// known yet, none. +func (in *serverInit) cpuPlan() cpuSplit { + if in.noNode { + return cpuSplit{judge: 2} + } + n := in.cpus + if n <= 0 { + n = runtime.NumCPU() + } + if n < 2 { + return cpuSplit{} + } + judges := 0 + if !in.noGuardModel { + judges = in.judgeFor().Replicas + } + judge := 1 + if n >= 8 { + judge = 2 + } + left := n - judges*judge + if left < 1 { + left = 0 + } + return cpuSplit{judge: judge, starter: min(left, 8)} +} diff --git a/internal/cli/serverinit_cluster_test.go b/internal/cli/serverinit_cluster_test.go index fa15400..401ca0f 100644 --- a/internal/cli/serverinit_cluster_test.go +++ b/internal/cli/serverinit_cluster_test.go @@ -28,7 +28,7 @@ func TestTheInitAgentUnitIsTheShippedUnit(t *testing.T) { } func TestTheStarterUsesTheGPUAndFallsBackToCPUOnThisMachine(t *testing.T) { - m := starterModel(starterChoices[0], defaultContext, "http://10.0.0.5:8081/qwen2.5-0.5b-instruct/starter/q.gguf", []string{"4090-24g"}, true) + m := starterModel(starterChoices[0], defaultContext, 0, "http://10.0.0.5:8081/qwen2.5-0.5b-instruct/starter/q.gguf", []string{"4090-24g"}, true) c, err := catalog.New([]catalog.Model{m}) if err != nil { t.Fatalf("the starter is not a valid catalog entry: %v", err) @@ -53,7 +53,7 @@ func TestTheStarterUsesTheGPUAndFallsBackToCPUOnThisMachine(t *testing.T) { t.Error("the starter should be running, not waiting to be asked for") } - if m := starterModel(starterChoices[0], defaultContext, "http://x/q.gguf", nil, true); len(m.Profiles) != 1 || m.Profiles[0].Name != "cpu" { + if m := starterModel(starterChoices[0], defaultContext, 0, "http://x/q.gguf", nil, true); len(m.Profiles) != 1 || m.Profiles[0].Name != "cpu" { t.Errorf("without a GPU only the CPU profile: %+v", m.Profiles) } } @@ -96,7 +96,7 @@ func TestTheMirrorUnitAdvertisesAnAddressNodesCanReach(t *testing.T) { } func TestWithNoNodeHereTheStarterGoesToTheFirstNodeThatJoins(t *testing.T) { - m := starterModel(starterChoices[0], defaultContext, "http://x/q.gguf", nil, false) + m := starterModel(starterChoices[0], defaultContext, 0, "http://x/q.gguf", nil, false) for _, p := range m.Profiles { if len(p.NodeSelector) > 0 { t.Errorf("profile %s is pinned to a machine that is not a node: %v", p.Name, p.NodeSelector) @@ -171,7 +171,7 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) { } func TestTheGuardModelIsAValidJudge(t *testing.T) { - m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2) + m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2) c, err := catalog.New([]catalog.Model{m}) if err != nil { t.Fatalf("not a valid catalog entry: %v", err)