diff --git a/CHANGELOG.md b/CHANGELOG.md index 28c0405..7a1a5b1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -87,6 +87,13 @@ APIs change between minors without a deprecation period. ### Fixed +- **The judges and the starter model divide a machine's CPUs instead of each taking all of them.** `server init` gave the + judge and the starter no cores of their own, so every llama.cpp engine ran on every core: two judges and the starter + on a 4-core control plane competed, and under load 38 of 60 requests were refused for want of a judgment. Moved to a + node of their own with 2 cores each, the same test refused 2. When this machine is the node, init now gives each judge + replica 1 core (2 from 8 cores) and the starter what is left (at most 8); judges bound for other nodes get 2 each. A + judge also reserves 4 GB, not 2: it uses about 3.2. + - **A lost store machine stops costing requests most of a minute.** The store's connections try each member for 3 seconds, not until the kernel gives up, and a connection to a machine that is gone is closed within about 10 seconds (TCP keepalive, and `TCP_USER_TIMEOUT` on Linux). Patroni replaces a lost primary within 20 seconds (`ttl` 20, `loop_wait` diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index c0e679f..e051f79 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -120,6 +120,9 @@ type serverInit struct { // whichever leads. standbys bool vip string + // cpus is this machine's CPU count for dividing it between init's + // models; zero asks the machine. A seam for tests. + cpus int // pgSetup prepares PostgreSQL and returns the store address; a seam for // tests. Nil does it for real. pgSetup func(context.Context) (string, error) diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index 6407ef3..2c025ec 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -81,13 +81,13 @@ func TestAStarterIsChosenByNameOrByItsFile(t *testing.T) { if _, err := starterFrom("gpt-5"); err == nil { t.Error("a model nobody offers was taken") } - m := starterModel(s, 8192, "http://x/p.gguf", []string{"6000-24g"}, true) + m := starterModel(s, 8192, 0, "http://x/p.gguf", []string{"6000-24g"}, true) if m.Name != "phi-4-q4_k_m" || m.ContextLength != 8192 || len(m.Aliases) != 0 || !slices.Equal(m.Profiles[0].Args, []string{"--ctx-size", "32768"}) { t.Errorf("entry %+v", m) } big, _ := starterFrom("qwen3-14b") - if m := starterModel(big, 16384, "http://x/q.gguf", nil, true); m.Profiles[0].MinMemoryGB != 11 || + if m := starterModel(big, 16384, 0, "http://x/q.gguf", nil, true); m.Profiles[0].MinMemoryGB != 11 || !slices.Equal(m.Profiles[0].Args, []string{"--ctx-size", "65536"}) { t.Errorf("a 9 GB starter asks for %d GB, %v", m.Profiles[0].MinMemoryGB, m.Profiles[0].Args) } @@ -177,3 +177,32 @@ func TestTwoClustersOnOneNetworkKeepApartFloatingAddresses(t *testing.T) { t.Errorf("conf:\n%s", conf) } } + +func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T) { + for _, c := range []struct { + in serverInit + judge, starter int + }{ + {serverInit{cpus: 4}, 1, 2}, + {serverInit{cpus: 8}, 2, 4}, + {serverInit{cpus: 32}, 2, 8}, + {serverInit{cpus: 2}, 1, 0}, // nothing left: the starter shares rather than fits nowhere + {serverInit{cpus: 1}, 0, 0}, + {serverInit{cpus: 4, noGuardModel: true}, 1, 4}, + {serverInit{cpus: 8, judge: judgeSpec{Replicas: 3}}, 2, 2}, + {serverInit{cpus: 4, noNode: true}, 2, 0}, // on nodes not known yet + } { + got := c.in.cpuPlan() + if got.judge != c.judge || got.starter != c.starter { + t.Errorf("%+v: judge %d starter %d, want %d %d", c.in, got.judge, got.starter, c.judge, c.starter) + } + } + m := guardModel("http://x/g.gguf", 2, 1) + if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 4 { + t.Errorf("judge profile: %d cores, %d GB", p.CPUCores, p.MinMemoryGB) + } + s := starterModel(starterChoices[0], defaultContext, 2, "http://x/q.gguf", []string{"6000-24g"}, true) + if cpu := s.Profiles[len(s.Profiles)-1]; cpu.Name != "cpu" || cpu.CPUCores != 2 || s.Profiles[0].CPUCores != 0 { + t.Errorf("starter: GPU profile %d cores, CPU %+v", s.Profiles[0].CPUCores, cpu) + } +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index f047d85..98c847c 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "slices" "strconv" "strings" @@ -448,7 +449,7 @@ func hasNVIDIADevice() bool { // starterModel is the catalog entry: a GPU profile first when there is a card, // and a CPU profile always, both pinned to this machine. ctx is the context a // client may use; each engine holds four times that, for four at once. -func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, pinned bool) catalog.Model { +func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []string, pinned bool) catalog.Model { var pin map[string]string if pinned { pin = map[string]string{localLabelKey: localLabelValue} @@ -462,7 +463,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, }) } profiles = append(profiles, catalog.Profile{ - Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), NodeSelector: pin, + Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), CPUCores: cores, NodeSelector: pin, Args: []string{"--ctx-size", strconv.Itoa(max(16384, 4*ctx))}, }) var aliases []string @@ -484,7 +485,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, // guardModel is the judge's catalog entry: CPU only, so it takes no card from // the models people use, on whichever node has room; replicas copies of it. -func guardModel(source string, replicas int) catalog.Model { +func guardModel(source string, replicas, cores int) catalog.Model { return catalog.Model{ Name: guardModelName, Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.", @@ -495,7 +496,9 @@ func guardModel(source string, replicas int) catalog.Model { // judge that is not there refuses everything. Scaling: catalog.Scaling{Min: replicas, Max: replicas}, Profiles: []catalog.Profile{{ - Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2, + // It uses about 3.2 GB with its context: reserving 2 filled nodes + // past their memory. + Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 4, CPUCores: cores, Args: []string{"--ctx-size", "8192"}, }}, } @@ -508,14 +511,18 @@ func (in *serverInit) ensureStarter(ctx context.Context) error { pinned := !in.noNode spec, n := in.starterModelSpec(), in.starterContext() return in.ensureServed(ctx, spec.Name, spec.Source, starterVersion, pinned, - func(url string, classes []string) catalog.Model { return starterModel(spec, n, url, classes, pinned) }) + func(url string, classes []string) catalog.Model { + return starterModel(spec, n, in.cpuPlan().starter, url, classes, pinned) + }) } // ensureGuardModel fetches the guardrails' judge model into the mirror and // adds it to the catalog, to run on any node with room. func (in *serverInit) ensureGuardModel(ctx context.Context) error { return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false, - func(url string, _ []string) catalog.Model { return guardModel(url, in.judgeFor().Replicas) }) + func(url string, _ []string) catalog.Model { + return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge) + }) } // ensureServed fetches a model into the mirror, adds it to the catalog, and @@ -699,3 +706,43 @@ func waitFor(ctx context.Context, limit time.Duration, what string, done func() } } } + +// cpuSplit is the cores init's models reserve: each judge replica's, and +// the starter's on CPU. +type cpuSplit struct{ judge, starter int } + +// cpuPlan divides the CPUs between the judges and the starter, so that the +// engines do not each take every core. Without it, two judge replicas and +// the starter on one 4-core control plane all ran on all four: under load +// most judgments missed their budget and the requests were refused. +// +// When this machine is the node, by its own CPUs: a core a judge under 8, +// two from 8, and the starter what is left, up to 8, or no reservation at +// all when nothing is, rather than a starter that fits nowhere. Otherwise +// two a judge, on whichever node takes it, and the starter, on a node not +// known yet, none. +func (in *serverInit) cpuPlan() cpuSplit { + if in.noNode { + return cpuSplit{judge: 2} + } + n := in.cpus + if n <= 0 { + n = runtime.NumCPU() + } + if n < 2 { + return cpuSplit{} + } + judges := 0 + if !in.noGuardModel { + judges = in.judgeFor().Replicas + } + judge := 1 + if n >= 8 { + judge = 2 + } + left := n - judges*judge + if left < 1 { + left = 0 + } + return cpuSplit{judge: judge, starter: min(left, 8)} +} diff --git a/internal/cli/serverinit_cluster_test.go b/internal/cli/serverinit_cluster_test.go index fa15400..401ca0f 100644 --- a/internal/cli/serverinit_cluster_test.go +++ b/internal/cli/serverinit_cluster_test.go @@ -28,7 +28,7 @@ func TestTheInitAgentUnitIsTheShippedUnit(t *testing.T) { } func TestTheStarterUsesTheGPUAndFallsBackToCPUOnThisMachine(t *testing.T) { - m := starterModel(starterChoices[0], defaultContext, "http://10.0.0.5:8081/qwen2.5-0.5b-instruct/starter/q.gguf", []string{"4090-24g"}, true) + m := starterModel(starterChoices[0], defaultContext, 0, "http://10.0.0.5:8081/qwen2.5-0.5b-instruct/starter/q.gguf", []string{"4090-24g"}, true) c, err := catalog.New([]catalog.Model{m}) if err != nil { t.Fatalf("the starter is not a valid catalog entry: %v", err) @@ -53,7 +53,7 @@ func TestTheStarterUsesTheGPUAndFallsBackToCPUOnThisMachine(t *testing.T) { t.Error("the starter should be running, not waiting to be asked for") } - if m := starterModel(starterChoices[0], defaultContext, "http://x/q.gguf", nil, true); len(m.Profiles) != 1 || m.Profiles[0].Name != "cpu" { + if m := starterModel(starterChoices[0], defaultContext, 0, "http://x/q.gguf", nil, true); len(m.Profiles) != 1 || m.Profiles[0].Name != "cpu" { t.Errorf("without a GPU only the CPU profile: %+v", m.Profiles) } } @@ -96,7 +96,7 @@ func TestTheMirrorUnitAdvertisesAnAddressNodesCanReach(t *testing.T) { } func TestWithNoNodeHereTheStarterGoesToTheFirstNodeThatJoins(t *testing.T) { - m := starterModel(starterChoices[0], defaultContext, "http://x/q.gguf", nil, false) + m := starterModel(starterChoices[0], defaultContext, 0, "http://x/q.gguf", nil, false) for _, p := range m.Profiles { if len(p.NodeSelector) > 0 { t.Errorf("profile %s is pinned to a machine that is not a node: %v", p.Name, p.NodeSelector) @@ -171,7 +171,7 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) { } func TestTheGuardModelIsAValidJudge(t *testing.T) { - m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2) + m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2) c, err := catalog.New([]catalog.Model{m}) if err != nil { t.Fatalf("not a valid catalog entry: %v", err)