The judges and the starter divide a machine's CPUs instead of each taking all
CI / test (push) Failing after 30m7s

Measured on the three-VM cluster with ha1's CPUs loaded: 38 of 60 requests
refused with both judges and the starter on ha1, each engine on all four
cores. With the judges moved to a 4-core, 32 GB node at 2 cores each, 2 of
60 were refused, both the first after the judges started.

init now reserves cores by this machine's count: one per judge replica
under 8 CPUs and two from 8, and the rest, up to 8, for the starter on CPU.
The starter reserves nothing when nothing is left, so it still fits. Judges
bound for nodes not yet known get two each. A judge reserves 4 GB; it uses
about 3.2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-16 13:00:39 +02:00
1 parent 0b984a92bc
commit 63b2c6d9a4
5 files changed
+98 -12

No files matched your search

+7
View File
@@ -87,6 +87,13 @@ APIs change between minors without a deprecation period.
### Fixed
- **The judges and the starter model divide a machine's CPUs instead of each taking all of them.** `server init` gave the
judge and the starter no cores of their own, so every llama.cpp engine ran on every core: two judges and the starter
on a 4-core control plane competed, and under load 38 of 60 requests were refused for want of a judgment. Moved to a
node of their own with 2 cores each, the same test refused 2. When this machine is the node, init now gives each judge
replica 1 core (2 from 8 cores) and the starter what is left (at most 8); judges bound for other nodes get 2 each. A
judge also reserves 4 GB, not 2: it uses about 3.2.
- **A lost store machine stops costing requests most of a minute.** The store's connections try each member for 3 seconds,
not until the kernel gives up, and a connection to a machine that is gone is closed within about 10 seconds (TCP
keepalive, and `TCP_USER_TIMEOUT` on Linux). Patroni replaces a lost primary within 20 seconds (`ttl` 20, `loop_wait`
+3
View File
@@ -120,6 +120,9 @@ type serverInit struct {
// whichever leads.
standbys bool
vip string
// cpus is this machine's CPU count for dividing it between init's
// models; zero asks the machine. A seam for tests.
cpus int
// pgSetup prepares PostgreSQL and returns the store address; a seam for
// tests. Nil does it for real.
pgSetup func(context.Context) (string, error)
+31 -2
View File
@@ -81,13 +81,13 @@ func TestAStarterIsChosenByNameOrByItsFile(t *testing.T) {
if _, err := starterFrom("gpt-5"); err == nil {
t.Error("a model nobody offers was taken")
}
m := starterModel(s, 8192, "http://x/p.gguf", []string{"6000-24g"}, true)
m := starterModel(s, 8192, 0, "http://x/p.gguf", []string{"6000-24g"}, true)
if m.Name != "phi-4-q4_k_m" || m.ContextLength != 8192 || len(m.Aliases) != 0 ||
!slices.Equal(m.Profiles[0].Args, []string{"--ctx-size", "32768"}) {
t.Errorf("entry %+v", m)
}
big, _ := starterFrom("qwen3-14b")
if m := starterModel(big, 16384, "http://x/q.gguf", nil, true); m.Profiles[0].MinMemoryGB != 11 ||
if m := starterModel(big, 16384, 0, "http://x/q.gguf", nil, true); m.Profiles[0].MinMemoryGB != 11 ||
!slices.Equal(m.Profiles[0].Args, []string{"--ctx-size", "65536"}) {
t.Errorf("a 9 GB starter asks for %d GB, %v", m.Profiles[0].MinMemoryGB, m.Profiles[0].Args)
}
@@ -177,3 +177,32 @@ func TestTwoClustersOnOneNetworkKeepApartFloatingAddresses(t *testing.T) {
t.Errorf("conf:\n%s", conf)
}
}
func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T) {
for _, c := range []struct {
in serverInit
judge, starter int
}{
{serverInit{cpus: 4}, 1, 2},
{serverInit{cpus: 8}, 2, 4},
{serverInit{cpus: 32}, 2, 8},
{serverInit{cpus: 2}, 1, 0}, // nothing left: the starter shares rather than fits nowhere
{serverInit{cpus: 1}, 0, 0},
{serverInit{cpus: 4, noGuardModel: true}, 1, 4},
{serverInit{cpus: 8, judge: judgeSpec{Replicas: 3}}, 2, 2},
{serverInit{cpus: 4, noNode: true}, 2, 0}, // on nodes not known yet
} {
got := c.in.cpuPlan()
if got.judge != c.judge || got.starter != c.starter {
t.Errorf("%+v: judge %d starter %d, want %d %d", c.in, got.judge, got.starter, c.judge, c.starter)
}
}
m := guardModel("http://x/g.gguf", 2, 1)
if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 4 {
t.Errorf("judge profile: %d cores, %d GB", p.CPUCores, p.MinMemoryGB)
}
s := starterModel(starterChoices[0], defaultContext, 2, "http://x/q.gguf", []string{"6000-24g"}, true)
if cpu := s.Profiles[len(s.Profiles)-1]; cpu.Name != "cpu" || cpu.CPUCores != 2 || s.Profiles[0].CPUCores != 0 {
t.Errorf("starter: GPU profile %d cores, CPU %+v", s.Profiles[0].CPUCores, cpu)
}
}
+53 -6
View File
@@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
@@ -448,7 +449,7 @@ func hasNVIDIADevice() bool {
// starterModel is the catalog entry: a GPU profile first when there is a card,
// and a CPU profile always, both pinned to this machine. ctx is the context a
// client may use; each engine holds four times that, for four at once.
func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, pinned bool) catalog.Model {
func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []string, pinned bool) catalog.Model {
var pin map[string]string
if pinned {
pin = map[string]string{localLabelKey: localLabelValue}
@@ -462,7 +463,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string,
})
}
profiles = append(profiles, catalog.Profile{
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), NodeSelector: pin,
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), CPUCores: cores, NodeSelector: pin,
Args: []string{"--ctx-size", strconv.Itoa(max(16384, 4*ctx))},
})
var aliases []string
@@ -484,7 +485,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string,
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
// the models people use, on whichever node has room; replicas copies of it.
func guardModel(source string, replicas int) catalog.Model {
func guardModel(source string, replicas, cores int) catalog.Model {
return catalog.Model{
Name: guardModelName,
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
@@ -495,7 +496,9 @@ func guardModel(source string, replicas int) catalog.Model {
// judge that is not there refuses everything.
Scaling: catalog.Scaling{Min: replicas, Max: replicas},
Profiles: []catalog.Profile{{
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2,
// It uses about 3.2 GB with its context: reserving 2 filled nodes
// past their memory.
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 4, CPUCores: cores,
Args: []string{"--ctx-size", "8192"},
}},
}
@@ -508,14 +511,18 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
pinned := !in.noNode
spec, n := in.starterModelSpec(), in.starterContext()
return in.ensureServed(ctx, spec.Name, spec.Source, starterVersion, pinned,
func(url string, classes []string) catalog.Model { return starterModel(spec, n, url, classes, pinned) })
func(url string, classes []string) catalog.Model {
return starterModel(spec, n, in.cpuPlan().starter, url, classes, pinned)
})
}
// ensureGuardModel fetches the guardrails' judge model into the mirror and
// adds it to the catalog, to run on any node with room.
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
func(url string, _ []string) catalog.Model { return guardModel(url, in.judgeFor().Replicas) })
func(url string, _ []string) catalog.Model {
return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge)
})
}
// ensureServed fetches a model into the mirror, adds it to the catalog, and
@@ -699,3 +706,43 @@ func waitFor(ctx context.Context, limit time.Duration, what string, done func()
}
}
}
// cpuSplit is the cores init's models reserve: each judge replica's, and
// the starter's on CPU.
type cpuSplit struct{ judge, starter int }
// cpuPlan divides the CPUs between the judges and the starter, so that the
// engines do not each take every core. Without it, two judge replicas and
// the starter on one 4-core control plane all ran on all four: under load
// most judgments missed their budget and the requests were refused.
//
// When this machine is the node, by its own CPUs: a core a judge under 8,
// two from 8, and the starter what is left, up to 8, or no reservation at
// all when nothing is, rather than a starter that fits nowhere. Otherwise
// two a judge, on whichever node takes it, and the starter, on a node not
// known yet, none.
func (in *serverInit) cpuPlan() cpuSplit {
if in.noNode {
return cpuSplit{judge: 2}
}
n := in.cpus
if n <= 0 {
n = runtime.NumCPU()
}
if n < 2 {
return cpuSplit{}
}
judges := 0
if !in.noGuardModel {
judges = in.judgeFor().Replicas
}
judge := 1
if n >= 8 {
judge = 2
}
left := n - judges*judge
if left < 1 {
left = 0
}
return cpuSplit{judge: judge, starter: min(left, 8)}
}
+4 -4
View File
@@ -28,7 +28,7 @@ func TestTheInitAgentUnitIsTheShippedUnit(t *testing.T) {
}
func TestTheStarterUsesTheGPUAndFallsBackToCPUOnThisMachine(t *testing.T) {
m := starterModel(starterChoices[0], defaultContext, "http://10.0.0.5:8081/qwen2.5-0.5b-instruct/starter/q.gguf", []string{"4090-24g"}, true)
m := starterModel(starterChoices[0], defaultContext, 0, "http://10.0.0.5:8081/qwen2.5-0.5b-instruct/starter/q.gguf", []string{"4090-24g"}, true)
c, err := catalog.New([]catalog.Model{m})
if err != nil {
t.Fatalf("the starter is not a valid catalog entry: %v", err)
@@ -53,7 +53,7 @@ func TestTheStarterUsesTheGPUAndFallsBackToCPUOnThisMachine(t *testing.T) {
t.Error("the starter should be running, not waiting to be asked for")
}
if m := starterModel(starterChoices[0], defaultContext, "http://x/q.gguf", nil, true); len(m.Profiles) != 1 || m.Profiles[0].Name != "cpu" {
if m := starterModel(starterChoices[0], defaultContext, 0, "http://x/q.gguf", nil, true); len(m.Profiles) != 1 || m.Profiles[0].Name != "cpu" {
t.Errorf("without a GPU only the CPU profile: %+v", m.Profiles)
}
}
@@ -96,7 +96,7 @@ func TestTheMirrorUnitAdvertisesAnAddressNodesCanReach(t *testing.T) {
}
func TestWithNoNodeHereTheStarterGoesToTheFirstNodeThatJoins(t *testing.T) {
m := starterModel(starterChoices[0], defaultContext, "http://x/q.gguf", nil, false)
m := starterModel(starterChoices[0], defaultContext, 0, "http://x/q.gguf", nil, false)
for _, p := range m.Profiles {
if len(p.NodeSelector) > 0 {
t.Errorf("profile %s is pinned to a machine that is not a node: %v", p.Name, p.NodeSelector)
@@ -171,7 +171,7 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
}
func TestTheGuardModelIsAValidJudge(t *testing.T) {
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2)
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2)
c, err := catalog.New([]catalog.Model{m})
if err != nil {
t.Fatalf("not a valid catalog entry: %v", err)