The judges and the starter divide a machine's CPUs instead of each taking all
CI / test (push) Failing after 30m7s

Measured on the three-VM cluster with ha1's CPUs loaded: 38 of 60 requests
refused with both judges and the starter on ha1, each engine on all four
cores. With the judges moved to a 4-core, 32 GB node at 2 cores each, 2 of
60 were refused, both the first after the judges started.

init now reserves cores by this machine's count: one per judge replica
under 8 CPUs and two from 8, and the rest, up to 8, for the starter on CPU.
The starter reserves nothing when nothing is left, so it still fits. Judges
bound for nodes not yet known get two each. A judge reserves 4 GB; it uses
about 3.2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-16 13:00:39 +02:00
1 parent 0b984a92bc
commit 63b2c6d9a4
5 files changed
+98 -12

No files matched your search

+53 -6
View File
@@ -9,6 +9,7 @@ import (
"os"
"os/exec"
"path/filepath"
"runtime"
"slices"
"strconv"
"strings"
@@ -448,7 +449,7 @@ func hasNVIDIADevice() bool {
// starterModel is the catalog entry: a GPU profile first when there is a card,
// and a CPU profile always, both pinned to this machine. ctx is the context a
// client may use; each engine holds four times that, for four at once.
func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, pinned bool) catalog.Model {
func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []string, pinned bool) catalog.Model {
var pin map[string]string
if pinned {
pin = map[string]string{localLabelKey: localLabelValue}
@@ -462,7 +463,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string,
})
}
profiles = append(profiles, catalog.Profile{
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), NodeSelector: pin,
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), CPUCores: cores, NodeSelector: pin,
Args: []string{"--ctx-size", strconv.Itoa(max(16384, 4*ctx))},
})
var aliases []string
@@ -484,7 +485,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string,
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
// the models people use, on whichever node has room; replicas copies of it.
func guardModel(source string, replicas int) catalog.Model {
func guardModel(source string, replicas, cores int) catalog.Model {
return catalog.Model{
Name: guardModelName,
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
@@ -495,7 +496,9 @@ func guardModel(source string, replicas int) catalog.Model {
// judge that is not there refuses everything.
Scaling: catalog.Scaling{Min: replicas, Max: replicas},
Profiles: []catalog.Profile{{
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2,
// It uses about 3.2 GB with its context: reserving 2 filled nodes
// past their memory.
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 4, CPUCores: cores,
Args: []string{"--ctx-size", "8192"},
}},
}
@@ -508,14 +511,18 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
pinned := !in.noNode
spec, n := in.starterModelSpec(), in.starterContext()
return in.ensureServed(ctx, spec.Name, spec.Source, starterVersion, pinned,
func(url string, classes []string) catalog.Model { return starterModel(spec, n, url, classes, pinned) })
func(url string, classes []string) catalog.Model {
return starterModel(spec, n, in.cpuPlan().starter, url, classes, pinned)
})
}
// ensureGuardModel fetches the guardrails' judge model into the mirror and
// adds it to the catalog, to run on any node with room.
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
func(url string, _ []string) catalog.Model { return guardModel(url, in.judgeFor().Replicas) })
func(url string, _ []string) catalog.Model {
return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge)
})
}
// ensureServed fetches a model into the mirror, adds it to the catalog, and
@@ -699,3 +706,43 @@ func waitFor(ctx context.Context, limit time.Duration, what string, done func()
}
}
}
// cpuSplit is the cores init's models reserve: each judge replica's, and
// the starter's on CPU.
type cpuSplit struct{ judge, starter int }
// cpuPlan divides the CPUs between the judges and the starter, so that the
// engines do not each take every core. Without it, two judge replicas and
// the starter on one 4-core control plane all ran on all four: under load
// most judgments missed their budget and the requests were refused.
//
// When this machine is the node, by its own CPUs: a core a judge under 8,
// two from 8, and the starter what is left, up to 8, or no reservation at
// all when nothing is, rather than a starter that fits nowhere. Otherwise
// two a judge, on whichever node takes it, and the starter, on a node not
// known yet, none.
func (in *serverInit) cpuPlan() cpuSplit {
if in.noNode {
return cpuSplit{judge: 2}
}
n := in.cpus
if n <= 0 {
n = runtime.NumCPU()
}
if n < 2 {
return cpuSplit{}
}
judges := 0
if !in.noGuardModel {
judges = in.judgeFor().Replicas
}
judge := 1
if n >= 8 {
judge = 2
}
left := n - judges*judge
if left < 1 {
left = 0
}
return cpuSplit{judge: judge, starter: min(left, 8)}
}