The judges and the starter divide a machine's CPUs instead of each taking all
Measured on the three-VM cluster with ha1's CPUs loaded: 38 of 60 requests refused with both judges and the starter on ha1, each engine on all four cores. With the judges moved to a 4-core, 32 GB node at 2 cores each, 2 of 60 were refused, both the first after the judges started. init now reserves cores by this machine's count: one per judge replica under 8 CPUs and two from 8, and the rest, up to 8, for the starter on CPU. The starter reserves nothing when nothing is left, so it still fits. Judges bound for nodes not yet known get two each. A judge reserves 4 GB; it uses about 3.2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0b984a92bc
commit
63b2c6d9a4
5 files changed
+98
-12
No files matched your search
@@ -9,6 +9,7 @@ import (
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -448,7 +449,7 @@ func hasNVIDIADevice() bool {
|
||||
// starterModel is the catalog entry: a GPU profile first when there is a card,
|
||||
// and a CPU profile always, both pinned to this machine. ctx is the context a
|
||||
// client may use; each engine holds four times that, for four at once.
|
||||
func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string, pinned bool) catalog.Model {
|
||||
func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []string, pinned bool) catalog.Model {
|
||||
var pin map[string]string
|
||||
if pinned {
|
||||
pin = map[string]string{localLabelKey: localLabelValue}
|
||||
@@ -462,7 +463,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string,
|
||||
})
|
||||
}
|
||||
profiles = append(profiles, catalog.Profile{
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), NodeSelector: pin,
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: spec.minMemoryGB(), CPUCores: cores, NodeSelector: pin,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(max(16384, 4*ctx))},
|
||||
})
|
||||
var aliases []string
|
||||
@@ -484,7 +485,7 @@ func starterModel(spec starterSpec, ctx int, source string, gpuClasses []string,
|
||||
|
||||
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
|
||||
// the models people use, on whichever node has room; replicas copies of it.
|
||||
func guardModel(source string, replicas int) catalog.Model {
|
||||
func guardModel(source string, replicas, cores int) catalog.Model {
|
||||
return catalog.Model{
|
||||
Name: guardModelName,
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
@@ -495,7 +496,9 @@ func guardModel(source string, replicas int) catalog.Model {
|
||||
// judge that is not there refuses everything.
|
||||
Scaling: catalog.Scaling{Min: replicas, Max: replicas},
|
||||
Profiles: []catalog.Profile{{
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 2,
|
||||
// It uses about 3.2 GB with its context: reserving 2 filled nodes
|
||||
// past their memory.
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: 4, CPUCores: cores,
|
||||
Args: []string{"--ctx-size", "8192"},
|
||||
}},
|
||||
}
|
||||
@@ -508,14 +511,18 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
pinned := !in.noNode
|
||||
spec, n := in.starterModelSpec(), in.starterContext()
|
||||
return in.ensureServed(ctx, spec.Name, spec.Source, starterVersion, pinned,
|
||||
func(url string, classes []string) catalog.Model { return starterModel(spec, n, url, classes, pinned) })
|
||||
func(url string, classes []string) catalog.Model {
|
||||
return starterModel(spec, n, in.cpuPlan().starter, url, classes, pinned)
|
||||
})
|
||||
}
|
||||
|
||||
// ensureGuardModel fetches the guardrails' judge model into the mirror and
|
||||
// adds it to the catalog, to run on any node with room.
|
||||
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
|
||||
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
|
||||
func(url string, _ []string) catalog.Model { return guardModel(url, in.judgeFor().Replicas) })
|
||||
func(url string, _ []string) catalog.Model {
|
||||
return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge)
|
||||
})
|
||||
}
|
||||
|
||||
// ensureServed fetches a model into the mirror, adds it to the catalog, and
|
||||
@@ -699,3 +706,43 @@ func waitFor(ctx context.Context, limit time.Duration, what string, done func()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// cpuSplit is the cores init's models reserve: each judge replica's, and
|
||||
// the starter's on CPU.
|
||||
type cpuSplit struct{ judge, starter int }
|
||||
|
||||
// cpuPlan divides the CPUs between the judges and the starter, so that the
|
||||
// engines do not each take every core. Without it, two judge replicas and
|
||||
// the starter on one 4-core control plane all ran on all four: under load
|
||||
// most judgments missed their budget and the requests were refused.
|
||||
//
|
||||
// When this machine is the node, by its own CPUs: a core a judge under 8,
|
||||
// two from 8, and the starter what is left, up to 8, or no reservation at
|
||||
// all when nothing is, rather than a starter that fits nowhere. Otherwise
|
||||
// two a judge, on whichever node takes it, and the starter, on a node not
|
||||
// known yet, none.
|
||||
func (in *serverInit) cpuPlan() cpuSplit {
|
||||
if in.noNode {
|
||||
return cpuSplit{judge: 2}
|
||||
}
|
||||
n := in.cpus
|
||||
if n <= 0 {
|
||||
n = runtime.NumCPU()
|
||||
}
|
||||
if n < 2 {
|
||||
return cpuSplit{}
|
||||
}
|
||||
judges := 0
|
||||
if !in.noGuardModel {
|
||||
judges = in.judgeFor().Replicas
|
||||
}
|
||||
judge := 1
|
||||
if n >= 8 {
|
||||
judge = 2
|
||||
}
|
||||
left := n - judges*judge
|
||||
if left < 1 {
|
||||
left = 0
|
||||
}
|
||||
return cpuSplit{judge: judge, starter: min(left, 8)}
|
||||
}
|
||||
Reference in new issue
Block a user