A three-machine install can give the judges machines of their own

The setup screen asks for up to two judge machines after the store's
two, and server init takes --judge-node for each. Each is added as a
node labelled llmgw.role: judge, and the judge's profile selects that
label, with cores by init's rule over those machines. A judge beside a
busy control plane refused 38 of 60 requests under load; alone, 2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 17:45:00 +02:00
1 parent 8b122aae09
commit 51ca09287f
12 files changed
+346 -44

No files matched your search

+5 -2
View File
@@ -485,7 +485,7 @@ func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
// the models people use, on whichever node has room; replicas copies of it.
func guardModel(source string, replicas, cores int) catalog.Model {
func guardModel(source string, replicas, cores int, selector map[string]string) catalog.Model {
return catalog.Model{
Name: guardModelName,
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
@@ -503,6 +503,8 @@ func guardModel(source string, replicas, cores int) catalog.Model {
// that left a 7 GB machine room for only one of the two.
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: judgeMemoryGB, CPUCores: cores,
Args: []string{"--ctx-size", "8192", "--cache-ram", "0"},
// The judge machines, when the install was given some.
NodeSelector: selector,
}},
}
}
@@ -524,7 +526,8 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
func(url string, _ []string) catalog.Model {
return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge)
cores, selector := in.judgePlacement()
return guardModel(url, in.judgeFor().Replicas, cores, selector)
})
}