A three-machine install can give the judges machines of their own
The setup screen asks for up to two judge machines after the store's two, and server init takes --judge-node for each. Each is added as a node labelled llmgw.role: judge, and the judge's profile selects that label, with cores by init's rule over those machines. A judge beside a busy control plane refused 38 of 60 requests under load; alone, 2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8b122aae09
commit
51ca09287f
12 files changed
+346
-44
No files matched your search
@@ -485,7 +485,7 @@ func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []
|
||||
|
||||
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
|
||||
// the models people use, on whichever node has room; replicas copies of it.
|
||||
func guardModel(source string, replicas, cores int) catalog.Model {
|
||||
func guardModel(source string, replicas, cores int, selector map[string]string) catalog.Model {
|
||||
return catalog.Model{
|
||||
Name: guardModelName,
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
@@ -503,6 +503,8 @@ func guardModel(source string, replicas, cores int) catalog.Model {
|
||||
// that left a 7 GB machine room for only one of the two.
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: judgeMemoryGB, CPUCores: cores,
|
||||
Args: []string{"--ctx-size", "8192", "--cache-ram", "0"},
|
||||
// The judge machines, when the install was given some.
|
||||
NodeSelector: selector,
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -524,7 +526,8 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
|
||||
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
|
||||
func(url string, _ []string) catalog.Model {
|
||||
return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge)
|
||||
cores, selector := in.judgePlacement()
|
||||
return guardModel(url, in.judgeFor().Replicas, cores, selector)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user