A three-machine install can give the judges machines of their own
CI / test (push) Successful in 5m10s
CI / test (push) Successful in 5m10s
The setup screen asks for up to two judge machines after the store's two, and server init takes --judge-node for each. Each is added as a node labelled llmgw.role: judge, and the judge's profile selects that label, with cores by init's rule over those machines. A judge beside a busy control plane refused 38 of 60 requests under load; alone, 2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8b122aae09
commit
51ca09287f
12 files changed
+346
-44
No files matched your search
@@ -80,6 +80,10 @@ type serverInit struct {
|
||||
// setup screen does with the password it was given; when set, patroni
|
||||
// only names them.
|
||||
patroniNodes []pgha.Node
|
||||
// judgeMachines are machines for the guardrails' judges alone, added as
|
||||
// nodes labelled for them; judgeCPUs their cores, once added.
|
||||
judgeMachines []judgeMachine
|
||||
judgeCPUs []int
|
||||
// noMirror, noNode and noModel turn off the parts of a first install that
|
||||
// make it serve something: a mirror, this machine as a node, and a small
|
||||
// model on it.
|
||||
@@ -234,6 +238,8 @@ func newServerInitCmd() *cobra.Command {
|
||||
f.StringVar(&judgeCats, "judge-categories", strings.Join(defaultJudgeCategories, ","),
|
||||
"what the judge refuses, comma-separated, of: "+strings.Join(judgeCategories, ", "))
|
||||
f.IntVar(&in.judge.Replicas, "judge-replicas", 2, "how many copies of the judge run, each on CPUs")
|
||||
f.Var(judgeMachineFlag{&in.judgeMachines}, "judge-node",
|
||||
"user@host of a machine for the guardrails' judges alone: added as a node, and the judges run only there; repeatable")
|
||||
f.DurationVar(&in.judge.Budget, "judge-budget", 5*time.Second, "how long a request waits for the judge")
|
||||
f.StringVar(&in.language, "language", "", "the portal's language for a browser that asks for neither: en or tr")
|
||||
f.BoolVar(&in.signupOpen, "signup-open", false, "let anyone ask for an account on the portal, to be approved by an administrator")
|
||||
@@ -317,6 +323,11 @@ func (in *serverInit) run(ctx context.Context) error {
|
||||
if err := in.ensureWebhooks(ctx); err != nil {
|
||||
fmt.Fprintf(in.out, " !! webhooks: %v\n", err)
|
||||
}
|
||||
if hasSystemd && len(in.judgeMachines) > 0 {
|
||||
if err := in.ensureJudgeNodes(ctx); err != nil {
|
||||
return fmt.Errorf("judge nodes: %w", err)
|
||||
}
|
||||
}
|
||||
if in.judgeWanted() && in.configJudgesWith(guardModelName) {
|
||||
// Not fatal: without its judge the baseline policy still checks for
|
||||
// secrets and injection, and a judged request is served flagged.
|
||||
|
||||
Reference in new issue
Block a user