diff --git a/CHANGELOG.md b/CHANGELOG.md index 13d806e3..57717328 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -33,6 +33,10 @@ APIs change between minors without a deprecation period. which a judge never reuses: each text it reads is different. On a 4-core, 8 GB machine the second judge then fitted nowhere ("has 7 GB of memory with 1 free, needs 4"). The judge now runs with `--cache-ram 0`, uses about 1.6 GB and stays there, and reserves 2. +- **One machine runs one judge, on up to four cores.** A second replica on the same machine shares its cores and is + lost with it, so it bought nothing but half the speed of each. `server init` on a single machine now serves one judge + on up to four cores (2.3 s a judgment where two on two cores took 4.2). Where there will be other nodes, it serves two + on two cores each, as in HA setups. `--judge-replicas` and the setup's Copies still choose. - **A judge gets the two cores it needs.** v0.14.6 gave each judge replica one core on machines under eight. A judgment of about 280 tokens takes 8.0 s on one core, 4.2 s on two and 2.3 s on four, against the 5 s budget, so on a 4-core machine every judgment timed out and every request was refused. A judge replica now reserves two cores, one only diff --git a/docs/install.md b/docs/install.md index fa33a57b..f3eae0fe 100644 --- a/docs/install.md +++ b/docs/install.md @@ -152,9 +152,9 @@ That is the whole step. In order, it: `nvidia-smi` finds any, on CPU otherwise — and waits for it to install its engine 9. closes llmgw's internal ports to everything but the cluster's networks -10. fetches Qwen3Guard-Gen 0.6B, the guardrails' judge, and runs two - replicas on the CPUs of nodes with room — this one, or the first that - join ([guardrails.md](guardrails.md#a-judge-model) has one on a GPU too) +10. fetches Qwen3Guard-Gen 0.6B, the guardrails' judge, and runs it on + CPUs: one replica on up to four cores when this machine is the only + node, two on two cores each on the nodes of a larger cluster ([guardrails.md](guardrails.md#a-judge-model) has one on a GPU too) 11. fetches Qwen2.5 0.5B into the mirror and serves it on this node, so the first thing anyone opens `/chat` to find is a model that answers. A machine without room for it is told so within two minutes, and why. @@ -187,15 +187,19 @@ demonstration one: | Requests | every key limited (60 a minute, 200 000 tokens a minute) and priced, so budgets bind | | Guardrails | the `baseline` policy on every request: secrets in prompts and answers refused, card numbers masked, injection attempts recorded, and the judge refusing violence, crime, weapons, child abuse, privacy violations, hate, self-harm and sexual content, in prompts and answers | -The judge runs as two replicas, each on a CPU — about half a second for a -short text on any CPU with AVX2 — and reads a streamed answer every 600 -characters. The policy fails **closed**: a request the judge cannot read -within 5 seconds is refused with 503 rather than served unjudged, and -`judge_unavailable` goes to the notification webhooks. Two replicas are what -make that bearable; on one small machine, or a VM without AVX2 (see +The judge runs on CPUs. A judgment of about 280 tokens took 2.3 s on four +cores, 4.2 s on two and 8.0 s on one (a 4-core VM). It also reads a +streamed answer every 600 characters. The policy fails **closed**: a request +the judge cannot read within 5 seconds is refused with 503 rather than +served unjudged, and `judge_unavailable` goes to the notification webhooks. +So a judge gets cores of its own: on one machine one replica on up to four, +and on a larger cluster two replicas of two cores each, spread over its nodes +so that losing one leaves the other. On a machine of fewer than four cores, +a busy one, or a VM without AVX2 (see [step 6](#6-setting-the-nodes-cpu-model-if-it-is-a-vm)), the judge is the -first thing to fail, so `fail: open` — serve it, recorded as flagged — or -`--no-guard-model` is the choice there. Defamation, specialized advice, +first thing to fail. The choices there are a longer `latency_budget`, +`fail: open` (serve the request, recorded as flagged), the judge on a GPU, +or `--no-guard-model`. Defamation, specialized advice, intellectual property and elections are left out of what it refuses: they catch too much ordinary work. [guardrails.md](guardrails.md) has the rest. diff --git a/internal/cli/serverinit_choices.go b/internal/cli/serverinit_choices.go index 0b8f8cc3..4a0d6da1 100644 --- a/internal/cli/serverinit_choices.go +++ b/internal/cli/serverinit_choices.go @@ -194,6 +194,9 @@ func (in *serverInit) judgeFor() judgeSpec { } if j.Replicas <= 0 { j.Replicas = 2 + if in.singleMachine() { + j.Replicas = 1 + } } if j.Budget <= 0 { j.Budget = 5 * time.Second @@ -201,6 +204,14 @@ func (in *serverInit) judgeFor() judgeSpec { return j } +// singleMachine is an install where this machine is the only node there +// will be, as far as init knows: a second judge replica there shares the +// same cores and is lost with the same machine, so it buys nothing but +// half the speed of each. +func (in *serverInit) singleMachine() bool { + return !in.noNode && !in.standbys && len(in.patroni) == 0 && len(in.patroniNodes) == 0 +} + // The configuration's limits, as init writes them unless told otherwise. const ( defaultRPM = 60 @@ -346,7 +357,10 @@ func (in *serverInit) initFlags() []string { if in.noGuardModel { add("--no-guard-model") } else { - j, d := in.judgeFor(), (&serverInit{}).judgeFor() + // Against the defaults for an install of the same shape: one judge + // on one machine, two otherwise. + shape := serverInit{noNode: in.noNode, standbys: in.standbys, patroni: in.patroni, patroniNodes: in.patroniNodes} + j, d := in.judgeFor(), shape.judgeFor() if j.Fail != d.Fail { add("--judge-fail", j.Fail) } diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index 31757761..3cfe47fa 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -183,12 +183,17 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T) in serverInit judge, starter int }{ - {serverInit{cpus: 4}, 2, 0}, // two a judge, and the starter shares what is left - {serverInit{cpus: 8}, 2, 4}, - {serverInit{cpus: 32}, 2, 8}, - {serverInit{cpus: 2}, 1, 0}, // below four, one each, so both fit + // One machine: one judge, on up to four cores; the starter has the rest. + {serverInit{cpus: 4}, 4, 0}, + {serverInit{cpus: 8}, 4, 4}, + {serverInit{cpus: 32}, 4, 8}, + {serverInit{cpus: 2}, 2, 0}, {serverInit{cpus: 1}, 0, 0}, - {serverInit{cpus: 4, noGuardModel: true}, 2, 4}, + {serverInit{cpus: 4, noGuardModel: true}, judgeCores, 4}, // no judge: all for the starter + // Two asked for on one machine: two cores each, one each where two + // would not fit both. + {serverInit{cpus: 8, judge: judgeSpec{Replicas: 2}}, 2, 4}, + {serverInit{cpus: 2, judge: judgeSpec{Replicas: 2}}, 1, 0}, {serverInit{cpus: 8, judge: judgeSpec{Replicas: 3}}, 2, 2}, {serverInit{cpus: 4, noNode: true}, 2, 0}, // on nodes not known yet } { @@ -206,3 +211,31 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T) t.Errorf("starter: GPU profile %d cores, CPU %+v", s.Profiles[0].CPUCores, cpu) } } + +// A second judge on the same machine shares its cores and is lost with it: +// one machine has one, more machines two. +func TestOneMachineHasOneJudgeAndMoreHaveTwo(t *testing.T) { + for _, c := range []struct { + in serverInit + want int + }{ + {serverInit{}, 1}, + {serverInit{noNode: true}, 2}, + {serverInit{patroni: []string{"a@db1", "b@db2"}}, 2}, + {serverInit{standbys: true}, 2}, + {serverInit{judge: judgeSpec{Replicas: 3}}, 3}, + } { + if got := c.in.judgeFor().Replicas; got != c.want { + t.Errorf("%+v: %d judges, want %d", c.in, got, c.want) + } + } + // The defaults are left out of the command; a choice against them is in. + ha := serverInit{noNode: true, patroni: []string{"a@db1", "b@db2"}, judge: judgeSpec{Replicas: 2}} + if f := strings.Join(ha.initFlags(), " "); strings.Contains(f, "--judge-replicas") { + t.Errorf("two judges on three machines named: %s", f) + } + one := serverInit{judge: judgeSpec{Replicas: 2}} + if f := strings.Join(one.initFlags(), " "); !strings.Contains(f, "--judge-replicas 2") { + t.Errorf("two judges on one machine not named: %s", f) + } +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index 6d5d1818..a63b4625 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -719,13 +719,13 @@ type cpuSplit struct{ judge, starter int } // the starter on one 4-core control plane all ran on all four: under load // most judgments missed their budget and the requests were refused. // -// A judge needs two cores: a judgment of about 280 tokens took 8.0 s on -// one, 4.2 s on two and 2.3 s on four, against a 5 s budget. When this -// machine is the node, each judge replica gets two (one each only below four -// CPUs, where two would not fit both), and the starter what is left, up to -// 8, or no reservation at all when nothing is, rather than a starter that -// fits nowhere. Otherwise two a judge, on whichever node takes it, and the -// starter, on a node not known yet, none. +// A judge needs cores: a judgment of about 280 tokens took 8.0 s on one, +// 4.2 s on two and 2.3 s on four, against a 5 s budget. On one machine its +// one replica gets up to four; where there are two, two each (one each only +// where two would not fit both). The starter has what is left, up to 8, or +// no reservation at all when nothing is, rather than a starter that fits +// nowhere. A machine that is not the node reserves two a judge on whichever +// node takes it, and the starter, on a node not known yet, none. func (in *serverInit) cpuPlan() cpuSplit { if in.noNode { return cpuSplit{judge: judgeCores} @@ -742,7 +742,10 @@ func (in *serverInit) cpuPlan() cpuSplit { judges = in.judgeFor().Replicas } judge := judgeCores - if judges > 0 && judges*judge > n { + switch { + case judges == 1: + judge = min(n, judgeCoresAlone) + case judges > 0 && judges*judge > n: judge = max(1, n/judges) } left := max(n-judges*judge, 0) @@ -751,8 +754,10 @@ func (in *serverInit) cpuPlan() cpuSplit { // judgeCores and judgeMemoryGB are what a judge replica needs: two cores // for a judgment within its budget, and 2 GB for its 1.6 with no prompt -// cache. +// cache. judgeCoresAlone is what the one judge of a single machine takes, +// past which a model this small hardly gets faster. const ( - judgeCores = 2 - judgeMemoryGB = 2 + judgeCores = 2 + judgeCoresAlone = 4 + judgeMemoryGB = 2 ) diff --git a/internal/cli/tui_choices.go b/internal/cli/tui_choices.go index ab3376f7..2677edd9 100644 --- a/internal/cli/tui_choices.go +++ b/internal/cli/tui_choices.go @@ -282,7 +282,7 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta "The request is served unjudged and marked so in the logs. Nothing is refused for the judge's sake.", } f.judgeCopies = textSetupField("Copies", strconv.Itoa(j.Replicas), "2", - "Each reserves 2 GB and two CPU cores on whichever node has room: on one core a judgment takes longer than its budget.") + "On one machine one copy, on up to four cores; with more machines two, two cores each, on different nodes. Each reserves 2 GB.") f.judgeBudget = textSetupField("Waits at most", durationYAML(j.Budget), "5s", "A short text is judged in about two seconds on CPUs.") judgeFields := []*setupField{f.judgeFail, f.judgeCopies, f.judgeBudget, headingField("Refuses")} diff --git a/internal/cli/tui_setup.go b/internal/cli/tui_setup.go index 976de622..845262c4 100644 --- a/internal/cli/tui_setup.go +++ b/internal/cli/tui_setup.go @@ -188,6 +188,8 @@ func newSetupModel(host string, ips []string) setupModel { // nodes are added once it is up. Either can still be chosen. in.noNode, in.noModel = true, true in.standbys = true + // Two judges, on two of the three machines' nodes. + in.judge.Replicas = 2 return setupModel{ ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true), db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,