From 51ca09287fd67e3dc553348681a21bb2237a8748 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Mon, 19 Oct 2026 17:45:00 +0200 Subject: [PATCH] A three-machine install can give the judges machines of their own The setup screen asks for up to two judge machines after the store's two, and server init takes --judge-node for each. Each is added as a node labelled llmgw.role: judge, and the judge's profile selects that label, with cores by init's rule over those machines. A judge beside a busy control plane refused 38 of 60 requests under load; alone, 2. Co-Authored-By: Claude Opus 5.5 (1M context) --- BACKLOG.md | 4 +- CHANGELOG.md | 8 ++ docs/install.md | 14 +- internal/cli/judgekeeper_test.go | 4 +- internal/cli/serverinit.go | 11 ++ internal/cli/serverinit_choices.go | 3 + internal/cli/serverinit_choices_test.go | 2 +- internal/cli/serverinit_cluster.go | 7 +- internal/cli/serverinit_cluster_test.go | 2 +- internal/cli/serverinit_judges.go | 106 ++++++++++++++++ internal/cli/serverinit_judges_test.go | 67 ++++++++++ internal/cli/tui_setup.go | 162 ++++++++++++++++++------ 12 files changed, 346 insertions(+), 44 deletions(-) create mode 100644 internal/cli/serverinit_judges.go create mode 100644 internal/cli/serverinit_judges_test.go diff --git a/BACKLOG.md b/BACKLOG.md index 1b3dec9..162323f 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -33,7 +33,9 @@ What is left, most needed first. The detail is in the item each names. sync. 6. ~~**J2**~~ — done 2026-10-19: a judge the leader adds by itself is sized as init's is (2 GB, init's cores, no prompt cache). -7. **J3** — HA setup puts the judges on nodes of their own. (S) +7. **J3** — HA setup puts the judges on nodes of their own. (S) Code + done 2026-10-19 (setup's judge machines, `--judge-node`); the load test + on a fresh HA install is still to do. 8. **J8** — a judge budget that fits the machine, measured at install. (S) 9. **F5** — large imports limited by the embedding engine's slots. (M) 10. **I1** — answers to similar questions from the cache. (M) diff --git a/CHANGELOG.md b/CHANGELOG.md index f80b21f..edf288b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,14 @@ APIs change between minors without a deprecation period. ## Unreleased +### Added + +- **Machines of the judges' own in a three-machine install.** The setup screen asks for up to two judge machines after + the store's two; `server init` takes `--judge-node user@host` for each. Each is added as a node labelled + `llmgw.role: judge`, and the judge's profile selects it, so the judges run there alone. Under load on a 4-core + control plane that was also a node, 38 of 60 requests were refused for want of a judgment. With the judges on a node + of their own, 2 were. + ### Fixed - **A judge the leader serves by itself is sized as init's is.** When a guard policy names a judge the catalog lacks, diff --git a/docs/install.md b/docs/install.md index 2c044b0..fd111bd 100644 --- a/docs/install.md +++ b/docs/install.md @@ -199,7 +199,19 @@ a busy one, or a VM without AVX2 (see [step 6](#6-setting-the-nodes-cpu-model-if-it-is-a-vm)), the judge is the first thing to fail. The choices there are a longer `latency_budget`, `fail: open` (serve the request, recorded as flagged), the judge on a GPU, -or `--no-guard-model`. Defamation, specialized advice, +or `--no-guard-model`. + +In a three-machine install the judges are better on machines of their own. +On a 4-core control plane that was also a node, 38 of 60 requests under load +were refused for want of a judgment. With the judges on a node of their own +(4 cores, two judges of two cores each), 2 were refused. The setup screen asks +for up to two **judge machines** after the store's two, or give +`--judge-node user@host` to `server init`, once for each. Each is added as a +node labelled `llmgw.role: judge`, and the judge's profile selects that +label, so the judges run only there and nothing else does. Four cores and +8 GB are enough for two judges. + +Defamation, specialized advice, intellectual property and elections are left out of what it refuses: they catch too much ordinary work. [guardrails.md](guardrails.md) has the rest. diff --git a/internal/cli/judgekeeper_test.go b/internal/cli/judgekeeper_test.go index 256a03d..d115772 100644 --- a/internal/cli/judgekeeper_test.go +++ b/internal/cli/judgekeeper_test.go @@ -149,7 +149,7 @@ func TestAnEarlierInstallsJudgeIsRefittedOnce(t *testing.T) { lib := guard.Config{Policies: map[string]guard.Policy{ "baseline": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}}, }} - earlier := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 1) + earlier := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 1, nil) earlier.Profiles[0].MinMemoryGB, earlier.Profiles[0].Args = 4, []string{"--ctx-size", "8192"} inCatalog := []catalog.Model{earlier} fp := &fakeProfiles{} @@ -178,7 +178,7 @@ func TestAnEarlierInstallsJudgeIsRefittedOnce(t *testing.T) { } // Changed by somebody: an extra arg. Left as it is. - mine := guardModel("http://x/g.gguf", 2, 3) + mine := guardModel("http://x/g.gguf", 2, 3, nil) mine.Profiles[0].Args = []string{"--ctx-size", "16384"} inCatalog = []catalog.Model{mine} fp.set, k.looked = nil, nil diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index e051f79..4c60e69 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -80,6 +80,10 @@ type serverInit struct { // setup screen does with the password it was given; when set, patroni // only names them. patroniNodes []pgha.Node + // judgeMachines are machines for the guardrails' judges alone, added as + // nodes labelled for them; judgeCPUs their cores, once added. + judgeMachines []judgeMachine + judgeCPUs []int // noMirror, noNode and noModel turn off the parts of a first install that // make it serve something: a mirror, this machine as a node, and a small // model on it. @@ -234,6 +238,8 @@ func newServerInitCmd() *cobra.Command { f.StringVar(&judgeCats, "judge-categories", strings.Join(defaultJudgeCategories, ","), "what the judge refuses, comma-separated, of: "+strings.Join(judgeCategories, ", ")) f.IntVar(&in.judge.Replicas, "judge-replicas", 2, "how many copies of the judge run, each on CPUs") + f.Var(judgeMachineFlag{&in.judgeMachines}, "judge-node", + "user@host of a machine for the guardrails' judges alone: added as a node, and the judges run only there; repeatable") f.DurationVar(&in.judge.Budget, "judge-budget", 5*time.Second, "how long a request waits for the judge") f.StringVar(&in.language, "language", "", "the portal's language for a browser that asks for neither: en or tr") f.BoolVar(&in.signupOpen, "signup-open", false, "let anyone ask for an account on the portal, to be approved by an administrator") @@ -317,6 +323,11 @@ func (in *serverInit) run(ctx context.Context) error { if err := in.ensureWebhooks(ctx); err != nil { fmt.Fprintf(in.out, " !! webhooks: %v\n", err) } + if hasSystemd && len(in.judgeMachines) > 0 { + if err := in.ensureJudgeNodes(ctx); err != nil { + return fmt.Errorf("judge nodes: %w", err) + } + } if in.judgeWanted() && in.configJudgesWith(guardModelName) { // Not fatal: without its judge the baseline policy still checks for // secrets and injection, and a judged request is served flagged. diff --git a/internal/cli/serverinit_choices.go b/internal/cli/serverinit_choices.go index 4a0d6da..654833b 100644 --- a/internal/cli/serverinit_choices.go +++ b/internal/cli/serverinit_choices.go @@ -325,6 +325,9 @@ func (in *serverInit) initFlags() []string { for _, t := range in.patroni { add("--patroni", t) } + for _, m := range in.judgeMachines { + add("--judge-node", m.target) + } switch { case in.mirrorURL != "": add("--mirror", in.mirrorURL) diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index 3cfe47f..b6ca6ec 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -202,7 +202,7 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T) t.Errorf("%+v: judge %d starter %d, want %d %d", c.in, got.judge, got.starter, c.judge, c.starter) } } - m := guardModel("http://x/g.gguf", 2, 1) + m := guardModel("http://x/g.gguf", 2, 1, nil) if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 2 || !slices.Contains(p.Args, "--cache-ram") { t.Errorf("judge profile: %d cores, %d GB, args %v", p.CPUCores, p.MinMemoryGB, p.Args) } diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index a63b462..45e7d06 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -485,7 +485,7 @@ func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses [] // guardModel is the judge's catalog entry: CPU only, so it takes no card from // the models people use, on whichever node has room; replicas copies of it. -func guardModel(source string, replicas, cores int) catalog.Model { +func guardModel(source string, replicas, cores int, selector map[string]string) catalog.Model { return catalog.Model{ Name: guardModelName, Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.", @@ -503,6 +503,8 @@ func guardModel(source string, replicas, cores int) catalog.Model { // that left a 7 GB machine room for only one of the two. Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: judgeMemoryGB, CPUCores: cores, Args: []string{"--ctx-size", "8192", "--cache-ram", "0"}, + // The judge machines, when the install was given some. + NodeSelector: selector, }}, } } @@ -524,7 +526,8 @@ func (in *serverInit) ensureStarter(ctx context.Context) error { func (in *serverInit) ensureGuardModel(ctx context.Context) error { return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false, func(url string, _ []string) catalog.Model { - return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge) + cores, selector := in.judgePlacement() + return guardModel(url, in.judgeFor().Replicas, cores, selector) }) } diff --git a/internal/cli/serverinit_cluster_test.go b/internal/cli/serverinit_cluster_test.go index 401ca0f..ccc90e4 100644 --- a/internal/cli/serverinit_cluster_test.go +++ b/internal/cli/serverinit_cluster_test.go @@ -171,7 +171,7 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) { } func TestTheGuardModelIsAValidJudge(t *testing.T) { - m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2) + m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2, nil) c, err := catalog.New([]catalog.Model{m}) if err != nil { t.Fatalf("not a valid catalog entry: %v", err) diff --git a/internal/cli/serverinit_judges.go b/internal/cli/serverinit_judges.go new file mode 100644 index 0000000..bcf6d09 --- /dev/null +++ b/internal/cli/serverinit_judges.go @@ -0,0 +1,106 @@ +package cli + +import ( + "context" + "fmt" + "strconv" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" + "gitea.mynodes.xyz/ilker/llmgw/internal/settings" +) + +// Machines of the judges' own. Under load on a 4-core control plane that was +// also a node, 38 of 60 requests were refused for want of a judgment; with the +// judges on a node of their own the same test refused 2. A judge machine is +// added as a node carrying judgeRoleLabel, and the judge's profile selects +// that label, so nothing else is placed beside it and it on nothing else. + +// judgeRoleLabel marks a node as the judges'. +const ( + judgeRoleLabel = "llmgw.role" + judgeRoleValue = "judge" +) + +// judgeMachine is a machine for the judges: user@host, and the connection the +// setup screen already made to it, when it made one. +type judgeMachine struct { + target string + remote onboard.Remote +} + +// ensureJudgeNodes adds each judge machine as a node labelled for the +// judges, and says how many cores the smallest of them has. +func (in *serverInit) ensureJudgeNodes(ctx context.Context) error { + if len(in.judgeMachines) == 0 { + return nil + } + na := nodeAdd{serverCfg: in.configPath(), plain: true} + na.opts.PKI = in.pkiDir + na.opts.NoFirewall = in.noFirewall + if err := na.prepare(); err != nil { + return err + } + in.judgeCPUs = nil + for _, jm := range in.judgeMachines { + o := na.target(jm.target) + fmt.Fprintf(in.out, " .. adding %s as a node for the judges\n", o.Addr) + r := onboard.New(o, nil) + if jm.remote != nil { + r.Remote = jm.remote + } + res, err := r.Run(ctx) + if err != nil { + return fmt.Errorf("%s: %w", o.Addr, err) + } + if err := na.watch(ctx, res.Node, func(onboard.Event) {}); err != nil && err != errNoAdmin { + return fmt.Errorf("%s: %w", res.Node, err) + } + if out, err := r.Remote.Run(ctx, "nproc", nil); err == nil { + if n, err := strconv.Atoi(strings.TrimSpace(string(out))); err == nil { + in.judgeCPUs = append(in.judgeCPUs, n) + } + } + if _, err := changeStoredSettings(in.configPath(), "a node for the judges, from init", func(doc *settings.Doc) error { + return doc.SetYAML([]string{"agents", "nodes", res.Node, "labels", judgeRoleLabel}, judgeRoleValue) + }); err != nil { + return fmt.Errorf("labelling %s for the judges: %w", res.Node, err) + } + in.done("%s is a node for the judges alone (%s: %s)", res.Node, judgeRoleLabel, judgeRoleValue) + } + return nil +} + +// judgePlacement is the judge's cores and where it may run: on the judge +// machines when there are some, by init's rule over their cores; otherwise +// on any node, with this machine's plan. +func (in *serverInit) judgePlacement() (cores int, selector map[string]string) { + if len(in.judgeMachines) == 0 { + return in.cpuPlan().judge, nil + } + return refitCores(in.judgeFor().Replicas, in.judgeCPUs), map[string]string{judgeRoleLabel: judgeRoleValue} +} + +// judgeMachineFlag is --judge-node, given again for each machine. +type judgeMachineFlag struct{ to *[]judgeMachine } + +func (f judgeMachineFlag) String() string { + if f.to == nil { + return "" + } + var out []string + for _, m := range *f.to { + out = append(out, m.target) + } + return strings.Join(out, ",") +} + +func (f judgeMachineFlag) Set(v string) error { + if strings.TrimSpace(v) == "" { + return fmt.Errorf("--judge-node wants user@host") + } + *f.to = append(*f.to, judgeMachine{target: strings.TrimSpace(v)}) + return nil +} + +func (judgeMachineFlag) Type() string { return "user@host" } diff --git a/internal/cli/serverinit_judges_test.go b/internal/cli/serverinit_judges_test.go new file mode 100644 index 0000000..8799927 --- /dev/null +++ b/internal/cli/serverinit_judges_test.go @@ -0,0 +1,67 @@ +package cli + +import ( + "strings" + "testing" +) + +func TestJudgesGoOnTheirOwnMachinesWhenTheInstallHasSome(t *testing.T) { + in := newDefaultServerInit() + if _, sel := in.judgePlacement(); sel != nil { + t.Errorf("no judge machines, yet a selector: %v", sel) + } + if err := (judgeMachineFlag{&in.judgeMachines}).Set("ubuntu@10.0.0.21"); err != nil { + t.Fatal(err) + } + _ = (judgeMachineFlag{&in.judgeMachines}).Set("ubuntu@10.0.0.22") + in.judge.Replicas, in.judgeCPUs = 2, []int{4, 4} + cores, sel := in.judgePlacement() + if sel[judgeRoleLabel] != judgeRoleValue || cores != refitCores(2, []int{4, 4}) { + t.Errorf("cores %d, selector %v", cores, sel) + } + m := guardModel("http://x/g.gguf", 2, cores, sel) + if m.Profiles[0].NodeSelector[judgeRoleLabel] != judgeRoleValue { + t.Errorf("the judge's profile does not select its machines: %+v", m.Profiles[0]) + } + // The same from a script. + flags := strings.Join(in.initFlags(), " ") + if !strings.Contains(flags, "--judge-node ubuntu@10.0.0.21") || !strings.Contains(flags, "--judge-node ubuntu@10.0.0.22") { + t.Errorf("flags %s", flags) + } + if err := (judgeMachineFlag{&in.judgeMachines}).Set(" "); err == nil { + t.Error("an empty --judge-node was taken") + } +} + +func TestTheSetupAsksForJudgeMachinesWhenChosen(t *testing.T) { + m := newSetupModel("ctl", []string{"10.0.0.5"}) + m.choice, m.page = 1, pMachines + m.mfocus = m.shown() * mfCount // the judges' choice + m = press(m, "right").(setupModel) + if m.judges != 1 || m.shown() != 3 || m.mfocus != 3*mfCount { + t.Fatalf("judges %d, shown %d, focus %d", m.judges, m.shown(), m.mfocus) + } + if v := m.machinesView(); !strings.Contains(v, "Judge machine 1") || !strings.Contains(v, "Check the 4 machines") { + t.Errorf("the page lacks the judge machine:\n%s", v) + } + for i, addr := range []string{"10.0.0.11", "10.0.0.12", ""} { + m.db[i].addr.SetValue(addr) + m.db[i].user.SetValue("ubuntu") + m.db[i].auth = authAgent + } + if err := m.validateMachines(); !strings.Contains(err, "judge machine 1 has no address") { + t.Errorf("err %q", err) + } + m.db[2].addr.SetValue("10.0.0.21") + if err := m.validateMachines(); err != "" { + t.Errorf("err %q", err) + } + if cmd := m.command(); !strings.Contains(cmd, "--judge-node ubuntu@10.0.0.21") || strings.Contains(cmd, "--patroni ubuntu@10.0.0.21") { + t.Errorf("command %s", cmd) + } + // None again: the judge machine is no longer asked for. + m = press(m, "left").(setupModel) + if m.judges != 0 || strings.Contains(m.machinesView(), "Judge machine 1") { + t.Errorf("judges %d after choosing none", m.judges) + } +} diff --git a/internal/cli/tui_setup.go b/internal/cli/tui_setup.go index 845262c..cbf85f5 100644 --- a/internal/cli/tui_setup.go +++ b/internal/cli/tui_setup.go @@ -89,14 +89,21 @@ func (d dbMachine) visible(f int) bool { return true } -// dbConn is a database machine as it is reached, whichever way. +// dbConn is a database machine as it is reached, whichever way, or a judge +// machine when judge is set. type dbConn struct { remote onboard.Remote addr string ssh *onboard.GoSSH joined *joinpkg.Remote + judge bool + target string } +// The machines the page asks for: the store's two, then up to maxJudges for +// the guardrails' judges alone. +const maxJudges = 2 + func newDBMachine(n int) dbMachine { f := func(ph string, mask bool) textField { t := newTextField() @@ -119,6 +126,24 @@ func newDBMachine(n int) dbMachine { return m } +// newJudgeMachine is the fields of a machine for the judges. +func newJudgeMachine(n int) dbMachine { + d := newDBMachine(n + 2) + d.addr.Placeholder = fmt.Sprintf("10.0.0.%d or judge%d.internal", 20+n, n) + return d +} + +// shown is how many machines the page asks for. +func (m setupModel) shown() int { return 2 + m.judges } + +// machineTitle names machine i: the store's, or the judges'. +func machineTitle(i int) string { + if i < 2 { + return fmt.Sprintf("Database machine %d", i+1) + } + return fmt.Sprintf("Judge machine %d", i-1) +} + // field is the text field at a machine's position, nil for the selector. func (d *dbMachine) field(i int) *textField { switch i { @@ -156,8 +181,10 @@ type setupModel struct { choices *installForm - db [2]dbMachine - mfocus int // over both machines' fields, then the check button + db [2 + maxJudges]dbMachine + // judges is how many judge machines are asked for, none to maxJudges. + judges int + mfocus int // over the machines' fields, then the judges' choice, then the check button conns []dbConn listener *joinpkg.Listener @@ -192,7 +219,8 @@ func newSetupModel(host string, ips []string) setupModel { in.judge.Replicas = 2 return setupModel{ ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true), - db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30, + db: [2 + maxJudges]dbMachine{newDBMachine(1), newDBMachine(2), newJudgeMachine(1), newJudgeMachine(2)}, + spin: newSpinner(), width: 100, height: 30, log: &[]string{}, logMu: &sync.Mutex{}, } } @@ -327,7 +355,7 @@ func (m *setupModel) focusMachine() { } } } - if m.mfocus < 2*mfCount { + if m.mfocus < m.shown()*mfCount { if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil { t.Focus() } @@ -335,7 +363,8 @@ func (m *setupModel) focusMachine() { } func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { - n := 2*mfCount + 1 + sel := m.shown() * mfCount + n := sel + 2 switch k.Type { case tea.KeyTab, tea.KeyDown: m.step(1, n) @@ -344,7 +373,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { m.step(-1, n) return m, nil case tea.KeyEnter: - if m.mfocus < 2*mfCount { + if m.mfocus <= sel { m.step(1, n) return m, nil } @@ -361,7 +390,17 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { cmd := m.startCheck() return m, cmd } - if m.mfocus == 2*mfCount { + if m.mfocus == sel { + switch k.String() { + case "left", "h": + m.judges = (m.judges + maxJudges) % (maxJudges + 1) + case "right", "l", " ": + m.judges = (m.judges + 1) % (maxJudges + 1) + } + m.mfocus = m.shown() * mfCount + return m, nil + } + if m.mfocus > sel { return m, nil } d := &m.db[m.mfocus/mfCount] @@ -391,7 +430,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { func (m *setupModel) step(d, n int) { for range n { m.mfocus = (m.mfocus + d + n) % n - if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) { + if m.mfocus >= m.shown()*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) { break } } @@ -399,7 +438,7 @@ func (m *setupModel) step(d, n int) { } func (m setupModel) allJoined() bool { - for _, d := range m.db { + for _, d := range m.db[:m.shown()] { if d.auth == authJoin && d.joined == nil { return false } @@ -411,7 +450,7 @@ func (m setupModel) allJoined() bool { // codes, and waits for them; it says whether there is anything to wait for. func (m *setupModel) startJoins() (tea.Cmd, bool) { var cmds []tea.Cmd - for i := range m.db { + for i := range m.db[:m.shown()] { d := &m.db[i] if d.auth != authJoin || d.joined != nil { continue @@ -429,7 +468,11 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) { m.listener = l } if d.arrive == nil { - c, ch := m.listener.Code("database") + role := "database" + if i >= 2 { + role = "judge" + } + c, ch := m.listener.Code(role) d.code, d.arrive = c.String(), ch } ch, i := d.arrive, i @@ -440,20 +483,20 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) { func (m setupModel) validateMachines() string { seen := map[string]bool{m.thisAddr(): true, m.host: true} - for i, d := range m.db { + for i, d := range m.db[:m.shown()] { if d.auth == authJoin { continue } - addr := strings.TrimSpace(d.addr.Value()) + addr, title := strings.TrimSpace(d.addr.Value()), strings.ToLower(machineTitle(i)[:1])+machineTitle(i)[1:] switch { case addr == "": - return fmt.Sprintf("database machine %d has no address", i+1) + return title + " has no address" case seen[addr]: - return fmt.Sprintf("%s is given twice, or is this machine: the three must be different machines", addr) + return fmt.Sprintf("%s is given twice, or is this machine: each must be a different machine", addr) case strings.TrimSpace(d.user.Value()) == "": - return fmt.Sprintf("database machine %d has no user to log in as", i+1) + return title + " has no user to log in as" case d.auth != authAgent && d.secret.Value() == "": - return fmt.Sprintf("database machine %d needs its %s", i+1, authNames[d.auth]) + return fmt.Sprintf("%s needs its %s", title, authNames[d.auth]) } seen[addr] = true } @@ -473,11 +516,16 @@ func (m *setupModel) startCheck() tea.Cmd { Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{}, Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness}, }} - for _, d := range m.db { + for i, d := range m.db[:m.shown()] { + roles, judge := m.dbRoles(), i >= 2 + if judge { + roles = []preflight.Role{preflight.RoleNode} + } if d.auth == authJoin { - m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined}) + m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined, + judge: judge, target: "root@" + d.joined.Addr}) machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)", - Addr: d.joined.Addr, Roles: m.dbRoles(), Remote: d.joined}) + Addr: d.joined.Addr, Roles: roles, Remote: d.joined}) continue } g := &onboard.GoSSH{ @@ -490,9 +538,8 @@ func (m *setupModel) startCheck() tea.Cmd { case authPassword: g.Password = d.secret.Value() } - m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g}) - machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, - Roles: m.dbRoles(), Remote: g}) + m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g, judge: judge, target: g.User + "@" + g.Addr}) + machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, Roles: roles, Remote: g}) } m.checking = true m.report = preflight.Report{} @@ -535,7 +582,7 @@ func (m setupModel) dbRoles() []preflight.Role { // targets are the database machines as user@address, for the command line. func (m setupModel) targets() []string { var out []string - for _, d := range m.db { + for _, d := range m.db[:2] { if d.joined != nil { out = append(out, "root@"+d.joined.Addr) continue @@ -545,11 +592,25 @@ func (m setupModel) targets() []string { return out } +// judgeTargets are the judge machines as user@address. +func (m setupModel) judgeTargets() []judgeMachine { + var out []judgeMachine + for _, d := range m.db[2:m.shown()] { + if d.joined != nil { + out = append(out, judgeMachine{target: "root@" + d.joined.Addr}) + continue + } + out = append(out, judgeMachine{target: strings.TrimSpace(d.user.Value()) + "@" + strings.TrimSpace(d.addr.Value())}) + } + return out +} + // command is what this setup is on the command line, for doing it again // from a script — with keys in place, since a password cannot be given there. func (m setupModel) command() string { in := *m.in in.patroni = m.targets() + in.judgeMachines = m.judgeTargets() return strings.Join(append([]string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init"}, in.initFlags()...), " \\\n ") } @@ -558,6 +619,7 @@ func (m setupModel) command() string { func (m *setupModel) startRun() tea.Cmd { in := m.in in.patroni = m.targets() + in.judgeMachines = nil m.run = newInitModel(in, nil, false) m.run.running, m.run.width, m.run.spin = true, m.width, m.spin ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener @@ -578,6 +640,11 @@ func (m *setupModel) startRun() tea.Cmd { return err } } + if c.judge { + // Added as a node once the control plane is up. + in.judgeMachines = append(in.judgeMachines, judgeMachine{target: c.target, remote: c.remote}) + continue + } out, err := c.remote.Run(ctx, "hostname", nil) if err != nil { return fmt.Errorf("%s: %v", c.addr, err) @@ -720,8 +787,12 @@ func (m setupModel) machinesView() string { var b strings.Builder b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" + " PostgreSQL, Patroni and etcd are installed on them.\n\n") - for i, d := range m.db { + for i, d := range m.db[:m.shown()] { base := i * mfCount + if i == 2 { + b.WriteString(" " + sDim.Render("Machines for the guardrails' judges alone: each becomes a node that runs only\n"+ + " the judges, so a busy control plane cannot leave a request waiting for a judgment.") + "\n\n") + } // Padded before it is coloured: the colour's escape codes count // toward a width and threw the focused row out of line. label := func(f int, s string) string { @@ -731,7 +802,7 @@ func (m setupModel) machinesView() string { } return " " + s } - fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))) + fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(machineTitle(i))) if d.visible(mfAddr) { fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View()) fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View()) @@ -756,7 +827,22 @@ func (m setupModel) machinesView() string { } b.WriteString("\n") } - b.WriteString(button(m.mfocus == 2*mfCount, "Check the three machines") + "\n\n") + sel := m.shown() * mfCount + var opts []string + for n, name := range []string{"none", "one", "two"} { + if n == m.judges { + opts = append(opts, "("+sOK.Render("•")+") "+name) + } else { + opts = append(opts, "( ) "+sDim.Render(name)) + } + } + judgeLabel := fmt.Sprintf(" %-15s", "Judge machines") + if m.mfocus == sel { + judgeLabel = fSelected.Render(fmt.Sprintf("› %-15s", "Judge machines")) + } + fmt.Fprintf(&b, " %s %s\n", judgeLabel, strings.Join(opts, " ")) + b.WriteString(" " + sDim.Render("Without them the judges run on whichever node has room, added later.") + "\n\n") + b.WriteString(button(m.mfocus == sel+1, fmt.Sprintf("Check the %d machines", m.shown()+1)) + "\n\n") b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n") return b.String() } @@ -764,11 +850,11 @@ func (m setupModel) machinesView() string { func (m setupModel) waitView() string { var b strings.Builder fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort) - for i, d := range m.db { + for i, d := range m.db[:m.shown()] { if d.auth != authJoin { continue } - title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)) + title := lipgloss.NewStyle().Bold(true).Render(machineTitle(i)) if d.joined != nil { fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr) continue @@ -787,7 +873,7 @@ func (m setupModel) checkView() string { var b strings.Builder b.WriteString("\n") if m.checking { - b.WriteString(" " + m.spin.View() + " checking the three machines…\n") + b.WriteString(" " + m.spin.View() + " checking the machines…\n") return b.String() } switch m.report.Worst() { @@ -796,7 +882,7 @@ func (m setupModel) checkView() string { case preflight.Warn: b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n") default: - b.WriteString(" " + sOK.Render("✓ All three are ready.") + "\n") + b.WriteString(" " + sOK.Render("✓ All are ready.") + "\n") } machine := "" for _, f := range m.report.Findings { @@ -822,7 +908,7 @@ func (m setupModel) checkView() string { func (m setupModel) planView() string { var b strings.Builder b.WriteString(m.choices.planView(m.width - navWidth - 6)) - b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The three machines") + "\n") + b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The machines") + "\n") here := "control plane, leading at first; model mirror; etcd" there := "PostgreSQL under Patroni, etcd" if m.in.standbys { @@ -834,12 +920,16 @@ func (m setupModel) planView() string { } fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), here) granted := false - for _, d := range m.db { + for i, d := range m.db[:m.shown()] { addr := strings.TrimSpace(d.addr.Value()) if d.joined != nil { addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined") } - fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), there) + role := there + if i >= 2 { + role = "a node for the guardrails' judges alone" + } + fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), role) granted = granted || d.sudo.Value() != "" } b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" + @@ -851,7 +941,7 @@ func (m setupModel) planView() string { for _, l := range strings.Split(m.command(), "\n") { b.WriteString(" " + l + "\n") } - for _, d := range m.db { + for _, d := range m.db[:m.shown()] { if d.joined != nil { b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n") break