A three-machine install can give the judges machines of their own
CI / test (push) Successful in 5m10s

The setup screen asks for up to two judge machines after the store's
two, and server init takes --judge-node for each. Each is added as a
node labelled llmgw.role: judge, and the judge's profile selects that
label, with cores by init's rule over those machines. A judge beside a
busy control plane refused 38 of 60 requests under load; alone, 2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 17:45:00 +02:00
1 parent 8b122aae09
commit 51ca09287f
12 files changed
+346 -44

No files matched your search

+3 -1
View File
@@ -33,7 +33,9 @@ What is left, most needed first. The detail is in the item each names.
sync.
6. ~~**J2**~~ — done 2026-10-19: a judge the leader adds by itself is
sized as init's is (2 GB, init's cores, no prompt cache).
7. **J3** — HA setup puts the judges on nodes of their own. (S)
7. **J3** — HA setup puts the judges on nodes of their own. (S) Code
done 2026-10-19 (setup's judge machines, `--judge-node`); the load test
on a fresh HA install is still to do.
8. **J8** — a judge budget that fits the machine, measured at install. (S)
9. **F5** — large imports limited by the embedding engine's slots. (M)
10. **I1** — answers to similar questions from the cache. (M)
+8
View File
@@ -26,6 +26,14 @@ APIs change between minors without a deprecation period.
## Unreleased
### Added
- **Machines of the judges' own in a three-machine install.** The setup screen asks for up to two judge machines after
the store's two; `server init` takes `--judge-node user@host` for each. Each is added as a node labelled
`llmgw.role: judge`, and the judge's profile selects it, so the judges run there alone. Under load on a 4-core
control plane that was also a node, 38 of 60 requests were refused for want of a judgment. With the judges on a node
of their own, 2 were.
### Fixed
- **A judge the leader serves by itself is sized as init's is.** When a guard policy names a judge the catalog lacks,
+13 -1
View File
@@ -199,7 +199,19 @@ a busy one, or a VM without AVX2 (see
[step 6](#6-setting-the-nodes-cpu-model-if-it-is-a-vm)), the judge is the
first thing to fail. The choices there are a longer `latency_budget`,
`fail: open` (serve the request, recorded as flagged), the judge on a GPU,
or `--no-guard-model`. Defamation, specialized advice,
or `--no-guard-model`.
In a three-machine install the judges are better on machines of their own.
On a 4-core control plane that was also a node, 38 of 60 requests under load
were refused for want of a judgment. With the judges on a node of their own
(4 cores, two judges of two cores each), 2 were refused. The setup screen asks
for up to two **judge machines** after the store's two, or give
`--judge-node user@host` to `server init`, once for each. Each is added as a
node labelled `llmgw.role: judge`, and the judge's profile selects that
label, so the judges run only there and nothing else does. Four cores and
8 GB are enough for two judges.
Defamation, specialized advice,
intellectual property and elections are left out of what it refuses: they
catch too much ordinary work. [guardrails.md](guardrails.md) has the rest.
+2 -2
View File
@@ -149,7 +149,7 @@ func TestAnEarlierInstallsJudgeIsRefittedOnce(t *testing.T) {
lib := guard.Config{Policies: map[string]guard.Policy{
"baseline": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}},
}}
earlier := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 1)
earlier := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 1, nil)
earlier.Profiles[0].MinMemoryGB, earlier.Profiles[0].Args = 4, []string{"--ctx-size", "8192"}
inCatalog := []catalog.Model{earlier}
fp := &fakeProfiles{}
@@ -178,7 +178,7 @@ func TestAnEarlierInstallsJudgeIsRefittedOnce(t *testing.T) {
}
// Changed by somebody: an extra arg. Left as it is.
mine := guardModel("http://x/g.gguf", 2, 3)
mine := guardModel("http://x/g.gguf", 2, 3, nil)
mine.Profiles[0].Args = []string{"--ctx-size", "16384"}
inCatalog = []catalog.Model{mine}
fp.set, k.looked = nil, nil
+11
View File
@@ -80,6 +80,10 @@ type serverInit struct {
// setup screen does with the password it was given; when set, patroni
// only names them.
patroniNodes []pgha.Node
// judgeMachines are machines for the guardrails' judges alone, added as
// nodes labelled for them; judgeCPUs their cores, once added.
judgeMachines []judgeMachine
judgeCPUs []int
// noMirror, noNode and noModel turn off the parts of a first install that
// make it serve something: a mirror, this machine as a node, and a small
// model on it.
@@ -234,6 +238,8 @@ func newServerInitCmd() *cobra.Command {
f.StringVar(&judgeCats, "judge-categories", strings.Join(defaultJudgeCategories, ","),
"what the judge refuses, comma-separated, of: "+strings.Join(judgeCategories, ", "))
f.IntVar(&in.judge.Replicas, "judge-replicas", 2, "how many copies of the judge run, each on CPUs")
f.Var(judgeMachineFlag{&in.judgeMachines}, "judge-node",
"user@host of a machine for the guardrails' judges alone: added as a node, and the judges run only there; repeatable")
f.DurationVar(&in.judge.Budget, "judge-budget", 5*time.Second, "how long a request waits for the judge")
f.StringVar(&in.language, "language", "", "the portal's language for a browser that asks for neither: en or tr")
f.BoolVar(&in.signupOpen, "signup-open", false, "let anyone ask for an account on the portal, to be approved by an administrator")
@@ -317,6 +323,11 @@ func (in *serverInit) run(ctx context.Context) error {
if err := in.ensureWebhooks(ctx); err != nil {
fmt.Fprintf(in.out, " !! webhooks: %v\n", err)
}
if hasSystemd && len(in.judgeMachines) > 0 {
if err := in.ensureJudgeNodes(ctx); err != nil {
return fmt.Errorf("judge nodes: %w", err)
}
}
if in.judgeWanted() && in.configJudgesWith(guardModelName) {
// Not fatal: without its judge the baseline policy still checks for
// secrets and injection, and a judged request is served flagged.
+3
View File
@@ -325,6 +325,9 @@ func (in *serverInit) initFlags() []string {
for _, t := range in.patroni {
add("--patroni", t)
}
for _, m := range in.judgeMachines {
add("--judge-node", m.target)
}
switch {
case in.mirrorURL != "":
add("--mirror", in.mirrorURL)
+1 -1
View File
@@ -202,7 +202,7 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T)
t.Errorf("%+v: judge %d starter %d, want %d %d", c.in, got.judge, got.starter, c.judge, c.starter)
}
}
m := guardModel("http://x/g.gguf", 2, 1)
m := guardModel("http://x/g.gguf", 2, 1, nil)
if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 2 || !slices.Contains(p.Args, "--cache-ram") {
t.Errorf("judge profile: %d cores, %d GB, args %v", p.CPUCores, p.MinMemoryGB, p.Args)
}
+5 -2
View File
@@ -485,7 +485,7 @@ func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
// the models people use, on whichever node has room; replicas copies of it.
func guardModel(source string, replicas, cores int) catalog.Model {
func guardModel(source string, replicas, cores int, selector map[string]string) catalog.Model {
return catalog.Model{
Name: guardModelName,
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
@@ -503,6 +503,8 @@ func guardModel(source string, replicas, cores int) catalog.Model {
// that left a 7 GB machine room for only one of the two.
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: judgeMemoryGB, CPUCores: cores,
Args: []string{"--ctx-size", "8192", "--cache-ram", "0"},
// The judge machines, when the install was given some.
NodeSelector: selector,
}},
}
}
@@ -524,7 +526,8 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
func(url string, _ []string) catalog.Model {
return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge)
cores, selector := in.judgePlacement()
return guardModel(url, in.judgeFor().Replicas, cores, selector)
})
}
+1 -1
View File
@@ -171,7 +171,7 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
}
func TestTheGuardModelIsAValidJudge(t *testing.T) {
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2)
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2, nil)
c, err := catalog.New([]catalog.Model{m})
if err != nil {
t.Fatalf("not a valid catalog entry: %v", err)
+106
View File
@@ -0,0 +1,106 @@
package cli
import (
"context"
"fmt"
"strconv"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
)
// Machines of the judges' own. Under load on a 4-core control plane that was
// also a node, 38 of 60 requests were refused for want of a judgment; with the
// judges on a node of their own the same test refused 2. A judge machine is
// added as a node carrying judgeRoleLabel, and the judge's profile selects
// that label, so nothing else is placed beside it and it on nothing else.
// judgeRoleLabel marks a node as the judges'.
const (
judgeRoleLabel = "llmgw.role"
judgeRoleValue = "judge"
)
// judgeMachine is a machine for the judges: user@host, and the connection the
// setup screen already made to it, when it made one.
type judgeMachine struct {
target string
remote onboard.Remote
}
// ensureJudgeNodes adds each judge machine as a node labelled for the
// judges, and says how many cores the smallest of them has.
func (in *serverInit) ensureJudgeNodes(ctx context.Context) error {
if len(in.judgeMachines) == 0 {
return nil
}
na := nodeAdd{serverCfg: in.configPath(), plain: true}
na.opts.PKI = in.pkiDir
na.opts.NoFirewall = in.noFirewall
if err := na.prepare(); err != nil {
return err
}
in.judgeCPUs = nil
for _, jm := range in.judgeMachines {
o := na.target(jm.target)
fmt.Fprintf(in.out, " .. adding %s as a node for the judges\n", o.Addr)
r := onboard.New(o, nil)
if jm.remote != nil {
r.Remote = jm.remote
}
res, err := r.Run(ctx)
if err != nil {
return fmt.Errorf("%s: %w", o.Addr, err)
}
if err := na.watch(ctx, res.Node, func(onboard.Event) {}); err != nil && err != errNoAdmin {
return fmt.Errorf("%s: %w", res.Node, err)
}
if out, err := r.Remote.Run(ctx, "nproc", nil); err == nil {
if n, err := strconv.Atoi(strings.TrimSpace(string(out))); err == nil {
in.judgeCPUs = append(in.judgeCPUs, n)
}
}
if _, err := changeStoredSettings(in.configPath(), "a node for the judges, from init", func(doc *settings.Doc) error {
return doc.SetYAML([]string{"agents", "nodes", res.Node, "labels", judgeRoleLabel}, judgeRoleValue)
}); err != nil {
return fmt.Errorf("labelling %s for the judges: %w", res.Node, err)
}
in.done("%s is a node for the judges alone (%s: %s)", res.Node, judgeRoleLabel, judgeRoleValue)
}
return nil
}
// judgePlacement is the judge's cores and where it may run: on the judge
// machines when there are some, by init's rule over their cores; otherwise
// on any node, with this machine's plan.
func (in *serverInit) judgePlacement() (cores int, selector map[string]string) {
if len(in.judgeMachines) == 0 {
return in.cpuPlan().judge, nil
}
return refitCores(in.judgeFor().Replicas, in.judgeCPUs), map[string]string{judgeRoleLabel: judgeRoleValue}
}
// judgeMachineFlag is --judge-node, given again for each machine.
type judgeMachineFlag struct{ to *[]judgeMachine }
func (f judgeMachineFlag) String() string {
if f.to == nil {
return ""
}
var out []string
for _, m := range *f.to {
out = append(out, m.target)
}
return strings.Join(out, ",")
}
func (f judgeMachineFlag) Set(v string) error {
if strings.TrimSpace(v) == "" {
return fmt.Errorf("--judge-node wants user@host")
}
*f.to = append(*f.to, judgeMachine{target: strings.TrimSpace(v)})
return nil
}
func (judgeMachineFlag) Type() string { return "user@host" }
+67
View File
@@ -0,0 +1,67 @@
package cli
import (
"strings"
"testing"
)
func TestJudgesGoOnTheirOwnMachinesWhenTheInstallHasSome(t *testing.T) {
in := newDefaultServerInit()
if _, sel := in.judgePlacement(); sel != nil {
t.Errorf("no judge machines, yet a selector: %v", sel)
}
if err := (judgeMachineFlag{&in.judgeMachines}).Set("ubuntu@10.0.0.21"); err != nil {
t.Fatal(err)
}
_ = (judgeMachineFlag{&in.judgeMachines}).Set("ubuntu@10.0.0.22")
in.judge.Replicas, in.judgeCPUs = 2, []int{4, 4}
cores, sel := in.judgePlacement()
if sel[judgeRoleLabel] != judgeRoleValue || cores != refitCores(2, []int{4, 4}) {
t.Errorf("cores %d, selector %v", cores, sel)
}
m := guardModel("http://x/g.gguf", 2, cores, sel)
if m.Profiles[0].NodeSelector[judgeRoleLabel] != judgeRoleValue {
t.Errorf("the judge's profile does not select its machines: %+v", m.Profiles[0])
}
// The same from a script.
flags := strings.Join(in.initFlags(), " ")
if !strings.Contains(flags, "--judge-node ubuntu@10.0.0.21") || !strings.Contains(flags, "--judge-node ubuntu@10.0.0.22") {
t.Errorf("flags %s", flags)
}
if err := (judgeMachineFlag{&in.judgeMachines}).Set(" "); err == nil {
t.Error("an empty --judge-node was taken")
}
}
func TestTheSetupAsksForJudgeMachinesWhenChosen(t *testing.T) {
m := newSetupModel("ctl", []string{"10.0.0.5"})
m.choice, m.page = 1, pMachines
m.mfocus = m.shown() * mfCount // the judges' choice
m = press(m, "right").(setupModel)
if m.judges != 1 || m.shown() != 3 || m.mfocus != 3*mfCount {
t.Fatalf("judges %d, shown %d, focus %d", m.judges, m.shown(), m.mfocus)
}
if v := m.machinesView(); !strings.Contains(v, "Judge machine 1") || !strings.Contains(v, "Check the 4 machines") {
t.Errorf("the page lacks the judge machine:\n%s", v)
}
for i, addr := range []string{"10.0.0.11", "10.0.0.12", ""} {
m.db[i].addr.SetValue(addr)
m.db[i].user.SetValue("ubuntu")
m.db[i].auth = authAgent
}
if err := m.validateMachines(); !strings.Contains(err, "judge machine 1 has no address") {
t.Errorf("err %q", err)
}
m.db[2].addr.SetValue("10.0.0.21")
if err := m.validateMachines(); err != "" {
t.Errorf("err %q", err)
}
if cmd := m.command(); !strings.Contains(cmd, "--judge-node ubuntu@10.0.0.21") || strings.Contains(cmd, "--patroni ubuntu@10.0.0.21") {
t.Errorf("command %s", cmd)
}
// None again: the judge machine is no longer asked for.
m = press(m, "left").(setupModel)
if m.judges != 0 || strings.Contains(m.machinesView(), "Judge machine 1") {
t.Errorf("judges %d after choosing none", m.judges)
}
}
+126 -36
View File
@@ -89,14 +89,21 @@ func (d dbMachine) visible(f int) bool {
return true
}
// dbConn is a database machine as it is reached, whichever way.
// dbConn is a database machine as it is reached, whichever way, or a judge
// machine when judge is set.
type dbConn struct {
remote onboard.Remote
addr string
ssh *onboard.GoSSH
joined *joinpkg.Remote
judge bool
target string
}
// The machines the page asks for: the store's two, then up to maxJudges for
// the guardrails' judges alone.
const maxJudges = 2
func newDBMachine(n int) dbMachine {
f := func(ph string, mask bool) textField {
t := newTextField()
@@ -119,6 +126,24 @@ func newDBMachine(n int) dbMachine {
return m
}
// newJudgeMachine is the fields of a machine for the judges.
func newJudgeMachine(n int) dbMachine {
d := newDBMachine(n + 2)
d.addr.Placeholder = fmt.Sprintf("10.0.0.%d or judge%d.internal", 20+n, n)
return d
}
// shown is how many machines the page asks for.
func (m setupModel) shown() int { return 2 + m.judges }
// machineTitle names machine i: the store's, or the judges'.
func machineTitle(i int) string {
if i < 2 {
return fmt.Sprintf("Database machine %d", i+1)
}
return fmt.Sprintf("Judge machine %d", i-1)
}
// field is the text field at a machine's position, nil for the selector.
func (d *dbMachine) field(i int) *textField {
switch i {
@@ -156,8 +181,10 @@ type setupModel struct {
choices *installForm
db [2]dbMachine
mfocus int // over both machines' fields, then the check button
db [2 + maxJudges]dbMachine
// judges is how many judge machines are asked for, none to maxJudges.
judges int
mfocus int // over the machines' fields, then the judges' choice, then the check button
conns []dbConn
listener *joinpkg.Listener
@@ -192,7 +219,8 @@ func newSetupModel(host string, ips []string) setupModel {
in.judge.Replicas = 2
return setupModel{
ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true),
db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,
db: [2 + maxJudges]dbMachine{newDBMachine(1), newDBMachine(2), newJudgeMachine(1), newJudgeMachine(2)},
spin: newSpinner(), width: 100, height: 30,
log: &[]string{}, logMu: &sync.Mutex{},
}
}
@@ -327,7 +355,7 @@ func (m *setupModel) focusMachine() {
}
}
}
if m.mfocus < 2*mfCount {
if m.mfocus < m.shown()*mfCount {
if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil {
t.Focus()
}
@@ -335,7 +363,8 @@ func (m *setupModel) focusMachine() {
}
func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
n := 2*mfCount + 1
sel := m.shown() * mfCount
n := sel + 2
switch k.Type {
case tea.KeyTab, tea.KeyDown:
m.step(1, n)
@@ -344,7 +373,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.step(-1, n)
return m, nil
case tea.KeyEnter:
if m.mfocus < 2*mfCount {
if m.mfocus <= sel {
m.step(1, n)
return m, nil
}
@@ -361,7 +390,17 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
cmd := m.startCheck()
return m, cmd
}
if m.mfocus == 2*mfCount {
if m.mfocus == sel {
switch k.String() {
case "left", "h":
m.judges = (m.judges + maxJudges) % (maxJudges + 1)
case "right", "l", " ":
m.judges = (m.judges + 1) % (maxJudges + 1)
}
m.mfocus = m.shown() * mfCount
return m, nil
}
if m.mfocus > sel {
return m, nil
}
d := &m.db[m.mfocus/mfCount]
@@ -391,7 +430,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
func (m *setupModel) step(d, n int) {
for range n {
m.mfocus = (m.mfocus + d + n) % n
if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
if m.mfocus >= m.shown()*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
break
}
}
@@ -399,7 +438,7 @@ func (m *setupModel) step(d, n int) {
}
func (m setupModel) allJoined() bool {
for _, d := range m.db {
for _, d := range m.db[:m.shown()] {
if d.auth == authJoin && d.joined == nil {
return false
}
@@ -411,7 +450,7 @@ func (m setupModel) allJoined() bool {
// codes, and waits for them; it says whether there is anything to wait for.
func (m *setupModel) startJoins() (tea.Cmd, bool) {
var cmds []tea.Cmd
for i := range m.db {
for i := range m.db[:m.shown()] {
d := &m.db[i]
if d.auth != authJoin || d.joined != nil {
continue
@@ -429,7 +468,11 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) {
m.listener = l
}
if d.arrive == nil {
c, ch := m.listener.Code("database")
role := "database"
if i >= 2 {
role = "judge"
}
c, ch := m.listener.Code(role)
d.code, d.arrive = c.String(), ch
}
ch, i := d.arrive, i
@@ -440,20 +483,20 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) {
func (m setupModel) validateMachines() string {
seen := map[string]bool{m.thisAddr(): true, m.host: true}
for i, d := range m.db {
for i, d := range m.db[:m.shown()] {
if d.auth == authJoin {
continue
}
addr := strings.TrimSpace(d.addr.Value())
addr, title := strings.TrimSpace(d.addr.Value()), strings.ToLower(machineTitle(i)[:1])+machineTitle(i)[1:]
switch {
case addr == "":
return fmt.Sprintf("database machine %d has no address", i+1)
return title + " has no address"
case seen[addr]:
return fmt.Sprintf("%s is given twice, or is this machine: the three must be different machines", addr)
return fmt.Sprintf("%s is given twice, or is this machine: each must be a different machine", addr)
case strings.TrimSpace(d.user.Value()) == "":
return fmt.Sprintf("database machine %d has no user to log in as", i+1)
return title + " has no user to log in as"
case d.auth != authAgent && d.secret.Value() == "":
return fmt.Sprintf("database machine %d needs its %s", i+1, authNames[d.auth])
return fmt.Sprintf("%s needs its %s", title, authNames[d.auth])
}
seen[addr] = true
}
@@ -473,11 +516,16 @@ func (m *setupModel) startCheck() tea.Cmd {
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
}}
for _, d := range m.db {
for i, d := range m.db[:m.shown()] {
roles, judge := m.dbRoles(), i >= 2
if judge {
roles = []preflight.Role{preflight.RoleNode}
}
if d.auth == authJoin {
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined})
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined,
judge: judge, target: "root@" + d.joined.Addr})
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
Addr: d.joined.Addr, Roles: m.dbRoles(), Remote: d.joined})
Addr: d.joined.Addr, Roles: roles, Remote: d.joined})
continue
}
g := &onboard.GoSSH{
@@ -490,9 +538,8 @@ func (m *setupModel) startCheck() tea.Cmd {
case authPassword:
g.Password = d.secret.Value()
}
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g})
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr,
Roles: m.dbRoles(), Remote: g})
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g, judge: judge, target: g.User + "@" + g.Addr})
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, Roles: roles, Remote: g})
}
m.checking = true
m.report = preflight.Report{}
@@ -535,7 +582,7 @@ func (m setupModel) dbRoles() []preflight.Role {
// targets are the database machines as user@address, for the command line.
func (m setupModel) targets() []string {
var out []string
for _, d := range m.db {
for _, d := range m.db[:2] {
if d.joined != nil {
out = append(out, "root@"+d.joined.Addr)
continue
@@ -545,11 +592,25 @@ func (m setupModel) targets() []string {
return out
}
// judgeTargets are the judge machines as user@address.
func (m setupModel) judgeTargets() []judgeMachine {
var out []judgeMachine
for _, d := range m.db[2:m.shown()] {
if d.joined != nil {
out = append(out, judgeMachine{target: "root@" + d.joined.Addr})
continue
}
out = append(out, judgeMachine{target: strings.TrimSpace(d.user.Value()) + "@" + strings.TrimSpace(d.addr.Value())})
}
return out
}
// command is what this setup is on the command line, for doing it again
// from a script — with keys in place, since a password cannot be given there.
func (m setupModel) command() string {
in := *m.in
in.patroni = m.targets()
in.judgeMachines = m.judgeTargets()
return strings.Join(append([]string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init"}, in.initFlags()...), " \\\n ")
}
@@ -558,6 +619,7 @@ func (m setupModel) command() string {
func (m *setupModel) startRun() tea.Cmd {
in := m.in
in.patroni = m.targets()
in.judgeMachines = nil
m.run = newInitModel(in, nil, false)
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
@@ -578,6 +640,11 @@ func (m *setupModel) startRun() tea.Cmd {
return err
}
}
if c.judge {
// Added as a node once the control plane is up.
in.judgeMachines = append(in.judgeMachines, judgeMachine{target: c.target, remote: c.remote})
continue
}
out, err := c.remote.Run(ctx, "hostname", nil)
if err != nil {
return fmt.Errorf("%s: %v", c.addr, err)
@@ -720,8 +787,12 @@ func (m setupModel) machinesView() string {
var b strings.Builder
b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" +
" PostgreSQL, Patroni and etcd are installed on them.\n\n")
for i, d := range m.db {
for i, d := range m.db[:m.shown()] {
base := i * mfCount
if i == 2 {
b.WriteString(" " + sDim.Render("Machines for the guardrails' judges alone: each becomes a node that runs only\n"+
" the judges, so a busy control plane cannot leave a request waiting for a judgment.") + "\n\n")
}
// Padded before it is coloured: the colour's escape codes count
// toward a width and threw the focused row out of line.
label := func(f int, s string) string {
@@ -731,7 +802,7 @@ func (m setupModel) machinesView() string {
}
return " " + s
}
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(machineTitle(i)))
if d.visible(mfAddr) {
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View())
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View())
@@ -756,7 +827,22 @@ func (m setupModel) machinesView() string {
}
b.WriteString("\n")
}
b.WriteString(button(m.mfocus == 2*mfCount, "Check the three machines") + "\n\n")
sel := m.shown() * mfCount
var opts []string
for n, name := range []string{"none", "one", "two"} {
if n == m.judges {
opts = append(opts, "("+sOK.Render("•")+") "+name)
} else {
opts = append(opts, "( ) "+sDim.Render(name))
}
}
judgeLabel := fmt.Sprintf(" %-15s", "Judge machines")
if m.mfocus == sel {
judgeLabel = fSelected.Render(fmt.Sprintf("› %-15s", "Judge machines"))
}
fmt.Fprintf(&b, " %s %s\n", judgeLabel, strings.Join(opts, " "))
b.WriteString(" " + sDim.Render("Without them the judges run on whichever node has room, added later.") + "\n\n")
b.WriteString(button(m.mfocus == sel+1, fmt.Sprintf("Check the %d machines", m.shown()+1)) + "\n\n")
b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n")
return b.String()
}
@@ -764,11 +850,11 @@ func (m setupModel) machinesView() string {
func (m setupModel) waitView() string {
var b strings.Builder
fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort)
for i, d := range m.db {
for i, d := range m.db[:m.shown()] {
if d.auth != authJoin {
continue
}
title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))
title := lipgloss.NewStyle().Bold(true).Render(machineTitle(i))
if d.joined != nil {
fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
continue
@@ -787,7 +873,7 @@ func (m setupModel) checkView() string {
var b strings.Builder
b.WriteString("\n")
if m.checking {
b.WriteString(" " + m.spin.View() + " checking the three machines…\n")
b.WriteString(" " + m.spin.View() + " checking the machines…\n")
return b.String()
}
switch m.report.Worst() {
@@ -796,7 +882,7 @@ func (m setupModel) checkView() string {
case preflight.Warn:
b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n")
default:
b.WriteString(" " + sOK.Render("✓ All three are ready.") + "\n")
b.WriteString(" " + sOK.Render("✓ All are ready.") + "\n")
}
machine := ""
for _, f := range m.report.Findings {
@@ -822,7 +908,7 @@ func (m setupModel) checkView() string {
func (m setupModel) planView() string {
var b strings.Builder
b.WriteString(m.choices.planView(m.width - navWidth - 6))
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The three machines") + "\n")
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The machines") + "\n")
here := "control plane, leading at first; model mirror; etcd"
there := "PostgreSQL under Patroni, etcd"
if m.in.standbys {
@@ -834,12 +920,16 @@ func (m setupModel) planView() string {
}
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), here)
granted := false
for _, d := range m.db {
for i, d := range m.db[:m.shown()] {
addr := strings.TrimSpace(d.addr.Value())
if d.joined != nil {
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
}
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), there)
role := there
if i >= 2 {
role = "a node for the guardrails' judges alone"
}
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), role)
granted = granted || d.sudo.Value() != ""
}
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +
@@ -851,7 +941,7 @@ func (m setupModel) planView() string {
for _, l := range strings.Split(m.command(), "\n") {
b.WriteString(" " + l + "\n")
}
for _, d := range m.db {
for _, d := range m.db[:m.shown()] {
if d.joined != nil {
b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n")
break