A three-machine install can give the judges machines of their own
CI / test (push) Successful in 5m10s

The setup screen asks for up to two judge machines after the store's
two, and server init takes --judge-node for each. Each is added as a
node labelled llmgw.role: judge, and the judge's profile selects that
label, with cores by init's rule over those machines. A judge beside a
busy control plane refused 38 of 60 requests under load; alone, 2.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-19 17:45:00 +02:00
1 parent 8b122aae09
commit 51ca09287f
12 files changed
+346 -44

No files matched your search

+126 -36
View File
@@ -89,14 +89,21 @@ func (d dbMachine) visible(f int) bool {
return true
}
// dbConn is a database machine as it is reached, whichever way.
// dbConn is a database machine as it is reached, whichever way, or a judge
// machine when judge is set.
type dbConn struct {
remote onboard.Remote
addr string
ssh *onboard.GoSSH
joined *joinpkg.Remote
judge bool
target string
}
// The machines the page asks for: the store's two, then up to maxJudges for
// the guardrails' judges alone.
const maxJudges = 2
func newDBMachine(n int) dbMachine {
f := func(ph string, mask bool) textField {
t := newTextField()
@@ -119,6 +126,24 @@ func newDBMachine(n int) dbMachine {
return m
}
// newJudgeMachine is the fields of a machine for the judges.
func newJudgeMachine(n int) dbMachine {
d := newDBMachine(n + 2)
d.addr.Placeholder = fmt.Sprintf("10.0.0.%d or judge%d.internal", 20+n, n)
return d
}
// shown is how many machines the page asks for.
func (m setupModel) shown() int { return 2 + m.judges }
// machineTitle names machine i: the store's, or the judges'.
func machineTitle(i int) string {
if i < 2 {
return fmt.Sprintf("Database machine %d", i+1)
}
return fmt.Sprintf("Judge machine %d", i-1)
}
// field is the text field at a machine's position, nil for the selector.
func (d *dbMachine) field(i int) *textField {
switch i {
@@ -156,8 +181,10 @@ type setupModel struct {
choices *installForm
db [2]dbMachine
mfocus int // over both machines' fields, then the check button
db [2 + maxJudges]dbMachine
// judges is how many judge machines are asked for, none to maxJudges.
judges int
mfocus int // over the machines' fields, then the judges' choice, then the check button
conns []dbConn
listener *joinpkg.Listener
@@ -192,7 +219,8 @@ func newSetupModel(host string, ips []string) setupModel {
in.judge.Replicas = 2
return setupModel{
ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true),
db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,
db: [2 + maxJudges]dbMachine{newDBMachine(1), newDBMachine(2), newJudgeMachine(1), newJudgeMachine(2)},
spin: newSpinner(), width: 100, height: 30,
log: &[]string{}, logMu: &sync.Mutex{},
}
}
@@ -327,7 +355,7 @@ func (m *setupModel) focusMachine() {
}
}
}
if m.mfocus < 2*mfCount {
if m.mfocus < m.shown()*mfCount {
if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil {
t.Focus()
}
@@ -335,7 +363,8 @@ func (m *setupModel) focusMachine() {
}
func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
n := 2*mfCount + 1
sel := m.shown() * mfCount
n := sel + 2
switch k.Type {
case tea.KeyTab, tea.KeyDown:
m.step(1, n)
@@ -344,7 +373,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
m.step(-1, n)
return m, nil
case tea.KeyEnter:
if m.mfocus < 2*mfCount {
if m.mfocus <= sel {
m.step(1, n)
return m, nil
}
@@ -361,7 +390,17 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
cmd := m.startCheck()
return m, cmd
}
if m.mfocus == 2*mfCount {
if m.mfocus == sel {
switch k.String() {
case "left", "h":
m.judges = (m.judges + maxJudges) % (maxJudges + 1)
case "right", "l", " ":
m.judges = (m.judges + 1) % (maxJudges + 1)
}
m.mfocus = m.shown() * mfCount
return m, nil
}
if m.mfocus > sel {
return m, nil
}
d := &m.db[m.mfocus/mfCount]
@@ -391,7 +430,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
func (m *setupModel) step(d, n int) {
for range n {
m.mfocus = (m.mfocus + d + n) % n
if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
if m.mfocus >= m.shown()*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
break
}
}
@@ -399,7 +438,7 @@ func (m *setupModel) step(d, n int) {
}
func (m setupModel) allJoined() bool {
for _, d := range m.db {
for _, d := range m.db[:m.shown()] {
if d.auth == authJoin && d.joined == nil {
return false
}
@@ -411,7 +450,7 @@ func (m setupModel) allJoined() bool {
// codes, and waits for them; it says whether there is anything to wait for.
func (m *setupModel) startJoins() (tea.Cmd, bool) {
var cmds []tea.Cmd
for i := range m.db {
for i := range m.db[:m.shown()] {
d := &m.db[i]
if d.auth != authJoin || d.joined != nil {
continue
@@ -429,7 +468,11 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) {
m.listener = l
}
if d.arrive == nil {
c, ch := m.listener.Code("database")
role := "database"
if i >= 2 {
role = "judge"
}
c, ch := m.listener.Code(role)
d.code, d.arrive = c.String(), ch
}
ch, i := d.arrive, i
@@ -440,20 +483,20 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) {
func (m setupModel) validateMachines() string {
seen := map[string]bool{m.thisAddr(): true, m.host: true}
for i, d := range m.db {
for i, d := range m.db[:m.shown()] {
if d.auth == authJoin {
continue
}
addr := strings.TrimSpace(d.addr.Value())
addr, title := strings.TrimSpace(d.addr.Value()), strings.ToLower(machineTitle(i)[:1])+machineTitle(i)[1:]
switch {
case addr == "":
return fmt.Sprintf("database machine %d has no address", i+1)
return title + " has no address"
case seen[addr]:
return fmt.Sprintf("%s is given twice, or is this machine: the three must be different machines", addr)
return fmt.Sprintf("%s is given twice, or is this machine: each must be a different machine", addr)
case strings.TrimSpace(d.user.Value()) == "":
return fmt.Sprintf("database machine %d has no user to log in as", i+1)
return title + " has no user to log in as"
case d.auth != authAgent && d.secret.Value() == "":
return fmt.Sprintf("database machine %d needs its %s", i+1, authNames[d.auth])
return fmt.Sprintf("%s needs its %s", title, authNames[d.auth])
}
seen[addr] = true
}
@@ -473,11 +516,16 @@ func (m *setupModel) startCheck() tea.Cmd {
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
}}
for _, d := range m.db {
for i, d := range m.db[:m.shown()] {
roles, judge := m.dbRoles(), i >= 2
if judge {
roles = []preflight.Role{preflight.RoleNode}
}
if d.auth == authJoin {
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined})
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined,
judge: judge, target: "root@" + d.joined.Addr})
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
Addr: d.joined.Addr, Roles: m.dbRoles(), Remote: d.joined})
Addr: d.joined.Addr, Roles: roles, Remote: d.joined})
continue
}
g := &onboard.GoSSH{
@@ -490,9 +538,8 @@ func (m *setupModel) startCheck() tea.Cmd {
case authPassword:
g.Password = d.secret.Value()
}
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g})
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr,
Roles: m.dbRoles(), Remote: g})
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g, judge: judge, target: g.User + "@" + g.Addr})
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, Roles: roles, Remote: g})
}
m.checking = true
m.report = preflight.Report{}
@@ -535,7 +582,7 @@ func (m setupModel) dbRoles() []preflight.Role {
// targets are the database machines as user@address, for the command line.
func (m setupModel) targets() []string {
var out []string
for _, d := range m.db {
for _, d := range m.db[:2] {
if d.joined != nil {
out = append(out, "root@"+d.joined.Addr)
continue
@@ -545,11 +592,25 @@ func (m setupModel) targets() []string {
return out
}
// judgeTargets are the judge machines as user@address.
func (m setupModel) judgeTargets() []judgeMachine {
var out []judgeMachine
for _, d := range m.db[2:m.shown()] {
if d.joined != nil {
out = append(out, judgeMachine{target: "root@" + d.joined.Addr})
continue
}
out = append(out, judgeMachine{target: strings.TrimSpace(d.user.Value()) + "@" + strings.TrimSpace(d.addr.Value())})
}
return out
}
// command is what this setup is on the command line, for doing it again
// from a script — with keys in place, since a password cannot be given there.
func (m setupModel) command() string {
in := *m.in
in.patroni = m.targets()
in.judgeMachines = m.judgeTargets()
return strings.Join(append([]string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init"}, in.initFlags()...), " \\\n ")
}
@@ -558,6 +619,7 @@ func (m setupModel) command() string {
func (m *setupModel) startRun() tea.Cmd {
in := m.in
in.patroni = m.targets()
in.judgeMachines = nil
m.run = newInitModel(in, nil, false)
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
@@ -578,6 +640,11 @@ func (m *setupModel) startRun() tea.Cmd {
return err
}
}
if c.judge {
// Added as a node once the control plane is up.
in.judgeMachines = append(in.judgeMachines, judgeMachine{target: c.target, remote: c.remote})
continue
}
out, err := c.remote.Run(ctx, "hostname", nil)
if err != nil {
return fmt.Errorf("%s: %v", c.addr, err)
@@ -720,8 +787,12 @@ func (m setupModel) machinesView() string {
var b strings.Builder
b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" +
" PostgreSQL, Patroni and etcd are installed on them.\n\n")
for i, d := range m.db {
for i, d := range m.db[:m.shown()] {
base := i * mfCount
if i == 2 {
b.WriteString(" " + sDim.Render("Machines for the guardrails' judges alone: each becomes a node that runs only\n"+
" the judges, so a busy control plane cannot leave a request waiting for a judgment.") + "\n\n")
}
// Padded before it is coloured: the colour's escape codes count
// toward a width and threw the focused row out of line.
label := func(f int, s string) string {
@@ -731,7 +802,7 @@ func (m setupModel) machinesView() string {
}
return " " + s
}
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(machineTitle(i)))
if d.visible(mfAddr) {
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View())
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View())
@@ -756,7 +827,22 @@ func (m setupModel) machinesView() string {
}
b.WriteString("\n")
}
b.WriteString(button(m.mfocus == 2*mfCount, "Check the three machines") + "\n\n")
sel := m.shown() * mfCount
var opts []string
for n, name := range []string{"none", "one", "two"} {
if n == m.judges {
opts = append(opts, "("+sOK.Render("•")+") "+name)
} else {
opts = append(opts, "( ) "+sDim.Render(name))
}
}
judgeLabel := fmt.Sprintf(" %-15s", "Judge machines")
if m.mfocus == sel {
judgeLabel = fSelected.Render(fmt.Sprintf("› %-15s", "Judge machines"))
}
fmt.Fprintf(&b, " %s %s\n", judgeLabel, strings.Join(opts, " "))
b.WriteString(" " + sDim.Render("Without them the judges run on whichever node has room, added later.") + "\n\n")
b.WriteString(button(m.mfocus == sel+1, fmt.Sprintf("Check the %d machines", m.shown()+1)) + "\n\n")
b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n")
return b.String()
}
@@ -764,11 +850,11 @@ func (m setupModel) machinesView() string {
func (m setupModel) waitView() string {
var b strings.Builder
fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort)
for i, d := range m.db {
for i, d := range m.db[:m.shown()] {
if d.auth != authJoin {
continue
}
title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))
title := lipgloss.NewStyle().Bold(true).Render(machineTitle(i))
if d.joined != nil {
fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
continue
@@ -787,7 +873,7 @@ func (m setupModel) checkView() string {
var b strings.Builder
b.WriteString("\n")
if m.checking {
b.WriteString(" " + m.spin.View() + " checking the three machines…\n")
b.WriteString(" " + m.spin.View() + " checking the machines…\n")
return b.String()
}
switch m.report.Worst() {
@@ -796,7 +882,7 @@ func (m setupModel) checkView() string {
case preflight.Warn:
b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n")
default:
b.WriteString(" " + sOK.Render("✓ All three are ready.") + "\n")
b.WriteString(" " + sOK.Render("✓ All are ready.") + "\n")
}
machine := ""
for _, f := range m.report.Findings {
@@ -822,7 +908,7 @@ func (m setupModel) checkView() string {
func (m setupModel) planView() string {
var b strings.Builder
b.WriteString(m.choices.planView(m.width - navWidth - 6))
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The three machines") + "\n")
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The machines") + "\n")
here := "control plane, leading at first; model mirror; etcd"
there := "PostgreSQL under Patroni, etcd"
if m.in.standbys {
@@ -834,12 +920,16 @@ func (m setupModel) planView() string {
}
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), here)
granted := false
for _, d := range m.db {
for i, d := range m.db[:m.shown()] {
addr := strings.TrimSpace(d.addr.Value())
if d.joined != nil {
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
}
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), there)
role := there
if i >= 2 {
role = "a node for the guardrails' judges alone"
}
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), role)
granted = granted || d.sudo.Value() != ""
}
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +
@@ -851,7 +941,7 @@ func (m setupModel) planView() string {
for _, l := range strings.Split(m.command(), "\n") {
b.WriteString(" " + l + "\n")
}
for _, d := range m.db {
for _, d := range m.db[:m.shown()] {
if d.joined != nil {
b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n")
break