A three-machine install can give the judges machines of their own
CI / test (push) Successful in 5m10s
CI / test (push) Successful in 5m10s
The setup screen asks for up to two judge machines after the store's two, and server init takes --judge-node for each. Each is added as a node labelled llmgw.role: judge, and the judge's profile selects that label, with cores by init's rule over those machines. A judge beside a busy control plane refused 38 of 60 requests under load; alone, 2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8b122aae09
commit
51ca09287f
12 files changed
+346
-44
No files matched your search
+126
-36
@@ -89,14 +89,21 @@ func (d dbMachine) visible(f int) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// dbConn is a database machine as it is reached, whichever way.
|
||||
// dbConn is a database machine as it is reached, whichever way, or a judge
|
||||
// machine when judge is set.
|
||||
type dbConn struct {
|
||||
remote onboard.Remote
|
||||
addr string
|
||||
ssh *onboard.GoSSH
|
||||
joined *joinpkg.Remote
|
||||
judge bool
|
||||
target string
|
||||
}
|
||||
|
||||
// The machines the page asks for: the store's two, then up to maxJudges for
|
||||
// the guardrails' judges alone.
|
||||
const maxJudges = 2
|
||||
|
||||
func newDBMachine(n int) dbMachine {
|
||||
f := func(ph string, mask bool) textField {
|
||||
t := newTextField()
|
||||
@@ -119,6 +126,24 @@ func newDBMachine(n int) dbMachine {
|
||||
return m
|
||||
}
|
||||
|
||||
// newJudgeMachine is the fields of a machine for the judges.
|
||||
func newJudgeMachine(n int) dbMachine {
|
||||
d := newDBMachine(n + 2)
|
||||
d.addr.Placeholder = fmt.Sprintf("10.0.0.%d or judge%d.internal", 20+n, n)
|
||||
return d
|
||||
}
|
||||
|
||||
// shown is how many machines the page asks for.
|
||||
func (m setupModel) shown() int { return 2 + m.judges }
|
||||
|
||||
// machineTitle names machine i: the store's, or the judges'.
|
||||
func machineTitle(i int) string {
|
||||
if i < 2 {
|
||||
return fmt.Sprintf("Database machine %d", i+1)
|
||||
}
|
||||
return fmt.Sprintf("Judge machine %d", i-1)
|
||||
}
|
||||
|
||||
// field is the text field at a machine's position, nil for the selector.
|
||||
func (d *dbMachine) field(i int) *textField {
|
||||
switch i {
|
||||
@@ -156,8 +181,10 @@ type setupModel struct {
|
||||
|
||||
choices *installForm
|
||||
|
||||
db [2]dbMachine
|
||||
mfocus int // over both machines' fields, then the check button
|
||||
db [2 + maxJudges]dbMachine
|
||||
// judges is how many judge machines are asked for, none to maxJudges.
|
||||
judges int
|
||||
mfocus int // over the machines' fields, then the judges' choice, then the check button
|
||||
|
||||
conns []dbConn
|
||||
listener *joinpkg.Listener
|
||||
@@ -192,7 +219,8 @@ func newSetupModel(host string, ips []string) setupModel {
|
||||
in.judge.Replicas = 2
|
||||
return setupModel{
|
||||
ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true),
|
||||
db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,
|
||||
db: [2 + maxJudges]dbMachine{newDBMachine(1), newDBMachine(2), newJudgeMachine(1), newJudgeMachine(2)},
|
||||
spin: newSpinner(), width: 100, height: 30,
|
||||
log: &[]string{}, logMu: &sync.Mutex{},
|
||||
}
|
||||
}
|
||||
@@ -327,7 +355,7 @@ func (m *setupModel) focusMachine() {
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.mfocus < 2*mfCount {
|
||||
if m.mfocus < m.shown()*mfCount {
|
||||
if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil {
|
||||
t.Focus()
|
||||
}
|
||||
@@ -335,7 +363,8 @@ func (m *setupModel) focusMachine() {
|
||||
}
|
||||
|
||||
func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
n := 2*mfCount + 1
|
||||
sel := m.shown() * mfCount
|
||||
n := sel + 2
|
||||
switch k.Type {
|
||||
case tea.KeyTab, tea.KeyDown:
|
||||
m.step(1, n)
|
||||
@@ -344,7 +373,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
m.step(-1, n)
|
||||
return m, nil
|
||||
case tea.KeyEnter:
|
||||
if m.mfocus < 2*mfCount {
|
||||
if m.mfocus <= sel {
|
||||
m.step(1, n)
|
||||
return m, nil
|
||||
}
|
||||
@@ -361,7 +390,17 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
cmd := m.startCheck()
|
||||
return m, cmd
|
||||
}
|
||||
if m.mfocus == 2*mfCount {
|
||||
if m.mfocus == sel {
|
||||
switch k.String() {
|
||||
case "left", "h":
|
||||
m.judges = (m.judges + maxJudges) % (maxJudges + 1)
|
||||
case "right", "l", " ":
|
||||
m.judges = (m.judges + 1) % (maxJudges + 1)
|
||||
}
|
||||
m.mfocus = m.shown() * mfCount
|
||||
return m, nil
|
||||
}
|
||||
if m.mfocus > sel {
|
||||
return m, nil
|
||||
}
|
||||
d := &m.db[m.mfocus/mfCount]
|
||||
@@ -391,7 +430,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
func (m *setupModel) step(d, n int) {
|
||||
for range n {
|
||||
m.mfocus = (m.mfocus + d + n) % n
|
||||
if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
|
||||
if m.mfocus >= m.shown()*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -399,7 +438,7 @@ func (m *setupModel) step(d, n int) {
|
||||
}
|
||||
|
||||
func (m setupModel) allJoined() bool {
|
||||
for _, d := range m.db {
|
||||
for _, d := range m.db[:m.shown()] {
|
||||
if d.auth == authJoin && d.joined == nil {
|
||||
return false
|
||||
}
|
||||
@@ -411,7 +450,7 @@ func (m setupModel) allJoined() bool {
|
||||
// codes, and waits for them; it says whether there is anything to wait for.
|
||||
func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||||
var cmds []tea.Cmd
|
||||
for i := range m.db {
|
||||
for i := range m.db[:m.shown()] {
|
||||
d := &m.db[i]
|
||||
if d.auth != authJoin || d.joined != nil {
|
||||
continue
|
||||
@@ -429,7 +468,11 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||||
m.listener = l
|
||||
}
|
||||
if d.arrive == nil {
|
||||
c, ch := m.listener.Code("database")
|
||||
role := "database"
|
||||
if i >= 2 {
|
||||
role = "judge"
|
||||
}
|
||||
c, ch := m.listener.Code(role)
|
||||
d.code, d.arrive = c.String(), ch
|
||||
}
|
||||
ch, i := d.arrive, i
|
||||
@@ -440,20 +483,20 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||||
|
||||
func (m setupModel) validateMachines() string {
|
||||
seen := map[string]bool{m.thisAddr(): true, m.host: true}
|
||||
for i, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
if d.auth == authJoin {
|
||||
continue
|
||||
}
|
||||
addr := strings.TrimSpace(d.addr.Value())
|
||||
addr, title := strings.TrimSpace(d.addr.Value()), strings.ToLower(machineTitle(i)[:1])+machineTitle(i)[1:]
|
||||
switch {
|
||||
case addr == "":
|
||||
return fmt.Sprintf("database machine %d has no address", i+1)
|
||||
return title + " has no address"
|
||||
case seen[addr]:
|
||||
return fmt.Sprintf("%s is given twice, or is this machine: the three must be different machines", addr)
|
||||
return fmt.Sprintf("%s is given twice, or is this machine: each must be a different machine", addr)
|
||||
case strings.TrimSpace(d.user.Value()) == "":
|
||||
return fmt.Sprintf("database machine %d has no user to log in as", i+1)
|
||||
return title + " has no user to log in as"
|
||||
case d.auth != authAgent && d.secret.Value() == "":
|
||||
return fmt.Sprintf("database machine %d needs its %s", i+1, authNames[d.auth])
|
||||
return fmt.Sprintf("%s needs its %s", title, authNames[d.auth])
|
||||
}
|
||||
seen[addr] = true
|
||||
}
|
||||
@@ -473,11 +516,16 @@ func (m *setupModel) startCheck() tea.Cmd {
|
||||
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
|
||||
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
|
||||
}}
|
||||
for _, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
roles, judge := m.dbRoles(), i >= 2
|
||||
if judge {
|
||||
roles = []preflight.Role{preflight.RoleNode}
|
||||
}
|
||||
if d.auth == authJoin {
|
||||
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined})
|
||||
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined,
|
||||
judge: judge, target: "root@" + d.joined.Addr})
|
||||
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
|
||||
Addr: d.joined.Addr, Roles: m.dbRoles(), Remote: d.joined})
|
||||
Addr: d.joined.Addr, Roles: roles, Remote: d.joined})
|
||||
continue
|
||||
}
|
||||
g := &onboard.GoSSH{
|
||||
@@ -490,9 +538,8 @@ func (m *setupModel) startCheck() tea.Cmd {
|
||||
case authPassword:
|
||||
g.Password = d.secret.Value()
|
||||
}
|
||||
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g})
|
||||
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr,
|
||||
Roles: m.dbRoles(), Remote: g})
|
||||
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g, judge: judge, target: g.User + "@" + g.Addr})
|
||||
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, Roles: roles, Remote: g})
|
||||
}
|
||||
m.checking = true
|
||||
m.report = preflight.Report{}
|
||||
@@ -535,7 +582,7 @@ func (m setupModel) dbRoles() []preflight.Role {
|
||||
// targets are the database machines as user@address, for the command line.
|
||||
func (m setupModel) targets() []string {
|
||||
var out []string
|
||||
for _, d := range m.db {
|
||||
for _, d := range m.db[:2] {
|
||||
if d.joined != nil {
|
||||
out = append(out, "root@"+d.joined.Addr)
|
||||
continue
|
||||
@@ -545,11 +592,25 @@ func (m setupModel) targets() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeTargets are the judge machines as user@address.
|
||||
func (m setupModel) judgeTargets() []judgeMachine {
|
||||
var out []judgeMachine
|
||||
for _, d := range m.db[2:m.shown()] {
|
||||
if d.joined != nil {
|
||||
out = append(out, judgeMachine{target: "root@" + d.joined.Addr})
|
||||
continue
|
||||
}
|
||||
out = append(out, judgeMachine{target: strings.TrimSpace(d.user.Value()) + "@" + strings.TrimSpace(d.addr.Value())})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// command is what this setup is on the command line, for doing it again
|
||||
// from a script — with keys in place, since a password cannot be given there.
|
||||
func (m setupModel) command() string {
|
||||
in := *m.in
|
||||
in.patroni = m.targets()
|
||||
in.judgeMachines = m.judgeTargets()
|
||||
return strings.Join(append([]string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init"}, in.initFlags()...), " \\\n ")
|
||||
}
|
||||
|
||||
@@ -558,6 +619,7 @@ func (m setupModel) command() string {
|
||||
func (m *setupModel) startRun() tea.Cmd {
|
||||
in := m.in
|
||||
in.patroni = m.targets()
|
||||
in.judgeMachines = nil
|
||||
m.run = newInitModel(in, nil, false)
|
||||
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
|
||||
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
|
||||
@@ -578,6 +640,11 @@ func (m *setupModel) startRun() tea.Cmd {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if c.judge {
|
||||
// Added as a node once the control plane is up.
|
||||
in.judgeMachines = append(in.judgeMachines, judgeMachine{target: c.target, remote: c.remote})
|
||||
continue
|
||||
}
|
||||
out, err := c.remote.Run(ctx, "hostname", nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %v", c.addr, err)
|
||||
@@ -720,8 +787,12 @@ func (m setupModel) machinesView() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" +
|
||||
" PostgreSQL, Patroni and etcd are installed on them.\n\n")
|
||||
for i, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
base := i * mfCount
|
||||
if i == 2 {
|
||||
b.WriteString(" " + sDim.Render("Machines for the guardrails' judges alone: each becomes a node that runs only\n"+
|
||||
" the judges, so a busy control plane cannot leave a request waiting for a judgment.") + "\n\n")
|
||||
}
|
||||
// Padded before it is coloured: the colour's escape codes count
|
||||
// toward a width and threw the focused row out of line.
|
||||
label := func(f int, s string) string {
|
||||
@@ -731,7 +802,7 @@ func (m setupModel) machinesView() string {
|
||||
}
|
||||
return " " + s
|
||||
}
|
||||
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
|
||||
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(machineTitle(i)))
|
||||
if d.visible(mfAddr) {
|
||||
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View())
|
||||
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View())
|
||||
@@ -756,7 +827,22 @@ func (m setupModel) machinesView() string {
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString(button(m.mfocus == 2*mfCount, "Check the three machines") + "\n\n")
|
||||
sel := m.shown() * mfCount
|
||||
var opts []string
|
||||
for n, name := range []string{"none", "one", "two"} {
|
||||
if n == m.judges {
|
||||
opts = append(opts, "("+sOK.Render("•")+") "+name)
|
||||
} else {
|
||||
opts = append(opts, "( ) "+sDim.Render(name))
|
||||
}
|
||||
}
|
||||
judgeLabel := fmt.Sprintf(" %-15s", "Judge machines")
|
||||
if m.mfocus == sel {
|
||||
judgeLabel = fSelected.Render(fmt.Sprintf("› %-15s", "Judge machines"))
|
||||
}
|
||||
fmt.Fprintf(&b, " %s %s\n", judgeLabel, strings.Join(opts, " "))
|
||||
b.WriteString(" " + sDim.Render("Without them the judges run on whichever node has room, added later.") + "\n\n")
|
||||
b.WriteString(button(m.mfocus == sel+1, fmt.Sprintf("Check the %d machines", m.shown()+1)) + "\n\n")
|
||||
b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -764,11 +850,11 @@ func (m setupModel) machinesView() string {
|
||||
func (m setupModel) waitView() string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort)
|
||||
for i, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
if d.auth != authJoin {
|
||||
continue
|
||||
}
|
||||
title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))
|
||||
title := lipgloss.NewStyle().Bold(true).Render(machineTitle(i))
|
||||
if d.joined != nil {
|
||||
fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
|
||||
continue
|
||||
@@ -787,7 +873,7 @@ func (m setupModel) checkView() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n")
|
||||
if m.checking {
|
||||
b.WriteString(" " + m.spin.View() + " checking the three machines…\n")
|
||||
b.WriteString(" " + m.spin.View() + " checking the machines…\n")
|
||||
return b.String()
|
||||
}
|
||||
switch m.report.Worst() {
|
||||
@@ -796,7 +882,7 @@ func (m setupModel) checkView() string {
|
||||
case preflight.Warn:
|
||||
b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n")
|
||||
default:
|
||||
b.WriteString(" " + sOK.Render("✓ All three are ready.") + "\n")
|
||||
b.WriteString(" " + sOK.Render("✓ All are ready.") + "\n")
|
||||
}
|
||||
machine := ""
|
||||
for _, f := range m.report.Findings {
|
||||
@@ -822,7 +908,7 @@ func (m setupModel) checkView() string {
|
||||
func (m setupModel) planView() string {
|
||||
var b strings.Builder
|
||||
b.WriteString(m.choices.planView(m.width - navWidth - 6))
|
||||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The three machines") + "\n")
|
||||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The machines") + "\n")
|
||||
here := "control plane, leading at first; model mirror; etcd"
|
||||
there := "PostgreSQL under Patroni, etcd"
|
||||
if m.in.standbys {
|
||||
@@ -834,12 +920,16 @@ func (m setupModel) planView() string {
|
||||
}
|
||||
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), here)
|
||||
granted := false
|
||||
for _, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
addr := strings.TrimSpace(d.addr.Value())
|
||||
if d.joined != nil {
|
||||
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
|
||||
}
|
||||
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), there)
|
||||
role := there
|
||||
if i >= 2 {
|
||||
role = "a node for the guardrails' judges alone"
|
||||
}
|
||||
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), role)
|
||||
granted = granted || d.sudo.Value() != ""
|
||||
}
|
||||
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +
|
||||
@@ -851,7 +941,7 @@ func (m setupModel) planView() string {
|
||||
for _, l := range strings.Split(m.command(), "\n") {
|
||||
b.WriteString(" " + l + "\n")
|
||||
}
|
||||
for _, d := range m.db {
|
||||
for _, d := range m.db[:m.shown()] {
|
||||
if d.joined != nil {
|
||||
b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n")
|
||||
break
|
||||
|
||||
Reference in new issue
Block a user