A three-machine install can give the judges machines of their own
CI / test (push) Successful in 5m10s
CI / test (push) Successful in 5m10s
The setup screen asks for up to two judge machines after the store's two, and server init takes --judge-node for each. Each is added as a node labelled llmgw.role: judge, and the judge's profile selects that label, with cores by init's rule over those machines. A judge beside a busy control plane refused 38 of 60 requests under load; alone, 2. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
8b122aae09
commit
51ca09287f
12 files changed
+346
-44
No files matched your search
+3
-1
@@ -33,7 +33,9 @@ What is left, most needed first. The detail is in the item each names.
|
||||
sync.
|
||||
6. ~~**J2**~~ — done 2026-10-19: a judge the leader adds by itself is
|
||||
sized as init's is (2 GB, init's cores, no prompt cache).
|
||||
7. **J3** — HA setup puts the judges on nodes of their own. (S)
|
||||
7. **J3** — HA setup puts the judges on nodes of their own. (S) Code
|
||||
done 2026-10-19 (setup's judge machines, `--judge-node`); the load test
|
||||
on a fresh HA install is still to do.
|
||||
8. **J8** — a judge budget that fits the machine, measured at install. (S)
|
||||
9. **F5** — large imports limited by the embedding engine's slots. (M)
|
||||
10. **I1** — answers to similar questions from the cache. (M)
|
||||
|
||||
@@ -26,6 +26,14 @@ APIs change between minors without a deprecation period.
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Added
|
||||
|
||||
- **Machines of the judges' own in a three-machine install.** The setup screen asks for up to two judge machines after
|
||||
the store's two; `server init` takes `--judge-node user@host` for each. Each is added as a node labelled
|
||||
`llmgw.role: judge`, and the judge's profile selects it, so the judges run there alone. Under load on a 4-core
|
||||
control plane that was also a node, 38 of 60 requests were refused for want of a judgment. With the judges on a node
|
||||
of their own, 2 were.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **A judge the leader serves by itself is sized as init's is.** When a guard policy names a judge the catalog lacks,
|
||||
|
||||
+13
-1
@@ -199,7 +199,19 @@ a busy one, or a VM without AVX2 (see
|
||||
[step 6](#6-setting-the-nodes-cpu-model-if-it-is-a-vm)), the judge is the
|
||||
first thing to fail. The choices there are a longer `latency_budget`,
|
||||
`fail: open` (serve the request, recorded as flagged), the judge on a GPU,
|
||||
or `--no-guard-model`. Defamation, specialized advice,
|
||||
or `--no-guard-model`.
|
||||
|
||||
In a three-machine install the judges are better on machines of their own.
|
||||
On a 4-core control plane that was also a node, 38 of 60 requests under load
|
||||
were refused for want of a judgment. With the judges on a node of their own
|
||||
(4 cores, two judges of two cores each), 2 were refused. The setup screen asks
|
||||
for up to two **judge machines** after the store's two, or give
|
||||
`--judge-node user@host` to `server init`, once for each. Each is added as a
|
||||
node labelled `llmgw.role: judge`, and the judge's profile selects that
|
||||
label, so the judges run only there and nothing else does. Four cores and
|
||||
8 GB are enough for two judges.
|
||||
|
||||
Defamation, specialized advice,
|
||||
intellectual property and elections are left out of what it refuses: they
|
||||
catch too much ordinary work. [guardrails.md](guardrails.md) has the rest.
|
||||
|
||||
|
||||
@@ -149,7 +149,7 @@ func TestAnEarlierInstallsJudgeIsRefittedOnce(t *testing.T) {
|
||||
lib := guard.Config{Policies: map[string]guard.Policy{
|
||||
"baseline": {Judge: &guard.JudgePolicy{Model: "qwen3guard-0.6b", Input: "block"}},
|
||||
}}
|
||||
earlier := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 1)
|
||||
earlier := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 1, nil)
|
||||
earlier.Profiles[0].MinMemoryGB, earlier.Profiles[0].Args = 4, []string{"--ctx-size", "8192"}
|
||||
inCatalog := []catalog.Model{earlier}
|
||||
fp := &fakeProfiles{}
|
||||
@@ -178,7 +178,7 @@ func TestAnEarlierInstallsJudgeIsRefittedOnce(t *testing.T) {
|
||||
}
|
||||
|
||||
// Changed by somebody: an extra arg. Left as it is.
|
||||
mine := guardModel("http://x/g.gguf", 2, 3)
|
||||
mine := guardModel("http://x/g.gguf", 2, 3, nil)
|
||||
mine.Profiles[0].Args = []string{"--ctx-size", "16384"}
|
||||
inCatalog = []catalog.Model{mine}
|
||||
fp.set, k.looked = nil, nil
|
||||
|
||||
@@ -80,6 +80,10 @@ type serverInit struct {
|
||||
// setup screen does with the password it was given; when set, patroni
|
||||
// only names them.
|
||||
patroniNodes []pgha.Node
|
||||
// judgeMachines are machines for the guardrails' judges alone, added as
|
||||
// nodes labelled for them; judgeCPUs their cores, once added.
|
||||
judgeMachines []judgeMachine
|
||||
judgeCPUs []int
|
||||
// noMirror, noNode and noModel turn off the parts of a first install that
|
||||
// make it serve something: a mirror, this machine as a node, and a small
|
||||
// model on it.
|
||||
@@ -234,6 +238,8 @@ func newServerInitCmd() *cobra.Command {
|
||||
f.StringVar(&judgeCats, "judge-categories", strings.Join(defaultJudgeCategories, ","),
|
||||
"what the judge refuses, comma-separated, of: "+strings.Join(judgeCategories, ", "))
|
||||
f.IntVar(&in.judge.Replicas, "judge-replicas", 2, "how many copies of the judge run, each on CPUs")
|
||||
f.Var(judgeMachineFlag{&in.judgeMachines}, "judge-node",
|
||||
"user@host of a machine for the guardrails' judges alone: added as a node, and the judges run only there; repeatable")
|
||||
f.DurationVar(&in.judge.Budget, "judge-budget", 5*time.Second, "how long a request waits for the judge")
|
||||
f.StringVar(&in.language, "language", "", "the portal's language for a browser that asks for neither: en or tr")
|
||||
f.BoolVar(&in.signupOpen, "signup-open", false, "let anyone ask for an account on the portal, to be approved by an administrator")
|
||||
@@ -317,6 +323,11 @@ func (in *serverInit) run(ctx context.Context) error {
|
||||
if err := in.ensureWebhooks(ctx); err != nil {
|
||||
fmt.Fprintf(in.out, " !! webhooks: %v\n", err)
|
||||
}
|
||||
if hasSystemd && len(in.judgeMachines) > 0 {
|
||||
if err := in.ensureJudgeNodes(ctx); err != nil {
|
||||
return fmt.Errorf("judge nodes: %w", err)
|
||||
}
|
||||
}
|
||||
if in.judgeWanted() && in.configJudgesWith(guardModelName) {
|
||||
// Not fatal: without its judge the baseline policy still checks for
|
||||
// secrets and injection, and a judged request is served flagged.
|
||||
|
||||
@@ -325,6 +325,9 @@ func (in *serverInit) initFlags() []string {
|
||||
for _, t := range in.patroni {
|
||||
add("--patroni", t)
|
||||
}
|
||||
for _, m := range in.judgeMachines {
|
||||
add("--judge-node", m.target)
|
||||
}
|
||||
switch {
|
||||
case in.mirrorURL != "":
|
||||
add("--mirror", in.mirrorURL)
|
||||
|
||||
@@ -202,7 +202,7 @@ func TestTheJudgesAndTheStarterShareTheCPUsRatherThanEachTakingAll(t *testing.T)
|
||||
t.Errorf("%+v: judge %d starter %d, want %d %d", c.in, got.judge, got.starter, c.judge, c.starter)
|
||||
}
|
||||
}
|
||||
m := guardModel("http://x/g.gguf", 2, 1)
|
||||
m := guardModel("http://x/g.gguf", 2, 1, nil)
|
||||
if p := m.Profiles[0]; p.CPUCores != 1 || p.MinMemoryGB != 2 || !slices.Contains(p.Args, "--cache-ram") {
|
||||
t.Errorf("judge profile: %d cores, %d GB, args %v", p.CPUCores, p.MinMemoryGB, p.Args)
|
||||
}
|
||||
|
||||
@@ -485,7 +485,7 @@ func starterModel(spec starterSpec, ctx, cores int, source string, gpuClasses []
|
||||
|
||||
// guardModel is the judge's catalog entry: CPU only, so it takes no card from
|
||||
// the models people use, on whichever node has room; replicas copies of it.
|
||||
func guardModel(source string, replicas, cores int) catalog.Model {
|
||||
func guardModel(source string, replicas, cores int, selector map[string]string) catalog.Model {
|
||||
return catalog.Model{
|
||||
Name: guardModelName,
|
||||
Description: "The guardrails' judge: reads prompts and answers for unsafe content. Not offered to clients.",
|
||||
@@ -503,6 +503,8 @@ func guardModel(source string, replicas, cores int) catalog.Model {
|
||||
// that left a 7 GB machine room for only one of the two.
|
||||
Name: "cpu", Engine: "llamacpp", Priority: 100, MinMemoryGB: judgeMemoryGB, CPUCores: cores,
|
||||
Args: []string{"--ctx-size", "8192", "--cache-ram", "0"},
|
||||
// The judge machines, when the install was given some.
|
||||
NodeSelector: selector,
|
||||
}},
|
||||
}
|
||||
}
|
||||
@@ -524,7 +526,8 @@ func (in *serverInit) ensureStarter(ctx context.Context) error {
|
||||
func (in *serverInit) ensureGuardModel(ctx context.Context) error {
|
||||
return in.ensureServed(ctx, guardModelName, guardModelSource, guardModelVersion, false,
|
||||
func(url string, _ []string) catalog.Model {
|
||||
return guardModel(url, in.judgeFor().Replicas, in.cpuPlan().judge)
|
||||
cores, selector := in.judgePlacement()
|
||||
return guardModel(url, in.judgeFor().Replicas, cores, selector)
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -171,7 +171,7 @@ func TestTheInstalledConfigurationIsHardenedAndLoads(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestTheGuardModelIsAValidJudge(t *testing.T) {
|
||||
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2)
|
||||
m := guardModel("http://10.0.0.5:8081/qwen3guard-0.6b/guard/g.gguf", 2, 2, nil)
|
||||
c, err := catalog.New([]catalog.Model{m})
|
||||
if err != nil {
|
||||
t.Fatalf("not a valid catalog entry: %v", err)
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/settings"
|
||||
)
|
||||
|
||||
// Machines of the judges' own. Under load on a 4-core control plane that was
|
||||
// also a node, 38 of 60 requests were refused for want of a judgment; with the
|
||||
// judges on a node of their own the same test refused 2. A judge machine is
|
||||
// added as a node carrying judgeRoleLabel, and the judge's profile selects
|
||||
// that label, so nothing else is placed beside it and it on nothing else.
|
||||
|
||||
// judgeRoleLabel marks a node as the judges'.
|
||||
const (
|
||||
judgeRoleLabel = "llmgw.role"
|
||||
judgeRoleValue = "judge"
|
||||
)
|
||||
|
||||
// judgeMachine is a machine for the judges: user@host, and the connection the
|
||||
// setup screen already made to it, when it made one.
|
||||
type judgeMachine struct {
|
||||
target string
|
||||
remote onboard.Remote
|
||||
}
|
||||
|
||||
// ensureJudgeNodes adds each judge machine as a node labelled for the
|
||||
// judges, and says how many cores the smallest of them has.
|
||||
func (in *serverInit) ensureJudgeNodes(ctx context.Context) error {
|
||||
if len(in.judgeMachines) == 0 {
|
||||
return nil
|
||||
}
|
||||
na := nodeAdd{serverCfg: in.configPath(), plain: true}
|
||||
na.opts.PKI = in.pkiDir
|
||||
na.opts.NoFirewall = in.noFirewall
|
||||
if err := na.prepare(); err != nil {
|
||||
return err
|
||||
}
|
||||
in.judgeCPUs = nil
|
||||
for _, jm := range in.judgeMachines {
|
||||
o := na.target(jm.target)
|
||||
fmt.Fprintf(in.out, " .. adding %s as a node for the judges\n", o.Addr)
|
||||
r := onboard.New(o, nil)
|
||||
if jm.remote != nil {
|
||||
r.Remote = jm.remote
|
||||
}
|
||||
res, err := r.Run(ctx)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %w", o.Addr, err)
|
||||
}
|
||||
if err := na.watch(ctx, res.Node, func(onboard.Event) {}); err != nil && err != errNoAdmin {
|
||||
return fmt.Errorf("%s: %w", res.Node, err)
|
||||
}
|
||||
if out, err := r.Remote.Run(ctx, "nproc", nil); err == nil {
|
||||
if n, err := strconv.Atoi(strings.TrimSpace(string(out))); err == nil {
|
||||
in.judgeCPUs = append(in.judgeCPUs, n)
|
||||
}
|
||||
}
|
||||
if _, err := changeStoredSettings(in.configPath(), "a node for the judges, from init", func(doc *settings.Doc) error {
|
||||
return doc.SetYAML([]string{"agents", "nodes", res.Node, "labels", judgeRoleLabel}, judgeRoleValue)
|
||||
}); err != nil {
|
||||
return fmt.Errorf("labelling %s for the judges: %w", res.Node, err)
|
||||
}
|
||||
in.done("%s is a node for the judges alone (%s: %s)", res.Node, judgeRoleLabel, judgeRoleValue)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// judgePlacement is the judge's cores and where it may run: on the judge
|
||||
// machines when there are some, by init's rule over their cores; otherwise
|
||||
// on any node, with this machine's plan.
|
||||
func (in *serverInit) judgePlacement() (cores int, selector map[string]string) {
|
||||
if len(in.judgeMachines) == 0 {
|
||||
return in.cpuPlan().judge, nil
|
||||
}
|
||||
return refitCores(in.judgeFor().Replicas, in.judgeCPUs), map[string]string{judgeRoleLabel: judgeRoleValue}
|
||||
}
|
||||
|
||||
// judgeMachineFlag is --judge-node, given again for each machine.
|
||||
type judgeMachineFlag struct{ to *[]judgeMachine }
|
||||
|
||||
func (f judgeMachineFlag) String() string {
|
||||
if f.to == nil {
|
||||
return ""
|
||||
}
|
||||
var out []string
|
||||
for _, m := range *f.to {
|
||||
out = append(out, m.target)
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
func (f judgeMachineFlag) Set(v string) error {
|
||||
if strings.TrimSpace(v) == "" {
|
||||
return fmt.Errorf("--judge-node wants user@host")
|
||||
}
|
||||
*f.to = append(*f.to, judgeMachine{target: strings.TrimSpace(v)})
|
||||
return nil
|
||||
}
|
||||
|
||||
func (judgeMachineFlag) Type() string { return "user@host" }
|
||||
@@ -0,0 +1,67 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestJudgesGoOnTheirOwnMachinesWhenTheInstallHasSome(t *testing.T) {
|
||||
in := newDefaultServerInit()
|
||||
if _, sel := in.judgePlacement(); sel != nil {
|
||||
t.Errorf("no judge machines, yet a selector: %v", sel)
|
||||
}
|
||||
if err := (judgeMachineFlag{&in.judgeMachines}).Set("ubuntu@10.0.0.21"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = (judgeMachineFlag{&in.judgeMachines}).Set("ubuntu@10.0.0.22")
|
||||
in.judge.Replicas, in.judgeCPUs = 2, []int{4, 4}
|
||||
cores, sel := in.judgePlacement()
|
||||
if sel[judgeRoleLabel] != judgeRoleValue || cores != refitCores(2, []int{4, 4}) {
|
||||
t.Errorf("cores %d, selector %v", cores, sel)
|
||||
}
|
||||
m := guardModel("http://x/g.gguf", 2, cores, sel)
|
||||
if m.Profiles[0].NodeSelector[judgeRoleLabel] != judgeRoleValue {
|
||||
t.Errorf("the judge's profile does not select its machines: %+v", m.Profiles[0])
|
||||
}
|
||||
// The same from a script.
|
||||
flags := strings.Join(in.initFlags(), " ")
|
||||
if !strings.Contains(flags, "--judge-node ubuntu@10.0.0.21") || !strings.Contains(flags, "--judge-node ubuntu@10.0.0.22") {
|
||||
t.Errorf("flags %s", flags)
|
||||
}
|
||||
if err := (judgeMachineFlag{&in.judgeMachines}).Set(" "); err == nil {
|
||||
t.Error("an empty --judge-node was taken")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheSetupAsksForJudgeMachinesWhenChosen(t *testing.T) {
|
||||
m := newSetupModel("ctl", []string{"10.0.0.5"})
|
||||
m.choice, m.page = 1, pMachines
|
||||
m.mfocus = m.shown() * mfCount // the judges' choice
|
||||
m = press(m, "right").(setupModel)
|
||||
if m.judges != 1 || m.shown() != 3 || m.mfocus != 3*mfCount {
|
||||
t.Fatalf("judges %d, shown %d, focus %d", m.judges, m.shown(), m.mfocus)
|
||||
}
|
||||
if v := m.machinesView(); !strings.Contains(v, "Judge machine 1") || !strings.Contains(v, "Check the 4 machines") {
|
||||
t.Errorf("the page lacks the judge machine:\n%s", v)
|
||||
}
|
||||
for i, addr := range []string{"10.0.0.11", "10.0.0.12", ""} {
|
||||
m.db[i].addr.SetValue(addr)
|
||||
m.db[i].user.SetValue("ubuntu")
|
||||
m.db[i].auth = authAgent
|
||||
}
|
||||
if err := m.validateMachines(); !strings.Contains(err, "judge machine 1 has no address") {
|
||||
t.Errorf("err %q", err)
|
||||
}
|
||||
m.db[2].addr.SetValue("10.0.0.21")
|
||||
if err := m.validateMachines(); err != "" {
|
||||
t.Errorf("err %q", err)
|
||||
}
|
||||
if cmd := m.command(); !strings.Contains(cmd, "--judge-node ubuntu@10.0.0.21") || strings.Contains(cmd, "--patroni ubuntu@10.0.0.21") {
|
||||
t.Errorf("command %s", cmd)
|
||||
}
|
||||
// None again: the judge machine is no longer asked for.
|
||||
m = press(m, "left").(setupModel)
|
||||
if m.judges != 0 || strings.Contains(m.machinesView(), "Judge machine 1") {
|
||||
t.Errorf("judges %d after choosing none", m.judges)
|
||||
}
|
||||
}
|
||||
+126
-36
@@ -89,14 +89,21 @@ func (d dbMachine) visible(f int) bool {
|
||||
return true
|
||||
}
|
||||
|
||||
// dbConn is a database machine as it is reached, whichever way.
|
||||
// dbConn is a database machine as it is reached, whichever way, or a judge
|
||||
// machine when judge is set.
|
||||
type dbConn struct {
|
||||
remote onboard.Remote
|
||||
addr string
|
||||
ssh *onboard.GoSSH
|
||||
joined *joinpkg.Remote
|
||||
judge bool
|
||||
target string
|
||||
}
|
||||
|
||||
// The machines the page asks for: the store's two, then up to maxJudges for
|
||||
// the guardrails' judges alone.
|
||||
const maxJudges = 2
|
||||
|
||||
func newDBMachine(n int) dbMachine {
|
||||
f := func(ph string, mask bool) textField {
|
||||
t := newTextField()
|
||||
@@ -119,6 +126,24 @@ func newDBMachine(n int) dbMachine {
|
||||
return m
|
||||
}
|
||||
|
||||
// newJudgeMachine is the fields of a machine for the judges.
|
||||
func newJudgeMachine(n int) dbMachine {
|
||||
d := newDBMachine(n + 2)
|
||||
d.addr.Placeholder = fmt.Sprintf("10.0.0.%d or judge%d.internal", 20+n, n)
|
||||
return d
|
||||
}
|
||||
|
||||
// shown is how many machines the page asks for.
|
||||
func (m setupModel) shown() int { return 2 + m.judges }
|
||||
|
||||
// machineTitle names machine i: the store's, or the judges'.
|
||||
func machineTitle(i int) string {
|
||||
if i < 2 {
|
||||
return fmt.Sprintf("Database machine %d", i+1)
|
||||
}
|
||||
return fmt.Sprintf("Judge machine %d", i-1)
|
||||
}
|
||||
|
||||
// field is the text field at a machine's position, nil for the selector.
|
||||
func (d *dbMachine) field(i int) *textField {
|
||||
switch i {
|
||||
@@ -156,8 +181,10 @@ type setupModel struct {
|
||||
|
||||
choices *installForm
|
||||
|
||||
db [2]dbMachine
|
||||
mfocus int // over both machines' fields, then the check button
|
||||
db [2 + maxJudges]dbMachine
|
||||
// judges is how many judge machines are asked for, none to maxJudges.
|
||||
judges int
|
||||
mfocus int // over the machines' fields, then the judges' choice, then the check button
|
||||
|
||||
conns []dbConn
|
||||
listener *joinpkg.Listener
|
||||
@@ -192,7 +219,8 @@ func newSetupModel(host string, ips []string) setupModel {
|
||||
in.judge.Replicas = 2
|
||||
return setupModel{
|
||||
ref: &progRef{}, host: host, ips: ips, in: &in, choices: newInstallForm(&in, host, ips, true),
|
||||
db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,
|
||||
db: [2 + maxJudges]dbMachine{newDBMachine(1), newDBMachine(2), newJudgeMachine(1), newJudgeMachine(2)},
|
||||
spin: newSpinner(), width: 100, height: 30,
|
||||
log: &[]string{}, logMu: &sync.Mutex{},
|
||||
}
|
||||
}
|
||||
@@ -327,7 +355,7 @@ func (m *setupModel) focusMachine() {
|
||||
}
|
||||
}
|
||||
}
|
||||
if m.mfocus < 2*mfCount {
|
||||
if m.mfocus < m.shown()*mfCount {
|
||||
if t := m.db[m.mfocus/mfCount].field(m.mfocus % mfCount); t != nil {
|
||||
t.Focus()
|
||||
}
|
||||
@@ -335,7 +363,8 @@ func (m *setupModel) focusMachine() {
|
||||
}
|
||||
|
||||
func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
n := 2*mfCount + 1
|
||||
sel := m.shown() * mfCount
|
||||
n := sel + 2
|
||||
switch k.Type {
|
||||
case tea.KeyTab, tea.KeyDown:
|
||||
m.step(1, n)
|
||||
@@ -344,7 +373,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
m.step(-1, n)
|
||||
return m, nil
|
||||
case tea.KeyEnter:
|
||||
if m.mfocus < 2*mfCount {
|
||||
if m.mfocus <= sel {
|
||||
m.step(1, n)
|
||||
return m, nil
|
||||
}
|
||||
@@ -361,7 +390,17 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
cmd := m.startCheck()
|
||||
return m, cmd
|
||||
}
|
||||
if m.mfocus == 2*mfCount {
|
||||
if m.mfocus == sel {
|
||||
switch k.String() {
|
||||
case "left", "h":
|
||||
m.judges = (m.judges + maxJudges) % (maxJudges + 1)
|
||||
case "right", "l", " ":
|
||||
m.judges = (m.judges + 1) % (maxJudges + 1)
|
||||
}
|
||||
m.mfocus = m.shown() * mfCount
|
||||
return m, nil
|
||||
}
|
||||
if m.mfocus > sel {
|
||||
return m, nil
|
||||
}
|
||||
d := &m.db[m.mfocus/mfCount]
|
||||
@@ -391,7 +430,7 @@ func (m setupModel) machinesKey(k tea.KeyMsg) (tea.Model, tea.Cmd) {
|
||||
func (m *setupModel) step(d, n int) {
|
||||
for range n {
|
||||
m.mfocus = (m.mfocus + d + n) % n
|
||||
if m.mfocus == 2*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
|
||||
if m.mfocus >= m.shown()*mfCount || m.db[m.mfocus/mfCount].visible(m.mfocus%mfCount) {
|
||||
break
|
||||
}
|
||||
}
|
||||
@@ -399,7 +438,7 @@ func (m *setupModel) step(d, n int) {
|
||||
}
|
||||
|
||||
func (m setupModel) allJoined() bool {
|
||||
for _, d := range m.db {
|
||||
for _, d := range m.db[:m.shown()] {
|
||||
if d.auth == authJoin && d.joined == nil {
|
||||
return false
|
||||
}
|
||||
@@ -411,7 +450,7 @@ func (m setupModel) allJoined() bool {
|
||||
// codes, and waits for them; it says whether there is anything to wait for.
|
||||
func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||||
var cmds []tea.Cmd
|
||||
for i := range m.db {
|
||||
for i := range m.db[:m.shown()] {
|
||||
d := &m.db[i]
|
||||
if d.auth != authJoin || d.joined != nil {
|
||||
continue
|
||||
@@ -429,7 +468,11 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||||
m.listener = l
|
||||
}
|
||||
if d.arrive == nil {
|
||||
c, ch := m.listener.Code("database")
|
||||
role := "database"
|
||||
if i >= 2 {
|
||||
role = "judge"
|
||||
}
|
||||
c, ch := m.listener.Code(role)
|
||||
d.code, d.arrive = c.String(), ch
|
||||
}
|
||||
ch, i := d.arrive, i
|
||||
@@ -440,20 +483,20 @@ func (m *setupModel) startJoins() (tea.Cmd, bool) {
|
||||
|
||||
func (m setupModel) validateMachines() string {
|
||||
seen := map[string]bool{m.thisAddr(): true, m.host: true}
|
||||
for i, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
if d.auth == authJoin {
|
||||
continue
|
||||
}
|
||||
addr := strings.TrimSpace(d.addr.Value())
|
||||
addr, title := strings.TrimSpace(d.addr.Value()), strings.ToLower(machineTitle(i)[:1])+machineTitle(i)[1:]
|
||||
switch {
|
||||
case addr == "":
|
||||
return fmt.Sprintf("database machine %d has no address", i+1)
|
||||
return title + " has no address"
|
||||
case seen[addr]:
|
||||
return fmt.Sprintf("%s is given twice, or is this machine: the three must be different machines", addr)
|
||||
return fmt.Sprintf("%s is given twice, or is this machine: each must be a different machine", addr)
|
||||
case strings.TrimSpace(d.user.Value()) == "":
|
||||
return fmt.Sprintf("database machine %d has no user to log in as", i+1)
|
||||
return title + " has no user to log in as"
|
||||
case d.auth != authAgent && d.secret.Value() == "":
|
||||
return fmt.Sprintf("database machine %d needs its %s", i+1, authNames[d.auth])
|
||||
return fmt.Sprintf("%s needs its %s", title, authNames[d.auth])
|
||||
}
|
||||
seen[addr] = true
|
||||
}
|
||||
@@ -473,11 +516,16 @@ func (m *setupModel) startCheck() tea.Cmd {
|
||||
Name: m.host + " (this machine)", Addr: m.thisAddr(), Local: true, Remote: pgha.Local{},
|
||||
Roles: []preflight.Role{preflight.RoleControl, preflight.RoleWitness},
|
||||
}}
|
||||
for _, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
roles, judge := m.dbRoles(), i >= 2
|
||||
if judge {
|
||||
roles = []preflight.Role{preflight.RoleNode}
|
||||
}
|
||||
if d.auth == authJoin {
|
||||
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined})
|
||||
m.conns = append(m.conns, dbConn{remote: d.joined, addr: d.joined.Addr, joined: d.joined,
|
||||
judge: judge, target: "root@" + d.joined.Addr})
|
||||
machines = append(machines, preflight.Machine{Name: d.joined.Hostname + " (" + d.joined.Addr + ", joined)",
|
||||
Addr: d.joined.Addr, Roles: m.dbRoles(), Remote: d.joined})
|
||||
Addr: d.joined.Addr, Roles: roles, Remote: d.joined})
|
||||
continue
|
||||
}
|
||||
g := &onboard.GoSSH{
|
||||
@@ -490,9 +538,8 @@ func (m *setupModel) startCheck() tea.Cmd {
|
||||
case authPassword:
|
||||
g.Password = d.secret.Value()
|
||||
}
|
||||
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g})
|
||||
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr,
|
||||
Roles: m.dbRoles(), Remote: g})
|
||||
m.conns = append(m.conns, dbConn{remote: g, addr: g.Addr, ssh: g, judge: judge, target: g.User + "@" + g.Addr})
|
||||
machines = append(machines, preflight.Machine{Name: g.Addr, Addr: g.Addr, Roles: roles, Remote: g})
|
||||
}
|
||||
m.checking = true
|
||||
m.report = preflight.Report{}
|
||||
@@ -535,7 +582,7 @@ func (m setupModel) dbRoles() []preflight.Role {
|
||||
// targets are the database machines as user@address, for the command line.
|
||||
func (m setupModel) targets() []string {
|
||||
var out []string
|
||||
for _, d := range m.db {
|
||||
for _, d := range m.db[:2] {
|
||||
if d.joined != nil {
|
||||
out = append(out, "root@"+d.joined.Addr)
|
||||
continue
|
||||
@@ -545,11 +592,25 @@ func (m setupModel) targets() []string {
|
||||
return out
|
||||
}
|
||||
|
||||
// judgeTargets are the judge machines as user@address.
|
||||
func (m setupModel) judgeTargets() []judgeMachine {
|
||||
var out []judgeMachine
|
||||
for _, d := range m.db[2:m.shown()] {
|
||||
if d.joined != nil {
|
||||
out = append(out, judgeMachine{target: "root@" + d.joined.Addr})
|
||||
continue
|
||||
}
|
||||
out = append(out, judgeMachine{target: strings.TrimSpace(d.user.Value()) + "@" + strings.TrimSpace(d.addr.Value())})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// command is what this setup is on the command line, for doing it again
|
||||
// from a script — with keys in place, since a password cannot be given there.
|
||||
func (m setupModel) command() string {
|
||||
in := *m.in
|
||||
in.patroni = m.targets()
|
||||
in.judgeMachines = m.judgeTargets()
|
||||
return strings.Join(append([]string{"sudo --preserve-env=SSH_AUTH_SOCK llmgw server init"}, in.initFlags()...), " \\\n ")
|
||||
}
|
||||
|
||||
@@ -558,6 +619,7 @@ func (m setupModel) command() string {
|
||||
func (m *setupModel) startRun() tea.Cmd {
|
||||
in := m.in
|
||||
in.patroni = m.targets()
|
||||
in.judgeMachines = nil
|
||||
m.run = newInitModel(in, nil, false)
|
||||
m.run.running, m.run.width, m.run.spin = true, m.width, m.spin
|
||||
ref, conns, logp, mu, listener := m.ref, m.conns, m.log, m.logMu, m.listener
|
||||
@@ -578,6 +640,11 @@ func (m *setupModel) startRun() tea.Cmd {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if c.judge {
|
||||
// Added as a node once the control plane is up.
|
||||
in.judgeMachines = append(in.judgeMachines, judgeMachine{target: c.target, remote: c.remote})
|
||||
continue
|
||||
}
|
||||
out, err := c.remote.Run(ctx, "hostname", nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: %v", c.addr, err)
|
||||
@@ -720,8 +787,12 @@ func (m setupModel) machinesView() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" +
|
||||
" PostgreSQL, Patroni and etcd are installed on them.\n\n")
|
||||
for i, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
base := i * mfCount
|
||||
if i == 2 {
|
||||
b.WriteString(" " + sDim.Render("Machines for the guardrails' judges alone: each becomes a node that runs only\n"+
|
||||
" the judges, so a busy control plane cannot leave a request waiting for a judgment.") + "\n\n")
|
||||
}
|
||||
// Padded before it is coloured: the colour's escape codes count
|
||||
// toward a width and threw the focused row out of line.
|
||||
label := func(f int, s string) string {
|
||||
@@ -731,7 +802,7 @@ func (m setupModel) machinesView() string {
|
||||
}
|
||||
return " " + s
|
||||
}
|
||||
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
|
||||
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(machineTitle(i)))
|
||||
if d.visible(mfAddr) {
|
||||
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View())
|
||||
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View())
|
||||
@@ -756,7 +827,22 @@ func (m setupModel) machinesView() string {
|
||||
}
|
||||
b.WriteString("\n")
|
||||
}
|
||||
b.WriteString(button(m.mfocus == 2*mfCount, "Check the three machines") + "\n\n")
|
||||
sel := m.shown() * mfCount
|
||||
var opts []string
|
||||
for n, name := range []string{"none", "one", "two"} {
|
||||
if n == m.judges {
|
||||
opts = append(opts, "("+sOK.Render("•")+") "+name)
|
||||
} else {
|
||||
opts = append(opts, "( ) "+sDim.Render(name))
|
||||
}
|
||||
}
|
||||
judgeLabel := fmt.Sprintf(" %-15s", "Judge machines")
|
||||
if m.mfocus == sel {
|
||||
judgeLabel = fSelected.Render(fmt.Sprintf("› %-15s", "Judge machines"))
|
||||
}
|
||||
fmt.Fprintf(&b, " %s %s\n", judgeLabel, strings.Join(opts, " "))
|
||||
b.WriteString(" " + sDim.Render("Without them the judges run on whichever node has room, added later.") + "\n\n")
|
||||
b.WriteString(button(m.mfocus == sel+1, fmt.Sprintf("Check the %d machines", m.shown()+1)) + "\n\n")
|
||||
b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n")
|
||||
return b.String()
|
||||
}
|
||||
@@ -764,11 +850,11 @@ func (m setupModel) machinesView() string {
|
||||
func (m setupModel) waitView() string {
|
||||
var b strings.Builder
|
||||
fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort)
|
||||
for i, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
if d.auth != authJoin {
|
||||
continue
|
||||
}
|
||||
title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))
|
||||
title := lipgloss.NewStyle().Bold(true).Render(machineTitle(i))
|
||||
if d.joined != nil {
|
||||
fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
|
||||
continue
|
||||
@@ -787,7 +873,7 @@ func (m setupModel) checkView() string {
|
||||
var b strings.Builder
|
||||
b.WriteString("\n")
|
||||
if m.checking {
|
||||
b.WriteString(" " + m.spin.View() + " checking the three machines…\n")
|
||||
b.WriteString(" " + m.spin.View() + " checking the machines…\n")
|
||||
return b.String()
|
||||
}
|
||||
switch m.report.Worst() {
|
||||
@@ -796,7 +882,7 @@ func (m setupModel) checkView() string {
|
||||
case preflight.Warn:
|
||||
b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n")
|
||||
default:
|
||||
b.WriteString(" " + sOK.Render("✓ All three are ready.") + "\n")
|
||||
b.WriteString(" " + sOK.Render("✓ All are ready.") + "\n")
|
||||
}
|
||||
machine := ""
|
||||
for _, f := range m.report.Findings {
|
||||
@@ -822,7 +908,7 @@ func (m setupModel) checkView() string {
|
||||
func (m setupModel) planView() string {
|
||||
var b strings.Builder
|
||||
b.WriteString(m.choices.planView(m.width - navWidth - 6))
|
||||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The three machines") + "\n")
|
||||
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render("The machines") + "\n")
|
||||
here := "control plane, leading at first; model mirror; etcd"
|
||||
there := "PostgreSQL under Patroni, etcd"
|
||||
if m.in.standbys {
|
||||
@@ -834,12 +920,16 @@ func (m setupModel) planView() string {
|
||||
}
|
||||
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()), here)
|
||||
granted := false
|
||||
for _, d := range m.db {
|
||||
for i, d := range m.db[:m.shown()] {
|
||||
addr := strings.TrimSpace(d.addr.Value())
|
||||
if d.joined != nil {
|
||||
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
|
||||
}
|
||||
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), there)
|
||||
role := there
|
||||
if i >= 2 {
|
||||
role = "a node for the guardrails' judges alone"
|
||||
}
|
||||
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(addr), role)
|
||||
granted = granted || d.sudo.Value() != ""
|
||||
}
|
||||
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +
|
||||
@@ -851,7 +941,7 @@ func (m setupModel) planView() string {
|
||||
for _, l := range strings.Split(m.command(), "\n") {
|
||||
b.WriteString(" " + l + "\n")
|
||||
}
|
||||
for _, d := range m.db {
|
||||
for _, d := range m.db[:m.shown()] {
|
||||
if d.joined != nil {
|
||||
b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n")
|
||||
break
|
||||
|
||||
Reference in new issue
Block a user