Add a standby control plane with server add, and draw the screens as windows

server add gives another machine this llmgw, a certificate, the
configuration and store address, lets it into the store through Patroni, and
starts it; --vip keeps a floating address on the leader with keepalived,
following a new /leader endpoint. Nodes learn every control plane from the
leader and notice a dead one by HTTP/2 ping instead of after the kernel's
fifteen minutes. The setup, console, node and init screens are drawn full
screen in one frame: title bar, sidebar, bordered panel, key bar, dialogs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 22:29:12 +02:00
1 parent f5c79052b1
commit aa3efcebbe
31 files changed
+1856 -394

No files matched your search

+39 -7
View File
@@ -73,8 +73,11 @@ type Agent struct {
lastEngineProbe time.Time
probingEngines bool
mu sync.Mutex
inv discovery.Inventory
mu sync.Mutex
// serverList is every control plane to dial: the configured ones, and
// any the leader has named since. Read through servers().
serverList []string
inv discovery.Inventory
// shardReplica remembers which replica a shard belongs to, so a status
// report can name both. The control plane keys on the replica.
shardReplica map[string]string
@@ -153,6 +156,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
load: discovery.NewLoadSampler(),
sensors: discovery.NewSensorSampler(),
wear: discovery.NewHealthSampler(),
serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
}
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
return a, nil
@@ -185,7 +189,9 @@ func (a *Agent) Run(ctx context.Context) error {
attempt := 0
next := 0
for ctx.Err() == nil {
server := a.cfg.Servers[next%len(a.cfg.Servers)]
// Read each time round: the leader may have named another since.
servers := a.servers()
server := servers[next%len(servers)]
err := a.session(ctx, server)
if ctx.Err() != nil {
break
@@ -200,12 +206,12 @@ func (a *Agent) Run(ctx context.Context) error {
// No wait when there is somewhere else to try: the delay exists to
// stop hammering one control plane, and the next address has not been
// hammered. With a single address this is the old behaviour exactly.
if len(a.cfg.Servers) > 1 && next%len(a.cfg.Servers) != 0 {
if len(servers) > 1 && next%len(servers) != 0 {
delay = 0
}
a.log.Warn("disconnected from the control plane, retrying",
"err", err, "server", server,
"next", a.cfg.Servers[next%len(a.cfg.Servers)],
"next", servers[next%len(servers)],
"in", delay.Round(time.Millisecond), "attempt", attempt)
select {
@@ -792,6 +798,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
a.setProvision(m.Provision)
case *llmgwv1.ControlMessage_Upgrade:
go a.upgrade(ctx, m.Upgrade, out)
case *llmgwv1.ControlMessage_ControlPlanes:
a.setControlPlanes(m.ControlPlanes)
case *llmgwv1.ControlMessage_Logs:
reply := a.logsReply(m.Logs)
// Off the receive loop: a full send queue must not stop commands being
@@ -1084,7 +1092,9 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
if !cfg.Enabled() {
return &http.Client{
Transport: &http2.Transport{
AllowHTTP: true,
AllowHTTP: true,
ReadIdleTimeout: streamIdle,
PingTimeout: streamPing,
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
},
@@ -1099,9 +1109,31 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
if err != nil {
return nil, err
}
return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
return &http.Client{Transport: &http2.Transport{
TLSClientConfig: tc,
ReadIdleTimeout: streamIdle,
PingTimeout: streamPing,
// Bounded, as the plaintext branch's is: dialling a control plane
// whose machine is off waits out the kernel's connect retries,
// about two minutes, before the next address is tried.
DialTLSContext: func(ctx context.Context, network, addr string, c *tls.Config) (net.Conn, error) {
d := tls.Dialer{NetDialer: &net.Dialer{Timeout: timeout}, Config: c}
return d.DialContext(ctx, network, addr)
},
}}, nil
}
// A stream to a control plane whose machine stopped dead — powered off, cut
// off — is never closed: nothing sends the FIN, and the agent's heartbeats sit
// in the socket's buffer until the kernel gives up retransmitting, fifteen
// minutes later. Found by switching off a leader: the standby took over in 14
// seconds and the node stayed with the dead one. An HTTP/2 ping after ten
// quiet seconds, given five to answer, finds it in a quarter of a minute.
const (
streamIdle = 10 * time.Second
streamPing = 5 * time.Second
)
// localizeSpec fills in what only this node knows.
//
// The interface NCCL bootstraps over is this node's, not the cluster's: the
+77
View File
@@ -0,0 +1,77 @@
package agent
import (
"encoding/json"
"os"
"path/filepath"
"slices"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
)
// controlPlanesFile keeps the control planes the leader last named, beside
// the agent's configuration rather than in it: the configuration is the
// operator's, and a list rewritten in it would be a file nobody wrote.
var controlPlanesFile = "/var/lib/llmgw-agent/control-planes.json"
// servers is every address to dial, in the order to try them: the
// configuration's, then any the leader named that it lacks.
func (a *Agent) servers() []string {
a.mu.Lock()
defer a.mu.Unlock()
if len(a.serverList) == 0 {
return a.cfg.Servers
}
return a.serverList
}
// mergeServers puts the leader's list after the configured one, without
// repeats, and says whether that changed anything.
func mergeServers(configured, told []string, secure bool) []string {
out := slices.Clone(configured)
for _, s := range told {
fixed, err := normalizeOne(s, secure)
if err != nil {
continue
}
if !slices.Contains(out, fixed) {
out = append(out, fixed)
}
}
return out
}
// setControlPlanes takes the leader's list: used from the next reconnect,
// and kept for the next start.
func (a *Agent) setControlPlanes(m *llmgwv1.ControlPlanes) {
told := m.GetServers()
merged := mergeServers(a.cfg.Servers, told, a.cfg.TLS.Enabled())
a.mu.Lock()
changed := !slices.Equal(merged, a.serverList)
a.serverList = merged
a.mu.Unlock()
if !changed {
return
}
a.log.Info("the control planes to dial", "servers", merged)
b, _ := json.Marshal(told)
if err := os.MkdirAll(filepath.Dir(controlPlanesFile), 0o755); err == nil {
tmp := controlPlanesFile + ".new"
if err := os.WriteFile(tmp, b, 0o644); err == nil {
_ = os.Rename(tmp, controlPlanesFile)
}
}
}
// loadControlPlanes is the list kept from the last time the leader named it.
func loadControlPlanes(configured []string, secure bool) []string {
b, err := os.ReadFile(controlPlanesFile)
if err != nil {
return configured
}
var told []string
if json.Unmarshal(b, &told) != nil {
return configured
}
return mergeServers(configured, told, secure)
}
+41
View File
@@ -0,0 +1,41 @@
package agent
import (
"log/slog"
"path/filepath"
"slices"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
func TestAControlPlaneTheLeaderNamesIsDialledAndRemembered(t *testing.T) {
old := controlPlanesFile
controlPlanesFile = filepath.Join(t.TempDir(), "control-planes.json")
t.Cleanup(func() { controlPlanesFile = old })
configured := []string{"https://10.0.0.140:7443"}
// TLS on, as every onboarded node has it: an agent without it refuses
// https:// addresses, and the leader's are https://.
a := &Agent{cfg: Config{Servers: configured, TLS: tlsconf.Config{CA: "/etc/llmgw/ca.pem"}}, log: slog.New(slog.DiscardHandler)}
if got := a.servers(); !slices.Equal(got, configured) {
t.Fatalf("before anything is named: %v", got)
}
// The configured one stays first; the new one is added after it, once.
a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}})
want := []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}
if got := a.servers(); !slices.Equal(got, want) {
t.Errorf("after the leader named two: %v", got)
}
// And after a restart, before the leader says anything.
if got := loadControlPlanes(configured, true); !slices.Equal(got, want) {
t.Errorf("after a restart: %v", got)
}
// A list that drops the configured one does not drop it: the
// configuration is the operator's.
a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.150:7443"}})
if got := a.servers(); got[0] != "https://10.0.0.140:7443" {
t.Errorf("the configured server was dropped: %v", got)
}
}
+19 -1
View File
@@ -451,7 +451,7 @@ func newServerCmd() *cobra.Command {
f.StringVar(&metricsListen, "metrics-listen", "", "serve metrics only on this address, safe to expose")
f.StringArrayVar(&endpoints, "endpoint", nil, "static endpoint: id=URL=model[,model] (repeatable)")
cmd.AddCommand(newServerInitCmd())
cmd.AddCommand(newServerInitCmd(), newServerAddCmd())
cmd.AddCommand(newBackupCmd())
return cmd
}
@@ -513,6 +513,9 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
if h, ok := sink.(interface{ Store() *storepkg.DB }); ok {
if st := h.Store(); st != nil {
lead = control.NewLeadership(st, cfg.Control.LeaseTTL, log)
if r, ok := sink.(interface{ SetLeader(func() bool) }); ok {
r.SetLeader(lead.IsLeader)
}
ctl.SetLeadership(lead)
// The tenancy tree's capacity quotas: what each part is
// guaranteed and may hold.
@@ -542,6 +545,21 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
if lead != nil {
svc.SetLeadership(lead)
}
// The control planes agents may dial, from the store every control plane
// shares: told to each agent as it registers, and to all of them when
// `server add` records another.
if h, ok := sink.(interface{ Store() *storepkg.DB }); ok && h.Store() != nil {
st := h.Store()
svc.SetControlPlanes(func(ctx context.Context) ([]string, error) {
cps, err := st.ControlPlanes(ctx)
urls := make([]string, len(cps))
for i, c := range cps {
urls[i] = c.URL
}
return urls, err
})
go svc.WatchControlPlanes(ctx, 30*time.Second)
}
fwd.set(svc)
// Every InfiniBand fabric mapped every five minutes, through one node.
go svc.ScanFabrics(ctx, 5*time.Minute)
+459
View File
@@ -0,0 +1,459 @@
package cli
import (
"bytes"
"context"
"crypto/rand"
"crypto/x509"
"encoding/hex"
"encoding/pem"
"errors"
"fmt"
"io"
"net"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
)
// serverAdd makes another machine a control plane of this cluster: a standby
// that takes over when this one is lost. Run on a control plane that holds
// the authority, as root, like `node add`.
type serverAdd struct {
etcDir, pkiDir string
// hosts are more names to put in the new control plane's certificate.
hosts []string
// vip is a floating address, "10.0.0.160/24", to keep on whichever of the
// two leads; "" leaves client traffic to whatever is in front.
vip string
remote onboard.Remote
// addr is the machine's address as given; ip what the others reach it at.
addr, ip, name string
out io.Writer
}
func newServerAddCmd() *cobra.Command {
sa := serverAdd{etcDir: "/etc/llmgw", pkiDir: defaultPKIDir}
cmd := &cobra.Command{
Use: "add [user@]address",
Short: "Make another machine a control plane of this cluster: a standby that takes over",
Long: "Run on a control plane, as root. It logs in to the machine with ssh, as\n" +
"node add does, and gives it this llmgw, a certificate from this\n" +
"cluster's authority, this control plane's configuration and the store's\n" +
"address, lets it into the store, and starts it. It stands by while this\n" +
"one leads, and takes over within seconds when this one is lost; every\n" +
"node is told of it and fails over to it without being touched.\n\n" +
"The store must be one both can reach — replicated with 'server init\n" +
"--patroni' — not a database on this machine.\n\n" +
"--vip keeps a floating address on whichever control plane leads, with\n" +
"keepalived on both: the address clients use for the gateway.\n\n" +
"Run it again at any time: it changes only what is missing.",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
if os.Geteuid() != 0 {
return errors.New("run it as root: it reads the certificate authority and the store's address")
}
user, host, ok := strings.Cut(args[0], "@")
if !ok {
user, host = "", args[0]
}
sa.remote = &onboard.SSH{Addr: host, User: user}
sa.addr = host
sa.out = cmd.OutOrStdout()
return sa.run(cmd.Context())
},
}
f := cmd.Flags()
f.StringArrayVar(&sa.hosts, "host", nil, "another name for the new control plane's certificate; repeatable")
f.StringVar(&sa.vip, "vip", "", "a floating address for whichever control plane leads, as 10.0.0.160/24")
f.StringVar(&sa.pkiDir, "pki", sa.pkiDir, "directory holding the certificate authority")
return cmd
}
func (sa *serverAdd) done(format string, a ...any) {
fmt.Fprintf(sa.out, " ok "+format+"\n", a...)
}
func (sa *serverAdd) kept(format string, a ...any) {
fmt.Fprintf(sa.out, " kept "+format+"\n", a...)
}
func (sa *serverAdd) sh(ctx context.Context, script string, stdin []byte) (string, error) {
out, err := sa.remote.Run(ctx, script, stdin)
if err != nil {
return string(out), fmt.Errorf("on %s: %v: %s", sa.addr, err, strings.TrimSpace(string(out)))
}
return string(out), nil
}
// put writes a file there, through standard input, with its mode and owner.
func (sa *serverAdd) put(ctx context.Context, path, mode, owner string, body []byte) error {
_, err := sa.sh(ctx, fmt.Sprintf(`set -e
sudo install -d -m 0755 %[1]s
sudo sh -c 'umask 077; cat > %[2]s.new'
sudo chmod %[3]s %[2]s.new
sudo chown %[4]s %[2]s.new
sudo mv -f %[2]s.new %[2]s`, filepath.Dir(path), path, mode, owner), body)
return err
}
func (sa *serverAdd) run(ctx context.Context) error {
steps := []struct {
name string
fn func(context.Context) error
}{
{"connect", sa.connect},
{"binary", sa.binary},
{"directories", sa.dirs},
{"certificate", sa.certificate},
{"configuration", sa.configuration},
{"store", sa.storeAccess},
{"service", sa.service},
}
if sa.vip != "" {
steps = append(steps, struct {
name string
fn func(context.Context) error
}{"floating address", sa.floating})
}
for _, s := range steps {
if err := s.fn(ctx); err != nil {
return fmt.Errorf("%s: %w", s.name, err)
}
}
fmt.Fprintf(sa.out, "\n%s is a control plane of this cluster, standing by while this one leads.\n", sa.name)
return nil
}
func (sa *serverAdd) connect(ctx context.Context) error {
out, err := sa.sh(ctx, "hostname; sudo -n true && echo sudo-ok", nil)
if err != nil {
return fmt.Errorf("%w — it needs ssh and sudo without a password", err)
}
lines := strings.Split(strings.TrimSpace(out), "\n")
sa.name = strings.TrimSpace(lines[0])
if sa.ip == "" {
if sa.ip, err = ipv4Of(sa.addr); err != nil {
return err
}
}
sa.done("%s (%s)", sa.name, sa.ip)
return nil
}
// binary gives it this very llmgw, so the two control planes never differ.
func (sa *serverAdd) binary(ctx context.Context) error {
have, _ := sa.sh(ctx, "llmgw --version 2>/dev/null || true", nil)
if strings.TrimSpace(have) == "llmgw version "+version.Version {
sa.kept("llmgw %s", version.Version)
return nil
}
exe, err := os.Executable()
if err != nil {
return err
}
b, err := os.ReadFile(exe)
if err != nil {
return err
}
if _, err := sa.sh(ctx, `set -e; t=$(mktemp); cat > "$t"; sudo install -m 0755 "$t" /usr/local/bin/llmgw; rm -f "$t"`, b); err != nil {
return err
}
sa.done("llmgw %s, copied from this machine", version.Version)
return nil
}
func (sa *serverAdd) dirs(ctx context.Context) error {
_, err := sa.sh(ctx, `set -e
id llmgw >/dev/null 2>&1 || sudo useradd --system --home /var/lib/llmgw --shell /usr/sbin/nologin llmgw
sudo install -d -m 0755 /etc/llmgw
sudo install -d -o llmgw -g llmgw -m 0700 /etc/llmgw/secrets
sudo install -d -o llmgw -g llmgw -m 0750 /var/lib/llmgw`, nil)
if err == nil {
sa.done("user llmgw, /etc/llmgw and /var/lib/llmgw")
}
return err
}
// certificate issues it a server certificate from this cluster's authority,
// for its name and addresses, unless it holds one that already names them.
func (sa *serverAdd) certificate(ctx context.Context) error {
ca, err := loadCA(sa.pkiDir)
if err != nil {
return fmt.Errorf("the authority's key is not in %s: run this where it is (%v)", sa.pkiDir, err)
}
ips, _ := sa.sh(ctx, "hostname -I", nil)
want := []string{sa.name, sa.ip}
for _, ip := range strings.Fields(ips) {
if net.ParseIP(ip).To4() != nil && !slices.Contains(want, ip) {
want = append(want, ip)
}
}
want = append(want, sa.hosts...)
if have, _ := sa.sh(ctx, "sudo cat /etc/llmgw/server.pem 2>/dev/null || true", nil); have != "" {
if blk, _ := pem.Decode([]byte(have)); blk != nil {
if cert, err := x509.ParseCertificate(blk.Bytes); err == nil && cert.CheckSignatureFrom(ca.Cert) == nil {
names := strings.Split(sanNames(cert), ", ")
missing := false
for _, w := range want {
missing = missing || !slices.Contains(names, w)
}
if !missing {
sa.kept("certificate for %s", strings.Join(names, ", "))
return sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM)
}
}
}
}
leaf, err := ca.IssueServer(sa.name, want)
if err != nil {
return err
}
if err := sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM); err != nil {
return err
}
if err := sa.put(ctx, "/etc/llmgw/server.pem", "0644", "root:root", leaf.CertPEM); err != nil {
return err
}
if err := sa.put(ctx, "/etc/llmgw/server-key.pem", "0640", "root:llmgw", leaf.KeyPEM); err != nil {
return err
}
sa.done("certificate for %s, from this cluster's authority", strings.Join(want, ", "))
return nil
}
// configuration copies this control plane's, and the secrets beside it: the
// two must agree on everything, the store above all.
func (sa *serverAdd) configuration(ctx context.Context) error {
if have, _ := sa.sh(ctx, "sudo test -f /etc/llmgw/server.yaml && echo yes || true", nil); strings.TrimSpace(have) == "yes" {
sa.kept("/etc/llmgw/server.yaml (never overwritten)")
} else {
b, err := os.ReadFile(filepath.Join(sa.etcDir, "server.yaml"))
if err != nil {
return err
}
if err := sa.put(ctx, "/etc/llmgw/server.yaml", "0644", "root:root", b); err != nil {
return err
}
sa.done("/etc/llmgw/server.yaml, as this control plane's")
}
secrets, err := os.ReadDir(filepath.Join(sa.etcDir, "secrets"))
if err != nil && !os.IsNotExist(err) {
return err
}
for _, e := range secrets {
if e.IsDir() || strings.Contains(e.Name(), ".pre-") || strings.HasSuffix(e.Name(), ".new") {
continue
}
path := "/etc/llmgw/secrets/" + e.Name()
mine, err := os.ReadFile(filepath.Join(sa.etcDir, "secrets", e.Name()))
if err != nil {
return err
}
theirs, _ := sa.sh(ctx, "sudo cat "+path+" 2>/dev/null || true", nil)
switch {
case theirs == string(mine):
sa.kept("%s", path)
case theirs != "":
// Two control planes on two stores are two clusters.
return fmt.Errorf("%s there differs from this one's; remove it there if that machine is to join this cluster", path)
default:
if err := sa.put(ctx, path, "0600", "llmgw:llmgw", mine); err != nil {
return err
}
sa.done("%s", path)
}
}
return nil
}
// storeAccess lets it into the store, and records both control planes for
// the nodes to be told.
func (sa *serverAdd) storeAccess(ctx context.Context) error {
cfg, err := config.Load(filepath.Join(sa.etcDir, "server.yaml"))
if err != nil {
return err
}
addr, err := store.ResolveAddress(cfg.Auth.Store)
if err != nil {
return err
}
addr = strings.TrimSpace(addr)
if !store.IsPostgres(addr) {
return errors.New("the store is SQLite on this machine; a second control plane needs one both reach — run 'llmgw server init --patroni' first")
}
hosts, _, _, err := store.SplitPostgres(addr)
if err != nil {
return err
}
for _, h := range hosts {
if ip := net.ParseIP(h); ip != nil && ip.IsLoopback() || h == "localhost" {
return fmt.Errorf("the store is at %s, on this machine only; replicate it first with 'llmgw server init --patroni'", h)
}
}
switch {
case slices.Contains(hosts, sa.ip):
sa.kept("store: %s is one of its members", sa.ip)
case len(hosts) > 1:
changed, err := pgha.AllowClient(ctx, hosts, sa.ip)
if err != nil {
return err
}
if changed {
sa.done("store: %s let in, through Patroni", sa.ip)
} else {
sa.kept("store: %s already let in", sa.ip)
}
default:
sa.kept("store at %s: make sure it lets %s in", hosts[0], sa.ip)
}
st, err := store.Open(ctx, addr)
if err != nil {
return err
}
defer st.Close()
_, port, _ := net.SplitHostPort(cfg.AgentListen)
if port == "" {
port = "7443"
}
self, err := sourceIPToward(sa.ip)
if err != nil {
return err
}
me, _ := os.Hostname()
for _, cp := range []store.ControlPlane{
{URL: "https://" + net.JoinHostPort(self, port), Name: me},
{URL: "https://" + net.JoinHostPort(sa.ip, port), Name: sa.name, AddedAt: time.Now().Add(time.Second)},
} {
if err := st.PutControlPlane(ctx, cp); err != nil {
return err
}
}
sa.done("nodes will be told of both control planes within half a minute")
return nil
}
// service installs the unit and starts it, then waits for it to answer.
func (sa *serverAdd) service(ctx context.Context) error {
if err := sa.put(ctx, "/etc/systemd/system/llmgw-server.service", "0644", "root:root", []byte(serverUnit)); err != nil {
return err
}
if _, err := sa.sh(ctx, "sudo systemctl daemon-reload && sudo systemctl enable llmgw-server >/dev/null 2>&1 && sudo systemctl restart llmgw-server", nil); err != nil {
return err
}
deadline := time.Now().Add(90 * time.Second)
for {
code, _ := sa.sh(ctx, "curl -s -o /dev/null -m 2 -w '%{http_code}' http://127.0.0.1:8080/leader || true", nil)
switch strings.TrimSpace(code) {
case "200":
sa.done("llmgw-server running, and leading")
return nil
case "503":
sa.done("llmgw-server running, standing by")
return nil
}
if time.Now().After(deadline) {
logs, _ := sa.sh(ctx, "sudo journalctl -u llmgw-server -n 15 --no-pager", nil)
return fmt.Errorf("it did not answer within 90s:\n%s", logs)
}
time.Sleep(2 * time.Second)
}
}
// floating keeps an address on whichever of the two leads, with keepalived
// on both, following /leader.
func (sa *serverAdd) floating(ctx context.Context) error {
vip, nw, err := net.ParseCIDR(sa.vip)
if err != nil || vip.To4() == nil {
return fmt.Errorf("--vip %q is not an IPv4 address with its prefix, like 10.0.0.160/24", sa.vip)
}
prefix, _ := nw.Mask.Size()
pass, err := vrrpPassword(sa.etcDir)
if err != nil {
return err
}
here := pgha.Local{}
for _, m := range []struct {
r onboard.Remote
name string
priority int
}{{here, "this machine", 110}, {sa.remote, sa.name, 100}} {
out, err := m.r.Run(ctx, fmt.Sprintf(`set -e
command -v keepalived >/dev/null 2>&1 || { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null; sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y keepalived >/dev/null; }
ip -o route get %s | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'`, vip), nil)
if err != nil {
return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out)))
}
iface := strings.TrimSpace(string(out))
conf := keepalivedConf(iface, vip.String(), prefix, m.priority, pass)
if out, err := m.r.Run(ctx, `set -e
sudo install -d -m 0755 /etc/keepalived
sudo sh -c 'umask 077; cat > /etc/keepalived/keepalived.conf'
sudo systemctl enable keepalived >/dev/null 2>&1
sudo systemctl restart keepalived`, []byte(conf)); err != nil {
return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out)))
}
}
sa.done("%s follows whichever control plane leads (keepalived on both)", sa.vip)
return nil
}
// vrrpPassword is the shared secret of the two keepalived, made once and
// kept with the other secrets, so running this again does not split them.
func vrrpPassword(etcDir string) (string, error) {
p := filepath.Join(etcDir, "secrets", "vrrp")
if b, err := os.ReadFile(p); err == nil && len(bytes.TrimSpace(b)) > 0 {
return string(bytes.TrimSpace(b)), nil
}
b := make([]byte, 4)
if _, err := rand.Read(b); err != nil {
return "", err
}
pass := hex.EncodeToString(b) // keepalived reads eight characters
if err := os.WriteFile(p, []byte(pass+"\n"), 0o600); err != nil {
return "", err
}
return pass, nil
}
func keepalivedConf(iface, vip string, prefix, priority int, pass string) string {
return fmt.Sprintf(`# llmgw: the gateway's floating address, on whichever control plane leads.
vrrp_script llmgw_leader {
script "/usr/bin/curl -fsS -o /dev/null -m 2 http://127.0.0.1:8080/leader"
interval 2
fall 2
rise 2
}
vrrp_instance llmgw {
state BACKUP
interface %s
virtual_router_id 51
priority %d
advert_int 1
authentication {
auth_type PASS
auth_pass %s
}
virtual_ipaddress {
%s/%d
}
track_script {
llmgw_leader
}
}
`, iface, priority, pass, vip, prefix)
}
+184
View File
@@ -0,0 +1,184 @@
package cli
import (
"strings"
"github.com/charmbracelet/lipgloss"
"github.com/charmbracelet/x/ansi"
)
// The frame every full-screen llmgw screen is drawn in: a title bar, a
// sidebar of steps or actions on the left, the content in a bordered panel on
// the right, the keys along the bottom — and a dialog over it all when one is
// open. One frame, so the setup and the console read as one program.
//
// Keyboard only: these run on serial consoles and IPMI as often as in a
// desktop terminal.
// navItem is one line of the sidebar.
type navItem struct {
Label string
// State is "done", "current", "todo" for steps; "" for menu entries,
// where Current marks the one focused.
State string
Current bool
// Dim greys an entry that cannot be chosen now.
Dim bool
}
type frame struct {
Width, Height int
Title string // left of the title bar
Where string // right of the title bar: the machine
NavTitle string
Nav []navItem
PanelTitle string
Body string
// Scroll is how many of the body's lines are scrolled off the top.
Scroll int
Keys string
// Dialog, when set, is drawn over the middle of the screen.
Dialog string
DialogTitle string
}
var (
fBar = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.AdaptiveColor{Light: "#FFFFFF", Dark: "#0B1B1B"}).Background(tuiAccent)
fBorder = lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(tuiDim)
fFocus = lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(tuiAccent)
fKeys = lipgloss.NewStyle().Foreground(tuiDim)
fSelected = lipgloss.NewStyle().Bold(true).Foreground(tuiAccent)
fDialog = lipgloss.NewStyle().Border(lipgloss.DoubleBorder()).BorderForeground(tuiAccent).Padding(1, 2)
)
const navWidth = 24
// render draws the whole screen, exactly Width by Height.
func (f frame) render() string {
w, h := max(f.Width, 60), max(f.Height, 16)
title := " " + f.Title
right := f.Where + " "
gap := max(1, w-ansi.StringWidth(title)-ansi.StringWidth(right))
bar := fBar.Render(title + strings.Repeat(" ", gap) + right)
// Panels fill what the bars leave: one line each above and below.
ph := h - 2
inner := ph - 2 // borders
var nav strings.Builder
for _, it := range f.Nav {
mark := " "
label := it.Label
switch {
case it.State == "done":
mark = sOK.Render("✓ ")
case it.State == "current":
mark = fSelected.Render("› ")
label = fSelected.Render(label)
case it.Current:
mark = fSelected.Render("› ")
label = fSelected.Render(label)
case it.Dim || it.State == "todo":
label = sDim.Render(label)
}
nav.WriteString(truncateWidth(mark+label, navWidth-2) + "\n")
}
left := box(fBorder, f.NavTitle, nav.String(), navWidth, inner)
pw := w - navWidth - 2 - 2
lines := strings.Split(strings.TrimRight(f.Body, "\n"), "\n")
scroll := min(max(f.Scroll, 0), max(0, len(lines)-inner))
visible := lines[scroll:]
if len(visible) > inner {
visible = visible[:inner]
// Said in the last line, where it is seen.
visible[inner-1] = sDim.Render(" ↓ more — PgDn")
}
for i := range visible {
visible[i] = truncateWidth(visible[i], pw)
}
panelTitle := f.PanelTitle
if scroll > 0 {
panelTitle += " ↑"
}
main := box(fFocus, panelTitle, strings.Join(visible, "\n"), pw, inner)
keys := fKeys.Render(" " + truncateWidth(f.Keys, w-2))
screen := bar + "\n" + lipgloss.JoinHorizontal(lipgloss.Top, left, main) + "\n" + keys
if f.Dialog != "" {
dw := min(w-8, max(50, maxLineWidth(f.Dialog)+6))
d := fDialog.Width(dw).Render(f.Dialog)
if f.DialogTitle != "" {
d = titled(d, f.DialogTitle)
}
screen = overlay(screen, d, w, h)
}
return screen
}
// box draws content in a border of the given inner size, with a title set
// into the top edge.
func box(style lipgloss.Style, title, content string, width, height int) string {
b := style.Width(width).Height(height).Render(content)
if title == "" {
return b
}
return titled(b, title)
}
// titled sets " title " into a box's top border, after its corner.
func titled(b, title string) string {
lines := strings.Split(b, "\n")
if len(lines) == 0 {
return b
}
top := lines[0]
t := " " + title + " "
tw := ansi.StringWidth(t)
if ansi.StringWidth(top) < tw+4 {
return b
}
lines[0] = ansi.Truncate(top, 2, "") + sTitle.Render(t) + ansi.TruncateLeft(top, 2+tw, "")
return strings.Join(lines, "\n")
}
// overlay draws top over the middle of base, both w by h.
func overlay(base, top string, w, h int) string {
bl := strings.Split(base, "\n")
tl := strings.Split(top, "\n")
tw := maxLineWidth(top)
x := max(0, (w-tw)/2)
y := max(0, (h-len(tl))/2)
for i, l := range tl {
row := y + i
if row >= len(bl) {
break
}
under := bl[row]
left := ansi.Truncate(under, x, "")
if pad := x - ansi.StringWidth(left); pad > 0 {
left += strings.Repeat(" ", pad)
}
right := ansi.TruncateLeft(under, x+ansi.StringWidth(l), "")
bl[row] = left + l + right
}
return strings.Join(bl, "\n")
}
func maxLineWidth(s string) int {
n := 0
for _, l := range strings.Split(s, "\n") {
n = max(n, ansi.StringWidth(l))
}
return n
}
// truncateWidth cuts a line, colours and all, to w cells.
func truncateWidth(s string, w int) string {
if ansi.StringWidth(s) <= w {
return s
}
return ansi.Truncate(s, max(w-1, 0), "…")
}
+33 -24
View File
@@ -13,7 +13,6 @@ import (
"time"
tea "github.com/charmbracelet/bubbletea"
"github.com/charmbracelet/lipgloss"
"github.com/mattn/go-isatty"
"github.com/spf13/cobra"
@@ -88,6 +87,7 @@ type consoleModel struct {
prompt textField
action consoleAction
width int
height int
spin spinnerModel
loaded bool
}
@@ -122,7 +122,7 @@ func (m consoleModel) Init() tea.Cmd { return tea.Batch(m.fetch, m.spin.Tick) }
func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.width = msg.Width
m.width, m.height = msg.Width, msg.Height
case spinTickMsg:
var cmd tea.Cmd
m.spin, cmd = m.spin.Update(msg)
@@ -183,28 +183,30 @@ func (m consoleModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
}
func (m consoleModel) View() string {
var b strings.Builder
host, _ := os.Hostname()
b.WriteString(sTitle.Render("llmgw · console") + " " + sDim.Render(host) + "\n\n")
var nav []navItem
for i, it := range m.items() {
nav = append(nav, navItem{Label: it.label, Current: i == m.focus})
}
var body string
switch {
case !m.loaded:
b.WriteString(" " + m.spin.View() + " asking the control plane…\n")
body = "\n " + m.spin.View() + " asking the control plane…"
case m.err != nil:
b.WriteString(sBad.Render("The control plane does not answer: ") + m.err.Error() + "\n" +
sDim.Render("systemctl status llmgw-server says why.") + "\n")
body = "\n " + sBad.Render("The control plane does not answer:") + "\n " + m.err.Error() +
"\n\n " + sDim.Render("systemctl status llmgw-server says why.")
default:
b.WriteString(m.summary)
body = "\n" + indent(m.summary, " ")
}
b.WriteString("\n" + lipgloss.NewStyle().Bold(true).Render("Do") + "\n")
for i, it := range m.items() {
line := " " + it.label
if i == m.focus {
line = sTitle.Render("› " + it.label)
}
b.WriteString(line + "\n")
if i == m.focus && it.command != "" {
b.WriteString(" " + sDim.Render(it.command) + "\n")
}
if it := m.items()[m.focus]; it.command != "" {
body += "\n\n " + sDim.Render("On the command line:") + "\n " + it.command
}
f := frame{
Width: m.width, Height: m.height,
Title: "llmgw console", Where: host,
NavTitle: "Do", Nav: nav,
PanelTitle: "Cluster", Body: body,
Keys: "↑↓ choose · Enter do it · Q leave · refreshed every five seconds",
}
if m.asking {
it := m.items()[m.focus]
@@ -212,11 +214,18 @@ func (m consoleModel) View() string {
if it.n > 1 {
ask = fmt.Sprintf("%s %d of %d", ask, len(m.asked)+1, it.n)
}
b.WriteString("\n" + ask + "\n " + m.prompt.View() + "\n" + sDim.Render("enter · esc to cancel") + "\n")
} else {
b.WriteString("\n" + sDim.Render("↑↓ to choose · enter · q to leave · refreshed every five seconds") + "\n")
f.DialogTitle = it.label
f.Dialog = ask + "\n\n" + m.prompt.View() + "\n\n" + sDim.Render("Enter · Esc cancel")
}
return b.String()
return f.render()
}
func indent(s, pre string) string {
lines := strings.Split(strings.TrimRight(s, "\n"), "\n")
for i := range lines {
lines[i] = pre + lines[i]
}
return strings.Join(lines, "\n")
}
// runConsole shows the console, runs what is chosen on it, and comes back.
@@ -229,8 +238,8 @@ func runConsole(ctx context.Context, out io.Writer) error {
}
}
for {
m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100}
final, err := tea.NewProgram(m, tea.WithOutput(out)).Run()
m := consoleModel{client: newCtlClient(addr), spin: newSpinner(), width: 100, height: 30}
final, err := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen()).Run()
if err != nil {
return err
}
+47 -7
View File
@@ -77,10 +77,14 @@ type initModel struct {
next []string // the closing "Next:" text
spin spinnerModel
width int
done bool
err error
quit bool
start func()
height int
// framed draws it in the full-screen frame; off when another screen
// shows it in its own panel.
framed bool
done bool
err error
quit bool
start func()
}
func newInitModel(in *serverInit, found []string, askForm bool) initModel {
@@ -118,7 +122,10 @@ type startInitMsg struct{}
func (m initModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.width = msg.Width
m.width, m.height = msg.Width, msg.Height
if m.framed {
m.width = msg.Width - navWidth - 6
}
case spinTickMsg:
var cmd tea.Cmd
m.spin, cmd = m.spin.Update(msg)
@@ -132,12 +139,18 @@ func (m initModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.addLine(string(msg))
case initDoneMsg:
m.done, m.err = true, msg.err
if m.framed {
return m, nil // left on screen until a key: it is gone once closed
}
return m, tea.Quit
case tea.KeyMsg:
if msg.Type == tea.KeyCtrlC {
m.quit = true
return m, tea.Quit
}
if m.done && m.framed && (msg.Type == tea.KeyEnter || msg.String() == "q") {
return m, tea.Quit
}
if m.form {
return m.formKey(msg)
}
@@ -240,6 +253,30 @@ func (m *initModel) moveFocus(d int) {
}
func (m initModel) View() string {
if !m.framed {
return m.plainView()
}
body := m.plainView()
if i := strings.Index(body, "\n\n"); i >= 0 {
body = body[i+1:]
}
steps := []navItem{{Label: "Start", State: "done"}, {Label: "Settings", State: "current"}, {Label: "Install", State: "todo"}}
keys := "Tab/↑↓ move · Space tick · Enter on Set up · Ctrl+C leave"
if !m.form {
steps[1].State, steps[2].State = "done", "current"
keys = "Ctrl+C stop"
if m.done {
steps[2].State = "done"
keys = "Enter close"
}
}
host, _ := os.Hostname()
return frame{Width: m.width + navWidth + 6, Height: m.height, Title: "llmgw setup", Where: host,
NavTitle: "Steps", Nav: steps, PanelTitle: "This machine as a control plane", Body: body, Keys: keys}.render()
}
// plainView is the screen as text, for a panel or a terminal that is not one.
func (m initModel) plainView() string {
var b strings.Builder
b.WriteString(sTitle.Render("llmgw · setting up this machine as a control plane") + "\n\n")
if m.form {
@@ -338,7 +375,9 @@ func (m initModel) formView() string {
if m.formError != "" {
b.WriteString("\n" + sBad.Render(m.formError) + "\n")
}
b.WriteString("\n" + sDim.Render("tab or ↑↓ to move · space to tick · enter on Set up to start · ctrl+c to leave") + "\n")
if !m.framed { // the frame's key bar says it
b.WriteString("\n" + sDim.Render("tab or ↑↓ to move · space to tick · enter on Set up to start · ctrl+c to leave") + "\n")
}
return b.String()
}
@@ -350,6 +389,7 @@ func (in *serverInit) runInitTUI(ctx context.Context, out io.Writer) error {
found = append(found, ip.String())
}
m := newInitModel(in, found, in.admin == "")
m.framed = true
var p *tea.Program
var (
@@ -373,7 +413,7 @@ func (in *serverInit) runInitTUI(ctx context.Context, out io.Writer) error {
p.Send(initDoneMsg{err})
}()
}
p = tea.NewProgram(m, tea.WithOutput(out))
p = tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen())
final, err := p.Run()
if err != nil {
return err
+54 -19
View File
@@ -68,6 +68,7 @@ type nodeModel struct {
spin spinnerModel
cancel context.CancelFunc
width int
height int
done bool
err error
node string
@@ -91,7 +92,7 @@ func (m nodeModel) Init() tea.Cmd { return m.spin.Tick }
func (m nodeModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.width = msg.Width
m.width, m.height = msg.Width, msg.Height
case spinTickMsg:
var cmd tea.Cmd
m.spin, cmd = m.spin.Update(msg)
@@ -131,9 +132,13 @@ func (m nodeModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
case askContinueMsg:
m.confirm = &msg
case finishMsg:
// Left on the screen until a key: the frame is gone once it closes.
m.done, m.err, m.node, m.summary = true, msg.err, msg.node, msg.summary
return m, tea.Quit
return m, nil
case tea.KeyMsg:
if m.done && (msg.Type == tea.KeyEnter || msg.String() == "q" || msg.Type == tea.KeyEsc) {
return m, tea.Quit
}
return m.key(msg)
}
return m, nil
@@ -197,10 +202,52 @@ func (m *nodeModel) move(d int) {
m.inputs[m.focus].Focus()
}
// View draws the onboarding in the frame: its steps down the side, what each
// did in the panel, a question as a dialog.
func (m nodeModel) View() string {
var nav []navItem
current := false
for _, step := range onboard.Steps {
st, ok := m.steps[step]
state := "todo"
switch {
case ok && st.level == onboard.Running && !m.done && !current:
state, current = "current", true
case ok && st.level != onboard.Running:
state = "done"
}
nav = append(nav, navItem{Label: step, State: state})
}
keys := "Ctrl+C stop"
if m.done {
keys = "Enter close"
}
host, _ := os.Hostname()
f := frame{
Width: m.width, Height: m.height, Title: "llmgw · add a node", Where: host,
NavTitle: "Steps", Nav: nav, PanelTitle: m.title, Body: "\n" + m.content(), Keys: keys,
}
switch {
case m.fabrics != nil:
var q strings.Builder
q.WriteString("These adapters are up. Which fabric is each group on?\n")
q.WriteString(sDim.Render("Give each the name it has on every other node attached to it.\nLeave a name empty to leave that group out.") + "\n\n")
for i, fb := range m.fabrics.suggested {
fmt.Fprintf(&q, " %s %s\n", m.inputs[i].View(), sDim.Render(strings.Join(fb.HCAs, ", ")))
}
q.WriteString("\n" + sDim.Render("Tab next · Enter use these · Esc none"))
f.DialogTitle, f.Dialog = "Fabrics", q.String()
case m.confirm != nil:
f.DialogTitle = "Problems found"
f.Dialog = fmt.Sprintf("doctor found %d problem(s) on the machine.\nAdd the node anyway?\n\n%s", m.confirm.fails, sDim.Render("Y yes · N no"))
}
return f.render()
}
// content is the onboarding's steps and notes, as the panel shows them.
func (m nodeModel) content() string {
var b strings.Builder
b.WriteString(sTitle.Render(m.title) + "\n\n")
detailWidth := max(20, m.width-22)
detailWidth := max(20, m.width-navWidth-26)
for _, step := range onboard.Steps {
st, ok := m.steps[step]
var mark string
@@ -238,26 +285,12 @@ func (m nodeModel) View() string {
}
switch {
case m.fabrics != nil:
var q strings.Builder
q.WriteString("These adapters are up. Which fabric is each group on?\n")
q.WriteString(sDim.Render("Give each the name it has on every other node attached to it.\nLeave a name empty to leave that group out.") + "\n\n")
for i, f := range m.fabrics.suggested {
fmt.Fprintf(&q, " %s %s\n", m.inputs[i].View(), sDim.Render(strings.Join(f.HCAs, ", ")))
}
q.WriteString("\n" + sDim.Render("tab next · enter use these · esc none"))
b.WriteString("\n" + sBox.Render(q.String()) + "\n")
case m.confirm != nil:
q := fmt.Sprintf("doctor found %d problem(s) above.\nAdd the node anyway? %s", m.confirm.fails, sDim.Render("y / N"))
b.WriteString("\n" + sBox.BorderForeground(tuiWarn).Render(q) + "\n")
case m.done && m.err != nil:
b.WriteString("\n" + sBad.Render("✗ ") + wrap(m.err.Error(), m.width-4, " ") + "\n")
case m.done:
b.WriteString("\n" + sOK.Render("✓ ") + m.summary + "\n")
case m.quitAsked:
b.WriteString("\n" + sDim.Render("stopped") + "\n")
default:
b.WriteString("\n" + sDim.Render("ctrl+c to stop") + "\n")
}
return b.String()
}
@@ -298,7 +331,7 @@ func (na *nodeAdd) runTUI(ctx context.Context, o onboard.Options, title string)
ctx, cancel := context.WithCancel(ctx)
defer cancel()
m := newNodeModel(title, cancel)
p := tea.NewProgram(m)
p := tea.NewProgram(m, tea.WithAltScreen())
emit := func(e onboard.Event) { p.Send(evMsg(e)) }
o.AskFabrics = na.rememberFabrics(func(s []onboard.Fabric) []onboard.Fabric {
@@ -343,6 +376,8 @@ func (na *nodeAdd) runTUI(ctx context.Context, o onboard.Options, title string)
return "", err
}
fm := final.(nodeModel)
// The screen is gone; what it ended with is said again where it stays.
fmt.Println(fm.content())
if fm.quitAsked && !fm.done {
cancel()
return fm.node, fmt.Errorf("stopped; the node may be half set up, and running this again finishes it")
+162 -118
View File
@@ -169,7 +169,10 @@ type setupModel struct {
spin spinnerModel
formErr string
width int
quit bool
height int
// scroll is how far the panel is scrolled, reset with each page.
scroll int
quit bool
// single is the choice of one machine: the screen closes and init's own
// form takes over.
single bool
@@ -182,7 +185,7 @@ func newSetupModel(host string, ips []string) setupModel {
names.Placeholder, names.CharLimit, names.Width = "llm.example.com, 203.0.113.10", 400, 40
return setupModel{
ref: &progRef{}, host: host, ips: ips, admin: admin, names: names,
db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100,
db: [2]dbMachine{newDBMachine(1), newDBMachine(2)}, spin: newSpinner(), width: 100, height: 30,
log: &[]string{}, logMu: &sync.Mutex{},
}
}
@@ -199,8 +202,8 @@ func (m setupModel) thisAddr() string {
func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
switch msg := msg.(type) {
case tea.WindowSizeMsg:
m.width = msg.Width
m.run.width = msg.Width
m.width, m.height = msg.Width, msg.Height
m.run.width = msg.Width - navWidth - 6
case spinTickMsg:
var cmd tea.Cmd
m.spin, cmd = m.spin.Update(msg)
@@ -224,12 +227,32 @@ func (m setupModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) {
m.quit = true
return m, tea.Quit
}
return m.key(msg)
before := m.page
next, cmd := m.key(msg)
if nm := next.(setupModel); nm.page != before {
// A new page starts at its top.
nm.scroll = 0
return nm, cmd
}
return next, cmd
}
return m, nil
}
func (m setupModel) key(k tea.KeyMsg) (tea.Model, tea.Cmd) {
// A dialog open takes the next key, whatever it is, to close it.
if m.formErr != "" {
m.formErr = ""
return m, nil
}
switch k.Type {
case tea.KeyPgDown:
m.scroll += max(1, m.height-8)
return m, nil
case tea.KeyPgUp:
m.scroll = max(0, m.scroll-max(1, m.height-8))
return m, nil
}
if k.Type == tea.KeyEsc && m.page > pWelcome && m.page < pRun && !m.checking {
m.page--
if m.page == pWait {
@@ -627,78 +650,95 @@ func (m *setupModel) startRun() tea.Cmd {
}
}
func (m setupModel) View() string {
var b strings.Builder
b.WriteString(sTitle.Render("llmgw · setup") + " " + sDim.Render(m.crumbs()) + "\n\n")
switch m.page {
case pWelcome:
b.WriteString(m.welcomeView())
case pAdmin:
b.WriteString(m.adminView())
case pMachines:
b.WriteString(m.machinesView())
case pWait:
b.WriteString(m.waitView())
case pCheck:
b.WriteString(m.checkView())
case pPlan:
b.WriteString(m.planView())
case pRun:
v := m.run.View()
if i := strings.Index(v, "\n\n"); i >= 0 {
v = v[i+2:] // its own title; this screen has one
}
b.WriteString(v)
if m.finished {
if m.run.err == nil {
b.WriteString("\n" + sDim.Render("The same, from a script, with ssh keys in place:") + "\n" + m.command() + "\n")
}
b.WriteString("\n" + sDim.Render("enter or q to leave") + "\n")
}
}
if m.formErr != "" {
b.WriteString("\n" + sBad.Render(m.formErr) + "\n")
}
return b.String()
// setupSteps are the sidebar's steps, by page.
var setupSteps = []struct {
label string
page setupPage
}{
{"Start", pWelcome}, {"Administrator", pAdmin}, {"Machines", pMachines},
{"Checks", pCheck}, {"Plan", pPlan}, {"Install", pRun},
}
func (m setupModel) crumbs() string {
steps := []string{"start", "administrator", "machines", "checks", "plan", "install"}
if m.choice == 0 {
return steps[0]
func (m setupModel) View() string {
at := m.page
if at == pWait {
at = pMachines
}
at := int(m.page)
switch {
case m.page == pWait:
at = int(pMachines)
case m.page > pWait:
at--
}
var out []string
for i, s := range steps {
if i == at {
s = lipgloss.NewStyle().Bold(true).Render(s)
var nav []navItem
for _, st := range setupSteps {
if m.choice == 0 && st.page > pWelcome {
break
}
out = append(out, s)
state := "todo"
switch {
case st.page < at || (st.page == pRun && m.finished && m.run.err == nil):
state = "done"
case st.page == at:
state = "current"
}
nav = append(nav, navItem{Label: st.label, State: state})
}
return strings.Join(out, " › ")
title, body, keys := m.pageView()
f := frame{
Width: m.width, Height: m.height,
Title: "llmgw setup", Where: m.host + " · " + m.thisAddr(),
NavTitle: "Steps", Nav: nav,
PanelTitle: title, Body: body, Scroll: m.scroll, Keys: keys,
}
if m.formErr != "" {
f.DialogTitle, f.Dialog = "Not yet", m.formErr+"\n\n"+sDim.Render("any key to go back")
}
return f.render()
}
// pageView is the panel for the page: its title, what is in it, and its keys.
func (m setupModel) pageView() (string, string, string) {
switch m.page {
case pWelcome:
return "Start", m.welcomeView(), "↑↓ choose · Enter go on · Ctrl+C leave"
case pAdmin:
return "Administrator", m.adminView(), "Tab/↑↓ move · Enter next · Esc back · Ctrl+C leave"
case pMachines:
return "Database machines", m.machinesView(), "Tab/↑↓ move · ←→ how to log in · Enter next · Esc back · PgUp/PgDn scroll"
case pWait:
return "Waiting for machines to join", m.waitView(), "Esc back · PgUp/PgDn scroll · Ctrl+C leave"
case pCheck:
keys := "R check again · Esc back · PgUp/PgDn scroll"
if !m.checking && m.report.Worst() < preflight.Fail {
keys = "Enter see the plan · " + keys
}
return "Checks", m.checkView(), keys
case pPlan:
return "Plan", m.planView(), "Enter install · Esc back · PgUp/PgDn scroll · Ctrl+C leave"
}
v := m.run.View()
if i := strings.Index(v, "\n\n"); i >= 0 {
v = v[i+2:] // init's own title; the frame has one
}
keys := "Ctrl+C stop"
if m.finished {
keys = "Enter or Q leave · PgUp/PgDn scroll"
if m.run.err == nil {
v += "\n" + sDim.Render("The same, from a script, with ssh keys in place:") + "\n" + m.command() + "\n"
}
}
return "Install", v, keys
}
func (m setupModel) welcomeView() string {
var b strings.Builder
fmt.Fprintf(&b, "Nothing of llmgw is installed on %s (%s).\n\n", m.host, strings.Join(m.ips, ", "))
fmt.Fprintf(&b, "\n Nothing of llmgw is installed on %s.\n\n", lipgloss.NewStyle().Bold(true).Render(m.host))
opt := func(i int, title, detail string) {
mark := "( )"
if m.choice == i {
mark = "(" + sOK.Render("•") + ")"
title = sTitle.Render(title)
title = fSelected.Render(title)
}
fmt.Fprintf(&b, " %s %s\n %s\n\n", mark, title, sDim.Render(detail))
}
opt(0, "One machine", "this machine runs everything: the control plane, a model mirror, and models if it has room.")
opt(0, "One machine", "This machine runs everything: control plane, model mirror,\n and models if it has room.")
opt(1, "High availability — three machines",
"this one is the control plane; two more keep the store, replicated, so losing any one loses nothing.")
b.WriteString(sDim.Render("↑↓ to choose · enter to go on · ctrl+c to leave") + "\n")
"This one is the control plane; two more keep the store, replicated,\n so losing any one machine loses nothing.")
return b.String()
}
@@ -706,104 +746,114 @@ func (m setupModel) adminView() string {
var b strings.Builder
label := func(i int, s string) string {
if m.afocus == i {
return sTitle.Render("› " + s)
return fSelected.Render("› " + s)
}
return " " + s
}
b.WriteString("\n")
fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(0, "Administrator's email"), m.admin.View(),
sDim.Render("A password is generated and shown once, at the end."))
fmt.Fprintf(&b, "%s\n %s\n %s\n\n", label(1, "Other names nodes will reach this machine by"), m.names.View(),
sDim.Render("Already known: "+strings.Join(append([]string{m.host}, m.ips...), ", ")+". Add a DNS name or a floating address."))
b.WriteString(label(2, lipgloss.NewStyle().Bold(true).Render("Next")) + "\n\n")
b.WriteString(sDim.Render("tab or ↑↓ to move · enter on Next · esc back") + "\n")
sDim.Render("Known: "+strings.Join(append([]string{m.host}, m.ips...), ", ")+". Add a DNS name or a floating address."))
b.WriteString(button(m.afocus == 2, "Next") + "\n")
return b.String()
}
// button is a [ Label ] that shows when it has the focus.
func button(focused bool, label string) string {
if focused {
return " " + fBar.Render(" "+label+" ")
}
return " " + lipgloss.NewStyle().Bold(true).Render("[ "+label+" ]")
}
func (m setupModel) machinesView() string {
var b strings.Builder
b.WriteString("The two machines that keep the store. They need ssh, a user that can sudo, and\n" +
"Ubuntu 22.04/24.04 or Debian 12; PostgreSQL, Patroni and etcd are installed on them.\n\n")
b.WriteString("\n The two machines that keep the store: Ubuntu 22.04/24.04 or Debian 12.\n" +
" PostgreSQL, Patroni and etcd are installed on them.\n\n")
for i, d := range m.db {
base := i * mfCount
// Padded before it is coloured: the colour's escape codes count
// toward a width and threw the focused row out of line.
label := func(f int, s string) string {
s = fmt.Sprintf("%-16s", s)
s = fmt.Sprintf("%-15s", s)
if m.mfocus == base+f {
return sTitle.Render("› " + s)
return fSelected.Render("› " + s)
}
return " " + s
}
fmt.Fprintf(&b, "%s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
fmt.Fprintf(&b, " %s\n", lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1)))
if d.visible(mfAddr) {
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "address"), d.addr.View())
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "user"), d.user.View())
fmt.Fprintf(&b, " %s %s\n", label(mfAddr, "Address"), d.addr.View())
fmt.Fprintf(&b, " %s %s\n", label(mfUser, "User"), d.user.View())
}
var auth []string
for a := 0; a < authCount; a++ {
if a == d.auth {
auth = append(auth, sOK.Render("["+authNames[a]+"]"))
auth = append(auth, "("+sOK.Render("•")+") "+authNames[a])
} else {
auth = append(auth, sDim.Render(" "+authNames[a]+" "))
auth = append(auth, "( ) "+sDim.Render(authNames[a]))
}
}
fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "log in with"), strings.Join(auth, " "))
fmt.Fprintf(&b, " %s %s\n", label(mfAuth, "Log in with"), strings.Join(auth, " "))
if d.visible(mfSecret) {
fmt.Fprintf(&b, " %s %s\n", label(mfSecret, authNames[d.auth]), d.secret.View())
fmt.Fprintf(&b, " %s %s\n", label(mfSecret, strings.ToUpper(authNames[d.auth][:1])+authNames[d.auth][1:]), d.secret.View())
}
if d.visible(mfSudo) {
fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "sudo password"), d.sudo.View())
fmt.Fprintf(&b, " %s %s\n", label(mfSudo, "Sudo password"), d.sudo.View())
}
if d.auth == authJoin {
b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next, and dials out to this machine.") + "\n")
b.WriteString(" " + sDim.Render("Nothing logs into it: it runs llmgw join with a code shown next.") + "\n")
}
b.WriteString("\n")
}
btn := lipgloss.NewStyle().Bold(true).Render("Check the three machines")
if m.mfocus == 2*mfCount {
btn = sTitle.Render("› " + btn)
} else {
btn = " " + btn
}
b.WriteString(btn + "\n\n")
b.WriteString(sDim.Render("tab or ↑↓ to move · ←→ to choose how to log in · passwords are kept in memory only · esc back") + "\n")
b.WriteString(button(m.mfocus == 2*mfCount, "Check the three machines") + "\n\n")
b.WriteString(" " + sDim.Render("Passwords are kept in memory only, for this setup.") + "\n")
return b.String()
}
func (m setupModel) waitView() string {
var b strings.Builder
fmt.Fprintf(&b, "On each machine below, as root. It dials this machine on port %d; nothing dials in.\n\n", joinpkg.DefaultPort)
fmt.Fprintf(&b, "\n On each machine below, as root. It dials this machine on port %d;\n nothing dials in. Each code is for one machine and good for an hour.\n\n", joinpkg.DefaultPort)
for i, d := range m.db {
if d.auth != authJoin {
continue
}
title := lipgloss.NewStyle().Bold(true).Render(fmt.Sprintf("Database machine %d", i+1))
if d.joined != nil {
fmt.Fprintf(&b, "%s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
fmt.Fprintf(&b, " %s %s joined as %s (%s)\n\n", title, sOK.Render("✓"), d.joined.Hostname, d.joined.Addr)
continue
}
fmt.Fprintf(&b, "%s %s waiting\n", title, m.spin.View())
fmt.Fprintf(&b, " %s %s waiting\n", title, m.spin.View())
b.WriteString(" " + sDim.Render("without llmgw yet:") + "\n")
for _, l := range joinInstall() {
b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n")
b.WriteString(" " + strings.ReplaceAll(l, "\n", "\n ") + "\n")
}
b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n")
b.WriteString(" " + sDim.Render("then:") + "\n sudo llmgw join " + d.code + "\n\n")
}
b.WriteString(sDim.Render("Each code is for one machine and good for an hour. esc back · ctrl+c to leave") + "\n")
return b.String()
}
func (m setupModel) checkView() string {
var b strings.Builder
b.WriteString("\n")
if m.checking {
b.WriteString(" " + m.spin.View() + " checking the three machines…\n")
return b.String()
}
switch m.report.Worst() {
case preflight.Fail:
b.WriteString(" " + sBad.Render("✗ Fix what is marked ✗, then check again (R).") + "\n")
case preflight.Warn:
b.WriteString(" " + sWarn.Render("! It can go ahead; what is marked ! is worth a look.") + "\n")
default:
b.WriteString(" " + sOK.Render("✓ All three are ready.") + "\n")
}
machine := ""
for _, f := range m.report.Findings {
if f.Machine != machine {
machine = f.Machine
b.WriteString("\n" + lipgloss.NewStyle().Bold(true).Render(machine) + "\n")
b.WriteString("\n " + lipgloss.NewStyle().Bold(true).Render(machine) + "\n")
}
mark := sOK.Render("✓")
switch f.Level {
@@ -812,51 +862,45 @@ func (m setupModel) checkView() string {
case preflight.Fail:
mark = sBad.Render("✗")
}
line := fmt.Sprintf(" %s %-13s %s", mark, f.Check, truncate(f.Detail, max(20, m.width-22)))
fmt.Fprintf(&b, " %s %-13s %s\n", mark, f.Check, f.Detail)
if f.Level != preflight.OK && f.Fix != "" {
line += "\n " + sDim.Render(truncate(f.Fix, max(20, m.width-20)))
b.WriteString(" " + sDim.Render(f.Fix) + "\n")
}
b.WriteString(line + "\n")
}
b.WriteString("\n")
switch m.report.Worst() {
case preflight.Fail:
b.WriteString(sBad.Render("Fix what is marked ✗, then check again.") + "\n" +
sDim.Render("r to check again · esc to change the machines") + "\n")
case preflight.Warn:
b.WriteString(sWarn.Render("It can go ahead; what is marked ! is worth a look.") + "\n" +
sDim.Render("enter to see the plan · r to check again · esc back") + "\n")
default:
b.WriteString(sOK.Render("All three are ready.") + "\n" + sDim.Render("enter to see the plan · esc back") + "\n")
}
return b.String()
}
func (m setupModel) planView() string {
var b strings.Builder
b.WriteString("What will be installed:\n\n")
b.WriteString("\n What will be installed:\n\n")
fmt.Fprintf(&b, " %s %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(m.host), sDim.Render(m.thisAddr()),
"control plane and gateway, model mirror, etcd (the third vote), administrator "+strings.TrimSpace(m.admin.Value()))
"control plane and gateway, model mirror, etcd (the third vote)")
fmt.Fprintf(&b, " administrator %s\n", strings.TrimSpace(m.admin.Value()))
granted := false
for _, d := range m.db {
fmt.Fprintf(&b, " %s\n %s\n", lipgloss.NewStyle().Bold(true).Render(strings.TrimSpace(d.addr.Value())),
"PostgreSQL under Patroni, etcd")
addr := strings.TrimSpace(d.addr.Value())
if d.joined != nil {
addr = d.joined.Hostname + " " + sDim.Render(d.joined.Addr+", joined")
}
fmt.Fprintf(&b, " %s\n PostgreSQL under Patroni, etcd\n", lipgloss.NewStyle().Bold(true).Render(addr))
granted = granted || d.sudo.Value() != ""
}
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost the\n" +
" standby takes over in about thirty seconds, and the control plane follows it.\n")
b.WriteString("\n The store is replicated from the first to the second; if the primary is lost\n" +
" the standby takes over in about thirty seconds, and the control plane follows.\n")
if granted {
b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the install\n finishes, and not after.") + "\n")
b.WriteString(" " + sDim.Render("Where a sudo password was given, the user may sudo without it until the\n install finishes, and not after.") + "\n")
}
b.WriteString("\n " + sDim.Render("The same from a script, with ssh keys in place:") + "\n")
for _, l := range strings.Split(m.command(), "\n") {
b.WriteString(" " + l + "\n")
}
b.WriteString("\n" + sDim.Render("The same from a script, with ssh keys in place:") + "\n" + m.command() + "\n")
for _, d := range m.db {
if d.joined != nil {
b.WriteString(sDim.Render("(a machine that joined with a code needs ssh for that: the code is for these screens)") + "\n")
b.WriteString(" " + sDim.Render("(a machine that joined with a code needs ssh for that)") + "\n")
break
}
}
b.WriteString("\n")
b.WriteString(lipgloss.NewStyle().Bold(true).Render("enter to install") + sDim.Render(" · esc back · ctrl+c to leave") + "\n")
b.WriteString("\n" + button(true, "Enter to install") + "\n")
return b.String()
}
@@ -871,7 +915,7 @@ func runSetupTUI(ctx context.Context, out io.Writer) error {
}
}
m := newSetupModel(host, ips)
p := tea.NewProgram(m, tea.WithOutput(out))
p := tea.NewProgram(m, tea.WithOutput(out), tea.WithAltScreen())
m.ref.p = p
final, err := p.Run()
if err != nil {
+21 -2
View File
@@ -43,8 +43,12 @@ func TestTheSetupAsksForWhatHighAvailabilityNeedsAndRefusesWhatWillNotDo(t *test
if s := m.(setupModel); s.page != pAdmin || !strings.Contains(s.formErr, "email") {
t.Fatalf("went on without an administrator: %+v", s.formErr)
}
// The refusal puts the cursor back on the email.
m = press(m, "a@example.com", "tab", "tab", "enter")
// The refusal is a dialog; a key closes it, and the cursor is back on
// the email.
if !strings.Contains(m.View(), "Not yet") {
t.Error("the refusal was not shown as a dialog")
}
m = press(m, "x", "a@example.com", "tab", "tab", "enter")
s := m.(setupModel)
if s.page != pMachines {
t.Fatalf("page %d: %s", s.page, s.formErr)
@@ -117,3 +121,18 @@ func TestTheConsoleOffersToReplicateOnlyAStoreThatIsNot(t *testing.T) {
}
_ = adminapi.Summary{}
}
func TestTheSetupIsDrawnAsWindows(t *testing.T) {
m := newSetupModel("ctl", []string{"10.0.0.5"})
m.width, m.height = 100, 30
v := m.View()
lines := strings.Split(v, "\n")
if len(lines) != 30 {
t.Errorf("%d lines for a 30-line terminal", len(lines))
}
for _, want := range []string{"llmgw setup", "Steps", "Start", "╭", "One machine", "Ctrl+C"} {
if !strings.Contains(v, want) {
t.Errorf("the screen lacks %q:\n%s", want, v)
}
}
}
@@ -0,0 +1,65 @@
package agentsvc
import (
"context"
"slices"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
)
// SetControlPlanes gives the service where the list of control planes comes
// from. Every agent is told it when it registers, and again when it changes.
func (s *Service) SetControlPlanes(list func(context.Context) ([]string, error)) {
s.mu.Lock()
s.controlPlanes = list
s.mu.Unlock()
}
// controlPlanesMsg is the list as an agent is sent it, or nil when there is
// none to send: a cluster with no list recorded keeps what its agents were
// configured with.
func (s *Service) controlPlanesMsg(ctx context.Context) *llmgwv1.ControlMessage {
s.mu.RLock()
list := s.controlPlanes
s.mu.RUnlock()
if list == nil {
return nil
}
servers, err := list(ctx)
if err != nil || len(servers) == 0 {
return nil
}
return &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_ControlPlanes{
ControlPlanes: &llmgwv1.ControlPlanes{Servers: servers}}}
}
// WatchControlPlanes tells every connected agent the list when it changes,
// checking every so often. A control plane added by `llmgw server add` reaches
// the nodes within that time, without their being touched.
func (s *Service) WatchControlPlanes(ctx context.Context, every time.Duration) {
var last []string
t := time.NewTicker(every)
defer t.Stop()
for {
if msg := s.controlPlanesMsg(ctx); msg != nil {
now := msg.GetControlPlanes().GetServers()
if !slices.Equal(now, last) {
for _, node := range s.ConnectedNodes() {
if err := s.send(node, msg); err != nil {
s.log.Warn("could not tell a node the control planes", "node", node, "err", err)
}
}
if last != nil {
s.log.Info("told the nodes the control planes changed", "servers", now)
}
last = now
}
}
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
+10
View File
@@ -61,6 +61,9 @@ type Service struct {
choices func(nodeID string) map[string]string
installableBy map[string][]string
// controlPlanes lists what agents may dial; see SetControlPlanes.
controlPlanes func(context.Context) ([]string, error)
// logWaits are the log requests awaiting a reply, by request id.
logMu sync.Mutex
logWaits map[string]chan *llmgwv1.LogsReply
@@ -229,6 +232,13 @@ func (s *Service) Connect(
return err
}
}
// Which control planes there are, so an agent configured with one learns
// of a second added since.
if msg := s.controlPlanesMsg(ctx); msg != nil {
if err := stream.Send(msg); err != nil {
return err
}
}
// Recover replica state from what this node is already running, before any
// reconciliation pass can conclude the cluster is empty and place
+13 -6
View File
@@ -22,6 +22,9 @@ const Scope = "llmgw"
// control plane reads the cluster's state from.
const PatroniPort = 8008
// restPort is where this package asks Patroni; a test moves it.
var restPort = PatroniPort
// Role and Database are what the control plane connects as.
const (
Role = "llmgw"
@@ -83,6 +86,10 @@ func Patroni(m Member, db, etcd []Member, clients []string, version string, s Se
seen[c] = true
hba = append(hba, fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, c))
}
var hbaLines string
for _, h := range hba {
hbaLines += " - " + h + "\n"
}
var b strings.Builder
fmt.Fprintf(&b, `# llmgw: the control planes' store, kept by Patroni, its leader lock in etcd.
scope: %[1]s
@@ -113,7 +120,11 @@ bootstrap:
wal_keep_size: 1GB
wal_log_hints: "on"
password_encryption: scram-sha-256
initdb:
# In the cluster's shared configuration rather than each member's, so a
# control plane added later is let in by one change through Patroni's
# API (llmgw server add) instead of an edit on every member.
pg_hba:
%[9]s initdb:
- encoding: UTF8
- data-checksums
@@ -135,11 +146,7 @@ postgresql:
password: "%[7]s"
parameters:
unix_socket_directories: /var/run/postgresql
pg_hba:
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator)
for _, h := range hba {
fmt.Fprintf(&b, " - %s\n", h)
}
`, Scope, m.Name, m.IP, PatroniPort, strings.Join(hosts, ","), version, s.Superuser, s.Replicator, hbaLines)
b.WriteString(`
tags:
nofailover: false
+91
View File
@@ -0,0 +1,91 @@
package pgha
import (
"bytes"
"context"
"encoding/json"
"fmt"
"net"
"net/http"
"slices"
"strconv"
"time"
)
// ClientLine is the access rule that lets a control plane at ip in.
func ClientLine(ip string) string {
return fmt.Sprintf("host %s %s %s/32 scram-sha-256", Database, Role, ip)
}
// AllowClient lets a control plane at ip into the cluster's database: one
// change to the cluster's shared configuration through Patroni's API, which
// every member applies. It says whether anything changed.
//
// A cluster set up before the rules moved into the shared configuration
// keeps them in each member's own file, which this cannot reach; that is
// said, with what to add, rather than half done.
func AllowClient(ctx context.Context, hosts []string, ip string) (bool, error) {
client := &http.Client{Timeout: 5 * time.Second}
var lastErr error
for _, h := range hosts {
base := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort))
cfg, err := getConfig(ctx, client, base)
if err != nil {
lastErr = err
continue
}
var hba []string
if pg, ok := cfg["postgresql"].(map[string]any); ok {
if list, ok := pg["pg_hba"].([]any); ok {
for _, l := range list {
if s, ok := l.(string); ok {
hba = append(hba, s)
}
}
}
}
if len(hba) == 0 {
return false, fmt.Errorf("this cluster keeps its access rules in each member's "+
"/etc/patroni/config.yml, not in its shared configuration: add %q to pg_hba there "+
"on every member, and run `patronictl reload llmgw`", ClientLine(ip))
}
line := ClientLine(ip)
if slices.Contains(hba, line) {
return false, nil
}
body, _ := json.Marshal(map[string]any{"postgresql": map[string]any{"pg_hba": append(hba, line)}})
req, err := http.NewRequestWithContext(ctx, http.MethodPatch, base+"/config", bytes.NewReader(body))
if err != nil {
return false, err
}
req.Header.Set("Content-Type", "application/json")
resp, err := client.Do(req)
if err != nil {
lastErr = err
continue
}
resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return false, fmt.Errorf("Patroni on %s refused the change: %s", h, resp.Status)
}
return true, nil
}
return false, fmt.Errorf("no member's Patroni answered: %w", lastErr)
}
func getConfig(ctx context.Context, client *http.Client, base string) (map[string]any, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/config", nil)
if err != nil {
return nil, err
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
var cfg map[string]any
if err := json.NewDecoder(resp.Body).Decode(&cfg); err != nil {
return nil, fmt.Errorf("%s/config: %w", base, err)
}
return cfg, nil
}
+72
View File
@@ -3,6 +3,10 @@ package pgha
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"net/url"
"strconv"
"strings"
"sync"
"testing"
@@ -10,6 +14,7 @@ import (
"github.com/prometheus/client_golang/prometheus/testutil"
"github.com/prometheus/common/expfmt"
"go.yaml.in/yaml/v3"
)
var (
@@ -208,3 +213,70 @@ func TestAMemberThatStoppedIsReportedMissingNotForgotten(t *testing.T) {
t.Errorf("%s", cl)
}
}
func TestTheAccessRulesAreInTheClustersSharedConfiguration(t *testing.T) {
// Where `server add` can change them for every member at once.
cfg := Patroni(pg01, []Member{pg01, ctl02}, []Member{pg01, ctl02, ctl}, []string{"10.0.0.140"}, "16", Secrets{"a", "b", "c"})
var doc struct {
Bootstrap struct {
DCS struct {
PostgreSQL struct {
PgHBA []string `yaml:"pg_hba"`
} `yaml:"postgresql"`
} `yaml:"dcs"`
} `yaml:"bootstrap"`
PostgreSQL map[string]any `yaml:"postgresql"`
}
if err := yaml.Unmarshal([]byte(cfg), &doc); err != nil {
t.Fatalf("not YAML: %v\n%s", err, cfg)
}
hba := doc.Bootstrap.DCS.PostgreSQL.PgHBA
if len(hba) < 5 || hba[0] != "local all all peer" {
t.Errorf("shared pg_hba: %q", hba)
}
if _, local := doc.PostgreSQL["pg_hba"]; local {
t.Error("pg_hba is also in the member's own configuration, where it would win over the shared one")
}
}
func TestAControlPlaneIsLetInThroughTheSharedConfiguration(t *testing.T) {
var patched []string
hba := []any{"local all all peer", ClientLine("10.0.0.140")}
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
_ = json.NewEncoder(w).Encode(map[string]any{"postgresql": map[string]any{"pg_hba": hba}})
case http.MethodPatch:
var body struct {
PostgreSQL struct {
PgHBA []string `json:"pg_hba"`
} `json:"postgresql"`
}
_ = json.NewDecoder(r.Body).Decode(&body)
patched = body.PostgreSQL.PgHBA
}
}))
defer srv.Close()
u, _ := url.Parse(srv.URL)
old := restPort
restPort, _ = strconv.Atoi(u.Port())
defer func() { restPort = old }()
changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.150")
if err != nil || !changed {
t.Fatalf("%v %v", changed, err)
}
// Everything that was there is kept, and the new line added once.
if len(patched) != 3 || patched[2] != ClientLine("10.0.0.150") || patched[1] != ClientLine("10.0.0.140") {
t.Errorf("patched %q", patched)
}
if changed, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.140"); err != nil || changed {
t.Errorf("one already let in was changed again: %v %v", changed, err)
}
hba = nil // the rules are in each member's file
if _, err := AllowClient(context.Background(), []string{"127.0.0.1"}, "10.0.0.151"); err == nil ||
!strings.Contains(err.Error(), "each member") {
t.Errorf("rules kept per member: %v", err)
}
}
+1 -1
View File
@@ -73,7 +73,7 @@ func HTTPStatus(ctx context.Context, hosts []string) (Cluster, error) {
client := &http.Client{Timeout: 3 * time.Second}
var last error
for _, h := range hosts {
u := "http://" + net.JoinHostPort(h, strconv.Itoa(PatroniPort)) + "/cluster"
u := "http://" + net.JoinHostPort(h, strconv.Itoa(restPort)) + "/cluster"
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
if err != nil {
return Cluster{}, err
+14 -1
View File
@@ -123,6 +123,19 @@ func (r *Router) Handler() http.Handler {
w.WriteHeader(http.StatusOK)
})
// Leadership: 200 on the control plane that places work, 503 on a
// standby. What a floating address follows (llmgw server add --vip):
// unlike /ready it does not wait for a replica, so a cluster with no
// node yet still has its address on one machine. A process that is not
// a control plane, or one with no rival, is always the leader.
mux.HandleFunc("GET /leader", func(w http.ResponseWriter, _ *http.Request) {
if f := r.isLeader.Load(); f != nil && !(*f)() {
http.Error(w, "this control plane is a standby", http.StatusServiceUnavailable)
return
}
w.WriteHeader(http.StatusOK)
})
// The landing page says how to connect and nothing about what is here, so
// it is safe to serve without a key — and it has to be, because the person
// reading it is the one who does not have one yet.
@@ -227,7 +240,7 @@ func (r *Router) Handler() http.Handler {
// route must not be able to add an unauthenticated one by omission. Health
// stays open because a load balancer has no key.
return auth.Middleware(r.auth, r.log,
"/health", "/ready", "/", "/about", "/_static/", "/claim/",
"/health", "/ready", "/leader", "/", "/about", "/_static/", "/claim/",
"/signup", "/login", "/logout", "/profile", "/profile/",
"/admin", "/admin/", "/chat", "/chat/", "/org", "/org/", "/setup/")(mux)
}
+43
View File
@@ -0,0 +1,43 @@
package router
import (
"net/http"
"net/http/httptest"
"path/filepath"
"testing"
)
func TestTheLeaderEndpointFollowsLeadership(t *testing.T) {
rt := newTestRouter(t, baseConfig())
get := func() int {
rec := httptest.NewRecorder()
rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/leader", nil))
return rec.Code
}
// No rival, nothing to follow: the one process leads.
if c := get(); c != http.StatusOK {
t.Errorf("with no leadership: %d", c)
}
leading := false
rt.SetLeader(func() bool { return leading })
if c := get(); c != http.StatusServiceUnavailable {
t.Errorf("a standby: %d", c)
}
leading = true
if c := get(); c != http.StatusOK {
t.Errorf("the leader: %d", c)
}
}
func TestTheLeaderEndpointNeedsNoKey(t *testing.T) {
// keepalived and load balancers ask it, and they have no key.
cfg := baseConfig()
cfg.Auth.Enabled = true
cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db")
rt := newTestRouter(t, cfg)
rec := httptest.NewRecorder()
rt.Handler().ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/leader", nil))
if rec.Code != http.StatusOK {
t.Errorf("/leader without a key: %d", rec.Code)
}
}
+5
View File
@@ -83,6 +83,8 @@ type Router struct {
auth *auth.Authenticator
accounts *account.Service
store *storepkg.DB
// isLeader says whether this process leads; nil is always.
isLeader atomic.Pointer[func() bool]
acct *accounting.Accountant
// threads holds conversations and memory notes. It is store when auth is on,
@@ -474,6 +476,9 @@ func (r *Router) Stop() {
}
}
// SetLeader tells /leader how to know whether this process leads.
func (r *Router) SetLeader(f func() bool) { r.isLeader.Store(&f) }
// Registry exposes the endpoint registry, for tests and admin endpoints.
func (r *Router) Registry() *endpoint.Registry { return r.reg }
+44
View File
@@ -0,0 +1,44 @@
package store
import (
"context"
"database/sql"
"time"
)
// ControlPlane is one control plane agents may dial.
type ControlPlane struct {
// URL is what an agent dials: https://10.0.0.150:7443.
URL string
Name string
AddedAt time.Time
}
// ControlPlanes lists them, oldest first: the first is the one agents try
// first.
func (s *DB) ControlPlanes(ctx context.Context) ([]ControlPlane, error) {
return queryAll(ctx, s.db, `SELECT url, name, added_at FROM control_planes ORDER BY added_at, url`, nil,
func(rows *sql.Rows) (ControlPlane, error) {
var c ControlPlane
var at int64
err := rows.Scan(&c.URL, &c.Name, &at)
c.AddedAt = fromMillis(at)
return c, err
})
}
// PutControlPlane records one, keeping when it was first added.
func (s *DB) PutControlPlane(ctx context.Context, c ControlPlane) error {
if c.AddedAt.IsZero() {
c.AddedAt = time.Now()
}
_, err := s.db.ExecContext(ctx, `INSERT INTO control_planes (url, name, added_at) VALUES (?, ?, ?)
ON CONFLICT (url) DO UPDATE SET name = excluded.name`, c.URL, c.Name, millis(c.AddedAt))
return err
}
// RemoveControlPlane forgets one.
func (s *DB) RemoveControlPlane(ctx context.Context, url string) error {
_, err := s.db.ExecContext(ctx, `DELETE FROM control_planes WHERE url = ?`, url)
return err
}
+1
View File
@@ -46,6 +46,7 @@ var copyOrder = []string{
// of them at the moment of the move.
"cluster_secrets",
"node_engines",
"control_planes",
}
// notCopied are tables deliberately left behind.
@@ -0,0 +1,10 @@
-- Ported from the SQLite migration of the same number.
-- The control planes agents may dial, by the address they dial: what the
-- leader tells every agent, so a control plane added after the nodes were
-- onboarded is one they fail over to without their configuration being
-- edited. Written by `llmgw server add` and by init for the first one.
CREATE TABLE control_planes (
url TEXT PRIMARY KEY,
name TEXT NOT NULL,
added_at BIGINT NOT NULL
);
@@ -0,0 +1,9 @@
-- The control planes agents may dial, by the address they dial: what the
-- leader tells every agent, so a control plane added after the nodes were
-- onboarded is one they fail over to without their configuration being
-- edited. Written by `llmgw server add` and by init for the first one.
CREATE TABLE control_planes (
url TEXT PRIMARY KEY,
name TEXT NOT NULL,
added_at INTEGER NOT NULL
);
+1 -1
View File
@@ -43,7 +43,7 @@ func openPG(t *testing.T) *DB {
// every subtest gets a fresh file, and failed only here.
`TRUNCATE thread_messages, threads, memory_notes, usage, api_keys, memberships,
key_claims, limits, models, sessions, users, projects, teams,
organizations, leadership RESTART IDENTITY CASCADE`)
organizations, leadership, control_planes RESTART IDENTITY CASCADE`)
if err != nil {
t.Fatalf("clearing the test database: %v", err)
}
+25
View File
@@ -493,3 +493,28 @@ func TestNoDefaultsLeavesLimitsUnset(t *testing.T) {
t.Errorf("limits appeared with no defaults set: rpm=%v tpm=%v", e.RPM, e.TPM)
}
}
func TestControlPlanesAreKeptInTheOrderTheyWereAdded(t *testing.T) {
ctx := context.Background()
s := openDB(t)
at := time.Now()
for i, u := range []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"} {
if err := s.PutControlPlane(ctx, ControlPlane{URL: u, Name: "c", AddedAt: at.Add(time.Duration(i) * time.Second)}); err != nil {
t.Fatal(err)
}
}
// Recorded again, as init and server add do: not moved, not doubled.
if err := s.PutControlPlane(ctx, ControlPlane{URL: "https://10.0.0.140:7443", Name: "ctl"}); err != nil {
t.Fatal(err)
}
cps, err := s.ControlPlanes(ctx)
if err != nil || len(cps) != 2 || cps[0].URL != "https://10.0.0.140:7443" || cps[0].Name != "ctl" {
t.Fatalf("%+v %v", cps, err)
}
if err := s.RemoveControlPlane(ctx, cps[1].URL); err != nil {
t.Fatal(err)
}
if cps, _ := s.ControlPlanes(ctx); len(cps) != 1 {
t.Errorf("%+v", cps)
}
}