Add a standby control plane with server add, and draw the screens as windows

server add gives another machine this llmgw, a certificate, the
configuration and store address, lets it into the store through Patroni, and
starts it; --vip keeps a floating address on the leader with keepalived,
following a new /leader endpoint. Nodes learn every control plane from the
leader and notice a dead one by HTTP/2 ping instead of after the kernel's
fifteen minutes. The setup, console, node and init screens are drawn full
screen in one frame: title bar, sidebar, bordered panel, key bar, dialogs.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-03 22:29:12 +02:00
1 parent f5c79052b1
commit aa3efcebbe
31 files changed
+1856 -394

No files matched your search

+39 -7
View File
@@ -73,8 +73,11 @@ type Agent struct {
lastEngineProbe time.Time
probingEngines bool
mu sync.Mutex
inv discovery.Inventory
mu sync.Mutex
// serverList is every control plane to dial: the configured ones, and
// any the leader has named since. Read through servers().
serverList []string
inv discovery.Inventory
// shardReplica remembers which replica a shard belongs to, so a status
// report can name both. The control plane keys on the replica.
shardReplica map[string]string
@@ -153,6 +156,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
load: discovery.NewLoadSampler(),
sensors: discovery.NewSensorSampler(),
wear: discovery.NewHealthSampler(),
serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
}
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
return a, nil
@@ -185,7 +189,9 @@ func (a *Agent) Run(ctx context.Context) error {
attempt := 0
next := 0
for ctx.Err() == nil {
server := a.cfg.Servers[next%len(a.cfg.Servers)]
// Read each time round: the leader may have named another since.
servers := a.servers()
server := servers[next%len(servers)]
err := a.session(ctx, server)
if ctx.Err() != nil {
break
@@ -200,12 +206,12 @@ func (a *Agent) Run(ctx context.Context) error {
// No wait when there is somewhere else to try: the delay exists to
// stop hammering one control plane, and the next address has not been
// hammered. With a single address this is the old behaviour exactly.
if len(a.cfg.Servers) > 1 && next%len(a.cfg.Servers) != 0 {
if len(servers) > 1 && next%len(servers) != 0 {
delay = 0
}
a.log.Warn("disconnected from the control plane, retrying",
"err", err, "server", server,
"next", a.cfg.Servers[next%len(a.cfg.Servers)],
"next", servers[next%len(servers)],
"in", delay.Round(time.Millisecond), "attempt", attempt)
select {
@@ -792,6 +798,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
a.setProvision(m.Provision)
case *llmgwv1.ControlMessage_Upgrade:
go a.upgrade(ctx, m.Upgrade, out)
case *llmgwv1.ControlMessage_ControlPlanes:
a.setControlPlanes(m.ControlPlanes)
case *llmgwv1.ControlMessage_Logs:
reply := a.logsReply(m.Logs)
// Off the receive loop: a full send queue must not stop commands being
@@ -1084,7 +1092,9 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
if !cfg.Enabled() {
return &http.Client{
Transport: &http2.Transport{
AllowHTTP: true,
AllowHTTP: true,
ReadIdleTimeout: streamIdle,
PingTimeout: streamPing,
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
},
@@ -1099,9 +1109,31 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
if err != nil {
return nil, err
}
return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
return &http.Client{Transport: &http2.Transport{
TLSClientConfig: tc,
ReadIdleTimeout: streamIdle,
PingTimeout: streamPing,
// Bounded, as the plaintext branch's is: dialling a control plane
// whose machine is off waits out the kernel's connect retries,
// about two minutes, before the next address is tried.
DialTLSContext: func(ctx context.Context, network, addr string, c *tls.Config) (net.Conn, error) {
d := tls.Dialer{NetDialer: &net.Dialer{Timeout: timeout}, Config: c}
return d.DialContext(ctx, network, addr)
},
}}, nil
}
// A stream to a control plane whose machine stopped dead — powered off, cut
// off — is never closed: nothing sends the FIN, and the agent's heartbeats sit
// in the socket's buffer until the kernel gives up retransmitting, fifteen
// minutes later. Found by switching off a leader: the standby took over in 14
// seconds and the node stayed with the dead one. An HTTP/2 ping after ten
// quiet seconds, given five to answer, finds it in a quarter of a minute.
const (
streamIdle = 10 * time.Second
streamPing = 5 * time.Second
)
// localizeSpec fills in what only this node knows.
//
// The interface NCCL bootstraps over is this node's, not the cluster's: the
+77
View File
@@ -0,0 +1,77 @@
package agent
import (
"encoding/json"
"os"
"path/filepath"
"slices"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
)
// controlPlanesFile keeps the control planes the leader last named, beside
// the agent's configuration rather than in it: the configuration is the
// operator's, and a list rewritten in it would be a file nobody wrote.
var controlPlanesFile = "/var/lib/llmgw-agent/control-planes.json"
// servers is every address to dial, in the order to try them: the
// configuration's, then any the leader named that it lacks.
func (a *Agent) servers() []string {
a.mu.Lock()
defer a.mu.Unlock()
if len(a.serverList) == 0 {
return a.cfg.Servers
}
return a.serverList
}
// mergeServers puts the leader's list after the configured one, without
// repeats, and says whether that changed anything.
func mergeServers(configured, told []string, secure bool) []string {
out := slices.Clone(configured)
for _, s := range told {
fixed, err := normalizeOne(s, secure)
if err != nil {
continue
}
if !slices.Contains(out, fixed) {
out = append(out, fixed)
}
}
return out
}
// setControlPlanes takes the leader's list: used from the next reconnect,
// and kept for the next start.
func (a *Agent) setControlPlanes(m *llmgwv1.ControlPlanes) {
told := m.GetServers()
merged := mergeServers(a.cfg.Servers, told, a.cfg.TLS.Enabled())
a.mu.Lock()
changed := !slices.Equal(merged, a.serverList)
a.serverList = merged
a.mu.Unlock()
if !changed {
return
}
a.log.Info("the control planes to dial", "servers", merged)
b, _ := json.Marshal(told)
if err := os.MkdirAll(filepath.Dir(controlPlanesFile), 0o755); err == nil {
tmp := controlPlanesFile + ".new"
if err := os.WriteFile(tmp, b, 0o644); err == nil {
_ = os.Rename(tmp, controlPlanesFile)
}
}
}
// loadControlPlanes is the list kept from the last time the leader named it.
func loadControlPlanes(configured []string, secure bool) []string {
b, err := os.ReadFile(controlPlanesFile)
if err != nil {
return configured
}
var told []string
if json.Unmarshal(b, &told) != nil {
return configured
}
return mergeServers(configured, told, secure)
}
+41
View File
@@ -0,0 +1,41 @@
package agent
import (
"log/slog"
"path/filepath"
"slices"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
func TestAControlPlaneTheLeaderNamesIsDialledAndRemembered(t *testing.T) {
old := controlPlanesFile
controlPlanesFile = filepath.Join(t.TempDir(), "control-planes.json")
t.Cleanup(func() { controlPlanesFile = old })
configured := []string{"https://10.0.0.140:7443"}
// TLS on, as every onboarded node has it: an agent without it refuses
// https:// addresses, and the leader's are https://.
a := &Agent{cfg: Config{Servers: configured, TLS: tlsconf.Config{CA: "/etc/llmgw/ca.pem"}}, log: slog.New(slog.DiscardHandler)}
if got := a.servers(); !slices.Equal(got, configured) {
t.Fatalf("before anything is named: %v", got)
}
// The configured one stays first; the new one is added after it, once.
a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}})
want := []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}
if got := a.servers(); !slices.Equal(got, want) {
t.Errorf("after the leader named two: %v", got)
}
// And after a restart, before the leader says anything.
if got := loadControlPlanes(configured, true); !slices.Equal(got, want) {
t.Errorf("after a restart: %v", got)
}
// A list that drops the configured one does not drop it: the
// configuration is the operator's.
a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.150:7443"}})
if got := a.servers(); got[0] != "https://10.0.0.140:7443" {
t.Errorf("the configured server was dropped: %v", got)
}
}