Add a standby control plane with server add, and draw the screens as windows
server add gives another machine this llmgw, a certificate, the configuration and store address, lets it into the store through Patroni, and starts it; --vip keeps a floating address on the leader with keepalived, following a new /leader endpoint. Nodes learn every control plane from the leader and notice a dead one by HTTP/2 ping instead of after the kernel's fifteen minutes. The setup, console, node and init screens are drawn full screen in one frame: title bar, sidebar, bordered panel, key bar, dialogs. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
f5c79052b1
commit
aa3efcebbe
31 files changed
+1856
-394
No files matched your search
+39
-7
@@ -73,8 +73,11 @@ type Agent struct {
|
||||
lastEngineProbe time.Time
|
||||
probingEngines bool
|
||||
|
||||
mu sync.Mutex
|
||||
inv discovery.Inventory
|
||||
mu sync.Mutex
|
||||
// serverList is every control plane to dial: the configured ones, and
|
||||
// any the leader has named since. Read through servers().
|
||||
serverList []string
|
||||
inv discovery.Inventory
|
||||
// shardReplica remembers which replica a shard belongs to, so a status
|
||||
// report can name both. The control plane keys on the replica.
|
||||
shardReplica map[string]string
|
||||
@@ -153,6 +156,7 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
||||
load: discovery.NewLoadSampler(),
|
||||
sensors: discovery.NewSensorSampler(),
|
||||
wear: discovery.NewHealthSampler(),
|
||||
serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
|
||||
}
|
||||
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
||||
return a, nil
|
||||
@@ -185,7 +189,9 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
attempt := 0
|
||||
next := 0
|
||||
for ctx.Err() == nil {
|
||||
server := a.cfg.Servers[next%len(a.cfg.Servers)]
|
||||
// Read each time round: the leader may have named another since.
|
||||
servers := a.servers()
|
||||
server := servers[next%len(servers)]
|
||||
err := a.session(ctx, server)
|
||||
if ctx.Err() != nil {
|
||||
break
|
||||
@@ -200,12 +206,12 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
// No wait when there is somewhere else to try: the delay exists to
|
||||
// stop hammering one control plane, and the next address has not been
|
||||
// hammered. With a single address this is the old behaviour exactly.
|
||||
if len(a.cfg.Servers) > 1 && next%len(a.cfg.Servers) != 0 {
|
||||
if len(servers) > 1 && next%len(servers) != 0 {
|
||||
delay = 0
|
||||
}
|
||||
a.log.Warn("disconnected from the control plane, retrying",
|
||||
"err", err, "server", server,
|
||||
"next", a.cfg.Servers[next%len(a.cfg.Servers)],
|
||||
"next", servers[next%len(servers)],
|
||||
"in", delay.Round(time.Millisecond), "attempt", attempt)
|
||||
|
||||
select {
|
||||
@@ -792,6 +798,8 @@ func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out cha
|
||||
a.setProvision(m.Provision)
|
||||
case *llmgwv1.ControlMessage_Upgrade:
|
||||
go a.upgrade(ctx, m.Upgrade, out)
|
||||
case *llmgwv1.ControlMessage_ControlPlanes:
|
||||
a.setControlPlanes(m.ControlPlanes)
|
||||
case *llmgwv1.ControlMessage_Logs:
|
||||
reply := a.logsReply(m.Logs)
|
||||
// Off the receive loop: a full send queue must not stop commands being
|
||||
@@ -1084,7 +1092,9 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
|
||||
if !cfg.Enabled() {
|
||||
return &http.Client{
|
||||
Transport: &http2.Transport{
|
||||
AllowHTTP: true,
|
||||
AllowHTTP: true,
|
||||
ReadIdleTimeout: streamIdle,
|
||||
PingTimeout: streamPing,
|
||||
DialTLSContext: func(ctx context.Context, network, addr string, _ *tls.Config) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: timeout}).DialContext(ctx, network, addr)
|
||||
},
|
||||
@@ -1099,9 +1109,31 @@ func Client(cfg tlsconf.Config, timeout time.Duration) (*http.Client, error) {
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &http.Client{Transport: &http2.Transport{TLSClientConfig: tc}}, nil
|
||||
return &http.Client{Transport: &http2.Transport{
|
||||
TLSClientConfig: tc,
|
||||
ReadIdleTimeout: streamIdle,
|
||||
PingTimeout: streamPing,
|
||||
// Bounded, as the plaintext branch's is: dialling a control plane
|
||||
// whose machine is off waits out the kernel's connect retries,
|
||||
// about two minutes, before the next address is tried.
|
||||
DialTLSContext: func(ctx context.Context, network, addr string, c *tls.Config) (net.Conn, error) {
|
||||
d := tls.Dialer{NetDialer: &net.Dialer{Timeout: timeout}, Config: c}
|
||||
return d.DialContext(ctx, network, addr)
|
||||
},
|
||||
}}, nil
|
||||
}
|
||||
|
||||
// A stream to a control plane whose machine stopped dead — powered off, cut
|
||||
// off — is never closed: nothing sends the FIN, and the agent's heartbeats sit
|
||||
// in the socket's buffer until the kernel gives up retransmitting, fifteen
|
||||
// minutes later. Found by switching off a leader: the standby took over in 14
|
||||
// seconds and the node stayed with the dead one. An HTTP/2 ping after ten
|
||||
// quiet seconds, given five to answer, finds it in a quarter of a minute.
|
||||
const (
|
||||
streamIdle = 10 * time.Second
|
||||
streamPing = 5 * time.Second
|
||||
)
|
||||
|
||||
// localizeSpec fills in what only this node knows.
|
||||
//
|
||||
// The interface NCCL bootstraps over is this node's, not the cluster's: the
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
)
|
||||
|
||||
// controlPlanesFile keeps the control planes the leader last named, beside
|
||||
// the agent's configuration rather than in it: the configuration is the
|
||||
// operator's, and a list rewritten in it would be a file nobody wrote.
|
||||
var controlPlanesFile = "/var/lib/llmgw-agent/control-planes.json"
|
||||
|
||||
// servers is every address to dial, in the order to try them: the
|
||||
// configuration's, then any the leader named that it lacks.
|
||||
func (a *Agent) servers() []string {
|
||||
a.mu.Lock()
|
||||
defer a.mu.Unlock()
|
||||
if len(a.serverList) == 0 {
|
||||
return a.cfg.Servers
|
||||
}
|
||||
return a.serverList
|
||||
}
|
||||
|
||||
// mergeServers puts the leader's list after the configured one, without
|
||||
// repeats, and says whether that changed anything.
|
||||
func mergeServers(configured, told []string, secure bool) []string {
|
||||
out := slices.Clone(configured)
|
||||
for _, s := range told {
|
||||
fixed, err := normalizeOne(s, secure)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !slices.Contains(out, fixed) {
|
||||
out = append(out, fixed)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// setControlPlanes takes the leader's list: used from the next reconnect,
|
||||
// and kept for the next start.
|
||||
func (a *Agent) setControlPlanes(m *llmgwv1.ControlPlanes) {
|
||||
told := m.GetServers()
|
||||
merged := mergeServers(a.cfg.Servers, told, a.cfg.TLS.Enabled())
|
||||
a.mu.Lock()
|
||||
changed := !slices.Equal(merged, a.serverList)
|
||||
a.serverList = merged
|
||||
a.mu.Unlock()
|
||||
if !changed {
|
||||
return
|
||||
}
|
||||
a.log.Info("the control planes to dial", "servers", merged)
|
||||
b, _ := json.Marshal(told)
|
||||
if err := os.MkdirAll(filepath.Dir(controlPlanesFile), 0o755); err == nil {
|
||||
tmp := controlPlanesFile + ".new"
|
||||
if err := os.WriteFile(tmp, b, 0o644); err == nil {
|
||||
_ = os.Rename(tmp, controlPlanesFile)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// loadControlPlanes is the list kept from the last time the leader named it.
|
||||
func loadControlPlanes(configured []string, secure bool) []string {
|
||||
b, err := os.ReadFile(controlPlanesFile)
|
||||
if err != nil {
|
||||
return configured
|
||||
}
|
||||
var told []string
|
||||
if json.Unmarshal(b, &told) != nil {
|
||||
return configured
|
||||
}
|
||||
return mergeServers(configured, told, secure)
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
func TestAControlPlaneTheLeaderNamesIsDialledAndRemembered(t *testing.T) {
|
||||
old := controlPlanesFile
|
||||
controlPlanesFile = filepath.Join(t.TempDir(), "control-planes.json")
|
||||
t.Cleanup(func() { controlPlanesFile = old })
|
||||
|
||||
configured := []string{"https://10.0.0.140:7443"}
|
||||
// TLS on, as every onboarded node has it: an agent without it refuses
|
||||
// https:// addresses, and the leader's are https://.
|
||||
a := &Agent{cfg: Config{Servers: configured, TLS: tlsconf.Config{CA: "/etc/llmgw/ca.pem"}}, log: slog.New(slog.DiscardHandler)}
|
||||
if got := a.servers(); !slices.Equal(got, configured) {
|
||||
t.Fatalf("before anything is named: %v", got)
|
||||
}
|
||||
// The configured one stays first; the new one is added after it, once.
|
||||
a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}})
|
||||
want := []string{"https://10.0.0.140:7443", "https://10.0.0.150:7443"}
|
||||
if got := a.servers(); !slices.Equal(got, want) {
|
||||
t.Errorf("after the leader named two: %v", got)
|
||||
}
|
||||
// And after a restart, before the leader says anything.
|
||||
if got := loadControlPlanes(configured, true); !slices.Equal(got, want) {
|
||||
t.Errorf("after a restart: %v", got)
|
||||
}
|
||||
// A list that drops the configured one does not drop it: the
|
||||
// configuration is the operator's.
|
||||
a.setControlPlanes(&llmgwv1.ControlPlanes{Servers: []string{"https://10.0.0.150:7443"}})
|
||||
if got := a.servers(); got[0] != "https://10.0.0.140:7443" {
|
||||
t.Errorf("the configured server was dropped: %v", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user