Files
llmgw/internal/cli/controlplanes.go
T
Ilker ManapandClaude Opus 5.5 ec1918b23b Two control planes: a failover in seconds, and the console to set one up
A control plane being stopped held every agent's stream for the thirty
seconds of a graceful shutdown, so the standby led within four seconds and
had no nodes for thirty. The agent server is now closed, not drained. A
control plane that loses its lease while it lives on lets its agents go,
rather than keeping nodes the new leader cannot reach.

ctl status and the console's Overview say which control plane leads, which
stand by, and whether the floating address is on the leader. The console
adds a control plane (Do → Add a control plane), asking for the machine and
the floating address. server add says when the mirror would refuse the new
control plane.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-05 02:17:51 +02:00

134 lines
3.8 KiB
Go

package cli
import (
"bufio"
"context"
"net"
"net/http"
"net/url"
"os"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// keepalivedPath is where `server add --vip` writes the floating address.
const keepalivedPath = "/etc/keepalived/keepalived.conf"
// controlPlaneProbe answers the summary's question of which control plane
// leads: it asks each one's /leader, on the client port every one of them
// listens on, since they share a server.yaml.
//
// Asked, rather than read from the lease, because the lease names whoever
// held it last and says nothing about a standby that has stopped — and a
// standby that is not there is the thing worth seeing before it is needed.
type controlPlaneProbe struct {
st *store.DB
self string
floating string
client *http.Client
// leaderURL is where a host is asked; tests point it elsewhere.
leaderURL func(host string) string
mu sync.Mutex
at time.Time
last adminapi.ControlPlanes
}
func newControlPlaneProbe(st *store.DB, listen string) *controlPlaneProbe {
_, port, _ := net.SplitHostPort(listen)
if port == "" {
port = "8080"
}
self, _ := os.Hostname()
return &controlPlaneProbe{st: st, self: self,
leaderURL: func(host string) string { return "http://" + net.JoinHostPort(host, port) + "/leader" },
floating: floatingAddress(keepalivedPath),
client: &http.Client{Timeout: time.Second}}
}
// Status is the answer, kept for a few seconds: the console asks every three,
// and each asking is a request to every control plane.
func (p *controlPlaneProbe) Status() adminapi.ControlPlanes {
p.mu.Lock()
defer p.mu.Unlock()
if time.Since(p.at) < 2*time.Second {
return p.last
}
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
defer cancel()
cps, err := p.st.ControlPlanes(ctx)
if err != nil || len(cps) < 2 {
// One control plane is not a set of them; nothing to say.
p.at, p.last = time.Now(), adminapi.ControlPlanes{}
return p.last
}
out := adminapi.ControlPlanes{Members: make([]adminapi.ControlPlane, len(cps)), Floating: p.floating}
var wg sync.WaitGroup
for i, cp := range cps {
host := cp.URL
if u, err := url.Parse(cp.URL); err == nil && u.Hostname() != "" {
host = u.Hostname()
}
out.Members[i] = adminapi.ControlPlane{Name: cp.Name, Address: host, This: cp.Name == p.self}
wg.Go(func() { out.Members[i].State = p.ask(ctx, host) })
}
if p.floating != "" {
wg.Go(func() { out.FloatingUp = p.ask(ctx, p.floating) == "leading" })
}
wg.Wait()
p.at, p.last = time.Now(), out
return out
}
// ask is what one address's /leader says.
func (p *controlPlaneProbe) ask(ctx context.Context, host string) string {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.leaderURL(host), nil)
if err != nil {
return "not answering"
}
resp, err := p.client.Do(req)
if err != nil {
return "not answering"
}
_ = resp.Body.Close()
switch resp.StatusCode {
case http.StatusOK:
return "leading"
case http.StatusServiceUnavailable:
return "standing by"
}
return "not answering"
}
// floatingAddress is the address in the keepalived configuration server add
// wrote, without its prefix; "" when there is none, or the file is someone
// else's.
func floatingAddress(path string) string {
f, err := os.Open(path) //nolint:gosec // a fixed system path
if err != nil {
return ""
}
defer f.Close()
sc := bufio.NewScanner(f)
ours, in := false, false
for sc.Scan() {
line := strings.TrimSpace(sc.Text())
switch {
case strings.HasPrefix(line, "# llmgw:"):
ours = true
case strings.HasPrefix(line, "virtual_ipaddress"):
in = true
case in && line == "}":
return ""
case in && ours && line != "":
addr, _, _ := strings.Cut(strings.Fields(line)[0], "/")
return addr
}
}
return ""
}