A control plane being stopped held every agent's stream for the thirty seconds of a graceful shutdown, so the standby led within four seconds and had no nodes for thirty. The agent server is now closed, not drained. A control plane that loses its lease while it lives on lets its agents go, rather than keeping nodes the new leader cannot reach. ctl status and the console's Overview say which control plane leads, which stand by, and whether the floating address is on the leader. The console adds a control plane (Do → Add a control plane), asking for the machine and the floating address. server add says when the mirror would refuse the new control plane. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
134 lines
3.8 KiB
Go
134 lines
3.8 KiB
Go
package cli
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"os"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
// keepalivedPath is where `server add --vip` writes the floating address.
|
|
const keepalivedPath = "/etc/keepalived/keepalived.conf"
|
|
|
|
// controlPlaneProbe answers the summary's question of which control plane
|
|
// leads: it asks each one's /leader, on the client port every one of them
|
|
// listens on, since they share a server.yaml.
|
|
//
|
|
// Asked, rather than read from the lease, because the lease names whoever
|
|
// held it last and says nothing about a standby that has stopped — and a
|
|
// standby that is not there is the thing worth seeing before it is needed.
|
|
type controlPlaneProbe struct {
|
|
st *store.DB
|
|
self string
|
|
floating string
|
|
client *http.Client
|
|
// leaderURL is where a host is asked; tests point it elsewhere.
|
|
leaderURL func(host string) string
|
|
|
|
mu sync.Mutex
|
|
at time.Time
|
|
last adminapi.ControlPlanes
|
|
}
|
|
|
|
func newControlPlaneProbe(st *store.DB, listen string) *controlPlaneProbe {
|
|
_, port, _ := net.SplitHostPort(listen)
|
|
if port == "" {
|
|
port = "8080"
|
|
}
|
|
self, _ := os.Hostname()
|
|
return &controlPlaneProbe{st: st, self: self,
|
|
leaderURL: func(host string) string { return "http://" + net.JoinHostPort(host, port) + "/leader" },
|
|
floating: floatingAddress(keepalivedPath),
|
|
client: &http.Client{Timeout: time.Second}}
|
|
}
|
|
|
|
// Status is the answer, kept for a few seconds: the console asks every three,
|
|
// and each asking is a request to every control plane.
|
|
func (p *controlPlaneProbe) Status() adminapi.ControlPlanes {
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
if time.Since(p.at) < 2*time.Second {
|
|
return p.last
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
|
defer cancel()
|
|
cps, err := p.st.ControlPlanes(ctx)
|
|
if err != nil || len(cps) < 2 {
|
|
// One control plane is not a set of them; nothing to say.
|
|
p.at, p.last = time.Now(), adminapi.ControlPlanes{}
|
|
return p.last
|
|
}
|
|
out := adminapi.ControlPlanes{Members: make([]adminapi.ControlPlane, len(cps)), Floating: p.floating}
|
|
var wg sync.WaitGroup
|
|
for i, cp := range cps {
|
|
host := cp.URL
|
|
if u, err := url.Parse(cp.URL); err == nil && u.Hostname() != "" {
|
|
host = u.Hostname()
|
|
}
|
|
out.Members[i] = adminapi.ControlPlane{Name: cp.Name, Address: host, This: cp.Name == p.self}
|
|
wg.Go(func() { out.Members[i].State = p.ask(ctx, host) })
|
|
}
|
|
if p.floating != "" {
|
|
wg.Go(func() { out.FloatingUp = p.ask(ctx, p.floating) == "leading" })
|
|
}
|
|
wg.Wait()
|
|
p.at, p.last = time.Now(), out
|
|
return out
|
|
}
|
|
|
|
// ask is what one address's /leader says.
|
|
func (p *controlPlaneProbe) ask(ctx context.Context, host string) string {
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, p.leaderURL(host), nil)
|
|
if err != nil {
|
|
return "not answering"
|
|
}
|
|
resp, err := p.client.Do(req)
|
|
if err != nil {
|
|
return "not answering"
|
|
}
|
|
_ = resp.Body.Close()
|
|
switch resp.StatusCode {
|
|
case http.StatusOK:
|
|
return "leading"
|
|
case http.StatusServiceUnavailable:
|
|
return "standing by"
|
|
}
|
|
return "not answering"
|
|
}
|
|
|
|
// floatingAddress is the address in the keepalived configuration server add
|
|
// wrote, without its prefix; "" when there is none, or the file is someone
|
|
// else's.
|
|
func floatingAddress(path string) string {
|
|
f, err := os.Open(path) //nolint:gosec // a fixed system path
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
defer f.Close()
|
|
sc := bufio.NewScanner(f)
|
|
ours, in := false, false
|
|
for sc.Scan() {
|
|
line := strings.TrimSpace(sc.Text())
|
|
switch {
|
|
case strings.HasPrefix(line, "# llmgw:"):
|
|
ours = true
|
|
case strings.HasPrefix(line, "virtual_ipaddress"):
|
|
in = true
|
|
case in && line == "}":
|
|
return ""
|
|
case in && ours && line != "":
|
|
addr, _, _ := strings.Cut(strings.Fields(line)[0], "/")
|
|
return addr
|
|
}
|
|
}
|
|
return ""
|
|
}
|