A standby's admin API is the leader's
The console and ctl on a standby showed its own empty view: nodes and replicas live in the leader's memory. A standby now passes /api/ to the leader over the agent port, with the cluster's certificates both ways; the leader answers only a certificate naming a control plane the store lists, and only while it leads. Control plane certificates are issued for client use as well, and server add reissues older ones. An agent's reconnect backoff starts again after a session that worked, so each failover it lives through no longer makes the next one slower. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
adef25d709
commit
1d1409faad
12 files changed
+510
-13
No files matched your search
@@ -1940,6 +1940,14 @@ Without it, put a load balancer in front that checks `/ready` — not
|
||||
`CONTROL` line: which leads, which stand by, one that does not answer, and
|
||||
whether the floating address is on the leader. A standby with nothing to
|
||||
take over from is the thing to see before it is needed.
|
||||
- **Either machine's console.** A standby's admin API passes everything under
|
||||
`/api/` to the leader, over the port control planes already reach each
|
||||
other on and with the cluster's certificates both ways, so `sudo llmgw` and
|
||||
`llmgw ctl` show the same cluster on both — the leader's — and a change made
|
||||
on the standby is made by the leader. A node's certificate is refused
|
||||
there. Control plane certificates from before v0.11.3 cannot present
|
||||
themselves as clients: `server add` reissues them, and the running server
|
||||
picks the new one up when restarted.
|
||||
- **The mirror** answers only the addresses it was given with `--api-allow`.
|
||||
`server add` checks, and says what to add to the mirror's unit if it
|
||||
refuses the new control plane — which would otherwise find out the first
|
||||
|
||||
+18
-5
@@ -21,6 +21,7 @@ import (
|
||||
"slices"
|
||||
"strconv"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
@@ -42,11 +43,14 @@ import (
|
||||
|
||||
// Agent runs on a GPU node.
|
||||
type Agent struct {
|
||||
cfg Config
|
||||
log *slog.Logger
|
||||
sup *supervisor.Supervisor
|
||||
reg *engines.Registry
|
||||
http connectHTTPClient
|
||||
cfg Config
|
||||
// answered says the current session heard from its control plane, which
|
||||
// starts the reconnect backoff again when it ends.
|
||||
answered atomic.Bool
|
||||
log *slog.Logger
|
||||
sup *supervisor.Supervisor
|
||||
reg *engines.Registry
|
||||
http connectHTTPClient
|
||||
|
||||
// engineOnce probes the configured engines once before registering.
|
||||
//
|
||||
@@ -215,6 +219,14 @@ func (a *Agent) Run(ctx context.Context) error {
|
||||
// never answers — from here they are the same thing, and trying
|
||||
// somewhere else is the answer to all three.
|
||||
next++
|
||||
// A session that was answered was a connection that worked, however
|
||||
// it ended: the backoff starts again rather than carrying on from
|
||||
// the last outage. Without this every failover the agent lived
|
||||
// through made the next one slower — the third waited half a minute
|
||||
// on a control plane that had been ready for twenty seconds.
|
||||
if a.answered.Swap(false) {
|
||||
attempt = 0
|
||||
}
|
||||
attempt++
|
||||
delay := a.reconnectDelay(attempt)
|
||||
// No wait when there is somewhere else to try: the delay exists to
|
||||
@@ -568,6 +580,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
// why it is not registering is one an operator cannot fix.
|
||||
return err
|
||||
}
|
||||
a.answered.Store(true)
|
||||
a.handle(ctx, msg, out)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
// forwardPrefix is where the leader answers a standby's admin API, on the
|
||||
// agent port: the one port control planes already reach each other on, with
|
||||
// the cluster's certificates both ways. The admin port itself listens on
|
||||
// loopback, and should — it serves pprof.
|
||||
const forwardPrefix = "/_admin"
|
||||
|
||||
// Headers a forwarded request and its answer carry: which standby asked, and
|
||||
// which control plane answered.
|
||||
const (
|
||||
hdrForwardedBy = "X-Llmgw-Forwarded-By"
|
||||
hdrAnsweredBy = "X-Llmgw-Answered-By"
|
||||
)
|
||||
|
||||
// leaderForward sends a standby's admin API to the leader.
|
||||
//
|
||||
// Everything the console and ctl show is the leader's: nodes, replicas and
|
||||
// jobs live in its memory, put there by the agent streams only it holds. A
|
||||
// standby answering from its own would show an empty cluster, and accept
|
||||
// changes it cannot carry out. So a standby answers nothing under /api/
|
||||
// itself; the leader does, and the console looks the same on either machine.
|
||||
type leaderForward struct {
|
||||
st *store.DB
|
||||
lead interface{ IsLeader() bool }
|
||||
self string
|
||||
tls tlsconf.Config
|
||||
log *slog.Logger
|
||||
}
|
||||
|
||||
// standby wraps the admin handler: /api/ goes to the leader while this
|
||||
// process is not it.
|
||||
func (f *leaderForward) standby(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if !strings.HasPrefix(r.URL.Path, "/api/") || f.lead.IsLeader() {
|
||||
next.ServeHTTP(w, r)
|
||||
return
|
||||
}
|
||||
target, name, err := f.leader(r.Context())
|
||||
if err != nil {
|
||||
writeForwardError(w, http.StatusServiceUnavailable,
|
||||
"this control plane is standing by, and cannot reach the leader: "+err.Error())
|
||||
return
|
||||
}
|
||||
tc, err := f.tls.ClientConfig(target.Hostname())
|
||||
if err != nil {
|
||||
writeForwardError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
p := &httputil.ReverseProxy{
|
||||
Rewrite: func(pr *httputil.ProxyRequest) {
|
||||
pr.Out.URL.Scheme, pr.Out.URL.Host = "https", target.Host
|
||||
pr.Out.URL.Path = forwardPrefix + pr.In.URL.Path
|
||||
pr.Out.URL.RawPath = ""
|
||||
pr.Out.Host = target.Host
|
||||
// The leader's own token is not this one's, and the
|
||||
// certificate is what it believes.
|
||||
pr.Out.Header.Del("Authorization")
|
||||
pr.Out.Header.Set(hdrForwardedBy, f.self)
|
||||
},
|
||||
Transport: &http2.Transport{TLSClientConfig: tc},
|
||||
// Streams, such as a job's progress, pass through as they come.
|
||||
FlushInterval: -1,
|
||||
ErrorHandler: func(w http.ResponseWriter, _ *http.Request, err error) {
|
||||
writeForwardError(w, http.StatusBadGateway, fmt.Sprintf("the leader, %s, did not answer: %v", name, err))
|
||||
},
|
||||
}
|
||||
p.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// leader is where the lease's holder is reached, and its name.
|
||||
func (f *leaderForward) leader(ctx context.Context) (*url.URL, string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 3*time.Second)
|
||||
defer cancel()
|
||||
lease, err := f.st.Leadership(ctx, control.ControlRole)
|
||||
if err != nil || lease.Owner == "" || time.Now().After(lease.ExpiresAt) {
|
||||
return nil, "", errors.New("no control plane holds the lease right now")
|
||||
}
|
||||
name := control.HolderName(lease.Owner)
|
||||
cps, err := f.st.ControlPlanes(ctx)
|
||||
if err != nil {
|
||||
return nil, name, err
|
||||
}
|
||||
for _, cp := range cps {
|
||||
if cp.Name == name {
|
||||
u, err := url.Parse(cp.URL)
|
||||
return u, name, err
|
||||
}
|
||||
}
|
||||
return nil, name, fmt.Errorf("%s leads, and is not among the control planes the store lists", name)
|
||||
}
|
||||
|
||||
// leaderSide answers forwarded requests, on the agent port: only from another
|
||||
// control plane of this cluster, only while this one leads, and only the API.
|
||||
func (f *leaderForward) leaderSide(admin http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
path := strings.TrimPrefix(r.URL.Path, forwardPrefix)
|
||||
if !strings.HasPrefix(path, "/api/") {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
if err := f.fromControlPlane(r); err != nil {
|
||||
writeForwardError(w, http.StatusForbidden, err.Error())
|
||||
return
|
||||
}
|
||||
if !f.lead.IsLeader() {
|
||||
// Not passed on again: two standbys must not bounce a request
|
||||
// between them while the lease moves.
|
||||
writeForwardError(w, http.StatusServiceUnavailable, "this control plane no longer leads; ask again")
|
||||
return
|
||||
}
|
||||
r2 := r.Clone(r.Context())
|
||||
r2.URL.Path, r2.URL.RawPath = path, ""
|
||||
w.Header().Set(hdrAnsweredBy, f.self)
|
||||
admin.ServeHTTP(w, r2)
|
||||
})
|
||||
}
|
||||
|
||||
// fromControlPlane checks the client certificate names a control plane the
|
||||
// store lists. A node's certificate is from the same authority, and a node
|
||||
// must not reach the admin API by presenting it.
|
||||
func (f *leaderForward) fromControlPlane(r *http.Request) error {
|
||||
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 {
|
||||
return errors.New("a control plane's certificate is required")
|
||||
}
|
||||
cert := r.TLS.VerifiedChains[0][0]
|
||||
if !slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageServerAuth) {
|
||||
return errors.New("not a control plane's certificate")
|
||||
}
|
||||
cps, err := f.st.ControlPlanes(r.Context())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, cp := range cps {
|
||||
if cp.Name == cert.Subject.CommonName {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("%s is not a control plane of this cluster", cert.Subject.CommonName)
|
||||
}
|
||||
|
||||
func writeForwardError(w http.ResponseWriter, code int, msg string) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
w.WriteHeader(code)
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"error": msg})
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
type leading struct{ v atomic.Bool }
|
||||
|
||||
func (l *leading) IsLeader() bool { return l.v.Load() }
|
||||
|
||||
// forwardCluster is two control planes of one cluster on one store: a
|
||||
// leader whose agent port answers forwarded admin requests, and a standby.
|
||||
type forwardCluster struct {
|
||||
st *store.DB
|
||||
dir string
|
||||
leaderLeads *leading
|
||||
standby, leader *leaderForward
|
||||
leaderURL string
|
||||
}
|
||||
|
||||
func newForwardCluster(t *testing.T) *forwardCluster {
|
||||
t.Helper()
|
||||
dir := t.TempDir()
|
||||
ca, err := tlsconf.NewCA("test CA")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := ca.Write(dir); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, name := range []string{"ctl-a", "ctl-b"} {
|
||||
leaf, err := ca.IssuePeer(name, []string{"127.0.0.1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := leaf.Write(dir, name); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
node, err := ca.IssueClient("node1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := node.Write(dir, "node1"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfgFor := func(name string) tlsconf.Config {
|
||||
return tlsconf.Config{CA: filepath.Join(dir, "ca.pem"), Cert: filepath.Join(dir, name+".pem"), Key: filepath.Join(dir, name+"-key.pem")}
|
||||
}
|
||||
|
||||
st, err := store.Open(t.Context(), filepath.Join(dir, "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
t.Cleanup(func() { st.Close() })
|
||||
|
||||
c := &forwardCluster{st: st, dir: dir, leaderLeads: &leading{}}
|
||||
c.leaderLeads.v.Store(true)
|
||||
log := slog.New(slog.DiscardHandler)
|
||||
c.leader = &leaderForward{st: st, lead: c.leaderLeads, self: "ctl-a", tls: cfgFor("ctl-a"), log: log}
|
||||
standbyLeads := &leading{}
|
||||
c.standby = &leaderForward{st: st, lead: standbyLeads, self: "ctl-b", tls: cfgFor("ctl-b"), log: log}
|
||||
|
||||
// The leader's admin API: says whose it is, and who it was asked by.
|
||||
admin := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"path": r.URL.Path, "by": r.Header.Get(hdrForwardedBy),
|
||||
"auth": r.Header.Get("Authorization"), "actor": r.Header.Get("X-Llmgw-Actor")})
|
||||
})
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(forwardPrefix+"/", c.leader.leaderSide(admin))
|
||||
srv := httptest.NewUnstartedServer(mux)
|
||||
tc, err := cfgFor("ctl-a").ServerConfig()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
srv.TLS = tc
|
||||
srv.EnableHTTP2 = true
|
||||
srv.StartTLS()
|
||||
t.Cleanup(srv.Close)
|
||||
c.leaderURL = srv.URL
|
||||
|
||||
for _, cp := range []store.ControlPlane{{URL: srv.URL, Name: "ctl-a"}, {URL: "https://127.0.0.1:1", Name: "ctl-b"}} {
|
||||
if err := st.PutControlPlane(t.Context(), cp); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
if _, err := st.AcquireLeadership(t.Context(), control.ControlRole, "ctl-a-0123456789abcdef", time.Minute); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
// ask sends one request through the standby's admin API.
|
||||
func (c *forwardCluster) ask(t *testing.T, path string) (*http.Response, map[string]string) {
|
||||
t.Helper()
|
||||
local := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(map[string]string{"path": "local"})
|
||||
})
|
||||
srv := httptest.NewServer(c.standby.standby(local))
|
||||
defer srv.Close()
|
||||
req, _ := http.NewRequestWithContext(t.Context(), http.MethodGet, srv.URL+path, nil)
|
||||
req.Header.Set("Authorization", "Bearer the-standbys-own-token")
|
||||
req.Header.Set("X-Llmgw-Actor", "root@ctl-b")
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(resp.Body)
|
||||
var got map[string]string
|
||||
_ = json.Unmarshal(b, &got)
|
||||
return resp, got
|
||||
}
|
||||
|
||||
func TestAStandbysAdminAPIIsTheLeaders(t *testing.T) {
|
||||
c := newForwardCluster(t)
|
||||
resp, got := c.ask(t, "/api/v1/summary")
|
||||
if resp.StatusCode != http.StatusOK || got["path"] != "/api/v1/summary" {
|
||||
t.Fatalf("%d %v", resp.StatusCode, got)
|
||||
}
|
||||
if resp.Header.Get(hdrAnsweredBy) != "ctl-a" || got["by"] != "ctl-b" {
|
||||
t.Errorf("answered by %q, asked by %q", resp.Header.Get(hdrAnsweredBy), got["by"])
|
||||
}
|
||||
if got["auth"] != "" {
|
||||
t.Error("the standby's token was passed on to the leader")
|
||||
}
|
||||
if got["actor"] != "root@ctl-b" {
|
||||
t.Errorf("who asked was lost: %q", got["actor"])
|
||||
}
|
||||
// What is not the API stays on the machine: the dashboard, pprof.
|
||||
if _, got := c.ask(t, "/debug/pprof/"); got["path"] != "local" {
|
||||
t.Errorf("a non-API path was forwarded: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLeaderAnswersOnlyControlPlanesAndOnlyWhileItLeads(t *testing.T) {
|
||||
c := newForwardCluster(t)
|
||||
call := func(cert string) int {
|
||||
cfg := tlsconf.Config{CA: filepath.Join(c.dir, "ca.pem"), Cert: filepath.Join(c.dir, cert+".pem"), Key: filepath.Join(c.dir, cert+"-key.pem")}
|
||||
tc, err := cfg.ClientConfig("127.0.0.1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tc.NextProtos = nil
|
||||
client := &http.Client{Transport: &http.Transport{TLSClientConfig: tc}}
|
||||
resp, err := client.Get(c.leaderURL + forwardPrefix + "/api/v1/summary")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp.Body.Close()
|
||||
return resp.StatusCode
|
||||
}
|
||||
if code := call("ctl-b"); code != http.StatusOK {
|
||||
t.Errorf("a control plane was refused: %d", code)
|
||||
}
|
||||
// A node's certificate is from the same authority, and is no way in.
|
||||
if code := call("node1"); code != http.StatusForbidden {
|
||||
t.Errorf("a node reached the admin API: %d", code)
|
||||
}
|
||||
c.leaderLeads.v.Store(false)
|
||||
if code := call("ctl-b"); code != http.StatusServiceUnavailable {
|
||||
t.Errorf("a control plane that no longer leads answered: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStandbyWithNoLeaderSaysSo(t *testing.T) {
|
||||
c := newForwardCluster(t)
|
||||
if err := c.st.ReleaseLeadership(t.Context(), control.ControlRole, "ctl-a-0123456789abcdef"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, got := c.ask(t, "/api/v1/summary")
|
||||
if resp.StatusCode != http.StatusServiceUnavailable || !strings.Contains(got["error"], "standing by") {
|
||||
t.Errorf("%d %v", resp.StatusCode, got)
|
||||
}
|
||||
}
|
||||
+16
-1
@@ -502,8 +502,20 @@ func newServerCmd() *cobra.Command {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
adminHandler := auditAPI(rt, adminMux, log)
|
||||
// With a rival, a standby's admin API is the leader's: what the
|
||||
// console and ctl show lives in the leader's memory.
|
||||
var forward *leaderForward
|
||||
if admin != nil && admin.Leadership() != nil && rt.Store() != nil && cfg.AgentTLS.Enabled() {
|
||||
self, _ := os.Hostname()
|
||||
forward = &leaderForward{st: rt.Store(), lead: admin.Leadership(), self: self, tls: cfg.AgentTLS, log: log}
|
||||
}
|
||||
served := adminHandler
|
||||
if forward != nil {
|
||||
served = forward.standby(adminHandler)
|
||||
}
|
||||
adminSrv := withLimits(&http.Server{
|
||||
Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, auditAPI(rt, adminMux, log))),
|
||||
Handler: router.AdminGuard(router.AdminAuth(adminToken, cfg.DebugPprof, served)),
|
||||
ReadTimeout: time.Minute,
|
||||
})
|
||||
|
||||
@@ -569,6 +581,9 @@ func newServerCmd() *cobra.Command {
|
||||
if agentHandler != nil {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle(agentPath, agentHandler)
|
||||
if forward != nil {
|
||||
mux.Handle(forwardPrefix+"/", router.AdminGuard(forward.leaderSide(adminHandler)))
|
||||
}
|
||||
// No read timeout: an agent's stream is one request that
|
||||
// lasts as long as the agent does.
|
||||
agentSrv = withLimits(&http.Server{Handler: mux})
|
||||
|
||||
@@ -121,6 +121,7 @@ func (sa *serverAdd) run(ctx context.Context) error {
|
||||
{"connect", sa.connect},
|
||||
{"binary", sa.binary},
|
||||
{"directories", sa.dirs},
|
||||
{"this certificate", sa.ownCertificate},
|
||||
{"certificate", sa.certificate},
|
||||
{"configuration", sa.configuration},
|
||||
{"store", sa.storeAccess},
|
||||
@@ -231,6 +232,40 @@ sudo install -d -o llmgw -g llmgw -m 0750 /var/lib/llmgw`, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
// ownCertificate makes this control plane's certificate one it can also
|
||||
// present as a client — a standby asks the leader for the console's data
|
||||
// with it — reissuing one from before that with the same names. The running
|
||||
// server reads it as it starts.
|
||||
func (sa *serverAdd) ownCertificate(context.Context) error {
|
||||
path := filepath.Join(sa.etcDir, "server.pem")
|
||||
cert, err := readCert(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if clientCapable(cert) {
|
||||
sa.kept("this control plane's certificate")
|
||||
return nil
|
||||
}
|
||||
ca, err := loadCA(sa.pkiDir)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the authority's key is not in %s: run this where it is (%v)", sa.pkiDir, err)
|
||||
}
|
||||
hosts := strings.Split(sanNames(cert), ", ")
|
||||
leaf, err := ca.IssuePeer(cert.Subject.CommonName, hosts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(path, leaf.CertPEM, 0o644); err != nil { //nolint:gosec // a certificate
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(sa.etcDir, "server-key.pem"), leaf.KeyPEM, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
sa.done("this control plane's certificate reissued for %s, to ask the leader as well as answer;\n"+
|
||||
" restart llmgw-server here for it to take effect — %s takes over meanwhile", strings.Join(hosts, ", "), sa.name)
|
||||
return nil
|
||||
}
|
||||
|
||||
// certificate issues it a server certificate from this cluster's authority,
|
||||
// for its name and addresses, unless it holds one that already names them.
|
||||
func (sa *serverAdd) certificate(ctx context.Context) error {
|
||||
@@ -255,14 +290,14 @@ func (sa *serverAdd) certificate(ctx context.Context) error {
|
||||
for _, w := range want {
|
||||
missing = missing || !slices.Contains(names, w)
|
||||
}
|
||||
if !missing {
|
||||
if !missing && clientCapable(cert) {
|
||||
sa.kept("certificate for %s", strings.Join(names, ", "))
|
||||
return sa.put(ctx, "/etc/llmgw/ca.pem", "0644", "root:root", ca.PEM)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
leaf, err := ca.IssueServer(sa.name, want)
|
||||
leaf, err := ca.IssuePeer(sa.name, want)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -545,3 +580,9 @@ vrrp_instance llmgw {
|
||||
}
|
||||
`, iface, priority, pass, vip, prefix)
|
||||
}
|
||||
|
||||
// clientCapable says a control plane's certificate can also present itself
|
||||
// as a client, as a standby does to the leader.
|
||||
func clientCapable(cert *x509.Certificate) bool {
|
||||
return slices.Contains(cert.ExtKeyUsage, x509.ExtKeyUsageClientAuth)
|
||||
}
|
||||
@@ -462,7 +462,10 @@ func (in *serverInit) ensureCerts(context.Context) error {
|
||||
// A certificate from this authority naming every address is left
|
||||
// alone. One naming fewer is reissued with the union, so an address
|
||||
// that worked yesterday is never dropped by adding one today.
|
||||
if len(missing) == 0 && cert.CheckSignatureFrom(ca.Cert) == nil {
|
||||
// And one that can also present itself as a client, which a
|
||||
// standby's admin API needs to reach the leader's; certificates
|
||||
// from before that are reissued.
|
||||
if len(missing) == 0 && cert.CheckSignatureFrom(ca.Cert) == nil && clientCapable(cert) {
|
||||
in.kept("server certificate for %s", strings.Join(have, ", "))
|
||||
return nil
|
||||
}
|
||||
@@ -477,7 +480,7 @@ func (in *serverInit) ensureCerts(context.Context) error {
|
||||
if name == "" {
|
||||
name = "llmgw-control"
|
||||
}
|
||||
leaf, err := ca.IssueServer(name, want)
|
||||
leaf, err := ca.IssuePeer(name, want)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -1065,7 +1065,7 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
c := &cobra.Command{}
|
||||
c.SetOut(&b)
|
||||
printSummary(c, m.data.sum)
|
||||
return "Overview", "\n" + indent(b.String(), " "), keys
|
||||
return "Overview", m.standbyNote() + "\n" + indent(b.String(), " "), keys
|
||||
case cNodes, cLogs:
|
||||
if m.detail && m.row < len(m.data.nodes) {
|
||||
n := m.data.nodes[m.row]
|
||||
@@ -1569,3 +1569,21 @@ func runConsole(ctx context.Context, out io.Writer, file string) error {
|
||||
const floatingQuestion = "A floating address for clients, kept on whichever control plane leads,\n" +
|
||||
"as 10.0.0.161/24 — a free one on this network. Empty for none, when a load\n" +
|
||||
"balancer in front checks /ready instead."
|
||||
|
||||
// standbyNote says, on a control plane standing by, whose cluster the
|
||||
// console shows: the leader's, which this one's admin API passes it on to.
|
||||
func (m consoleModel) standbyNote() string {
|
||||
var here, leader string
|
||||
for _, cp := range m.data.sum.ControlPlanes.Members {
|
||||
if cp.This {
|
||||
here = cp.State
|
||||
}
|
||||
if cp.State == "leading" {
|
||||
leader = cp.Name
|
||||
}
|
||||
}
|
||||
if here != "standing by" || leader == "" {
|
||||
return ""
|
||||
}
|
||||
return "\n " + sDim.Render("Standing by. Shown, and changed, on the leader: "+leader) + "\n"
|
||||
}
|
||||
@@ -203,3 +203,18 @@ func TestTheConsoleAddsAControlPlaneWithAFloatingAddress(t *testing.T) {
|
||||
t.Errorf("dialog %q: %s", c.dialog, c.dialogText)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheConsoleOnAStandbySaysWhoseClusterItShows(t *testing.T) {
|
||||
m := consoleModel{menu: cOverview, loaded: true, width: 120, height: 40}
|
||||
m.data.sum.ControlPlanes = adminapi.ControlPlanes{Members: []adminapi.ControlPlane{
|
||||
{Name: "llmgw-ctl", State: "leading"},
|
||||
{Name: "llmgw-ctl02", State: "standing by", This: true},
|
||||
}}
|
||||
if v := m.View(); !strings.Contains(v, "Standing by. Shown, and changed, on the leader: llmgw-ctl") {
|
||||
t.Errorf("the overview on a standby:\n%s", v)
|
||||
}
|
||||
m.data.sum.ControlPlanes.Members[0].This, m.data.sum.ControlPlanes.Members[1].This = true, false
|
||||
if v := m.View(); strings.Contains(v, "Standing by.") {
|
||||
t.Error("the leader's console says it is standing by")
|
||||
}
|
||||
}
|
||||
@@ -1162,8 +1162,18 @@ func (a *API) removeModel(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, http.StatusOK, Removed{Removed: name})
|
||||
}
|
||||
|
||||
func (a *API) summary(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Summary())
|
||||
func (a *API) summary(w http.ResponseWriter, r *http.Request) {
|
||||
s := a.Summary()
|
||||
// Asked through a standby, which passes on the console's questions to
|
||||
// the leader: "this one" is the standby the console is on.
|
||||
if by := r.Header.Get("X-Llmgw-Forwarded-By"); by != "" && len(s.ControlPlanes.Members) > 0 {
|
||||
members := slices.Clone(s.ControlPlanes.Members)
|
||||
for i := range members {
|
||||
members[i].This = members[i].Name == by
|
||||
}
|
||||
s.ControlPlanes.Members = members
|
||||
}
|
||||
writeJSON(w, http.StatusOK, s)
|
||||
}
|
||||
|
||||
func (a *API) explainPlacement(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
@@ -231,6 +231,10 @@ func (c *Controller) SetLeadership(l *Leadership) {
|
||||
c.rec.SetLeadership(l)
|
||||
}
|
||||
|
||||
// Leadership is what SetLeadership set; nil when there is nobody to compete
|
||||
// with.
|
||||
func (c *Controller) Leadership() *Leadership { return c.lead }
|
||||
|
||||
// IsLeader reports whether this process is placing work. True when leadership
|
||||
// was never configured, because then there is nobody to compete with.
|
||||
func (c *Controller) IsLeader() bool { return c.lead == nil || c.lead.IsLeader() }
|
||||
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"errors"
|
||||
"log/slog"
|
||||
"os"
|
||||
"strings"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
@@ -183,3 +184,16 @@ func newOwnerID() string {
|
||||
}
|
||||
return host + "-" + hex.EncodeToString(b[:])
|
||||
}
|
||||
|
||||
// ControlRole is the lease's name in the store, for readers that ask who
|
||||
// holds it without competing.
|
||||
const ControlRole = controlRole
|
||||
|
||||
// HolderName is the machine an owner id names: the hostname it was made
|
||||
// from, without the random part.
|
||||
func HolderName(owner string) string {
|
||||
if i := strings.LastIndexByte(owner, '-'); i > 0 {
|
||||
return owner[:i]
|
||||
}
|
||||
return owner
|
||||
}
|
||||
Reference in new issue
Block a user