diff --git a/deploy/ha/README.md b/deploy/ha/README.md index fa679a6..1e2d531 100644 --- a/deploy/ha/README.md +++ b/deploy/ha/README.md @@ -1,4 +1,4 @@ -# Two control planes +# More than one control plane llmgw handles the hard half of this itself: only one control plane places work, and agents find whichever one is leading. What it does not do is move client @@ -37,7 +37,7 @@ nothing in front of the two had moved. So something has to. The options, in the order most people should consider them: -**A load balancer you already run.** Health-check `/ready` on each control +**A load balancer you already run.** Health-check `/leader` on each control plane and send traffic to whichever answers. This is the right answer if you have one, and it needs nothing else from this directory. @@ -50,25 +50,50 @@ in whatever your resolvers actually honour rather than what you set, which on a bad day is minutes. Fine for a cluster people use interactively, wrong for one an agent is driving. -## Check `/ready`, not `/health` +## Check `/leader`, not `/health` This is the one thing to get right, and the obvious choice is the wrong one. -`/health` says the process is up. It says nothing about whether it can serve, +`/health` says the process is up. It says nothing about whether it leads, and a standby is exactly that case: alive, answering, and holding no endpoints at all — its reconciler placed nothing and it has no agent streams to adopt -from. A load balancer checking `/health` would send it half the traffic, and -half the traffic would be a 503. +from. A standby passes client requests on to the leader, so traffic sent to +one still works — until the leader is the machine that was lost, and there +is nobody to pass them to. + +`/leader` is 200 on the control plane that leads and 503 on a standby. It is +what keepalived follows for the floating address, and what a load balancer +should send traffic by: to the one machine that answers 200. + +`/ready` is narrower: 200 only when this process has somewhere to send a +request, so 503 on a standby and also on a leader with no model running yet. +Checking it takes the portal and the admin pages away with the models — on a +new cluster with no node yet, nobody can sign in to add one. Use it to stop +sending inference only, if your load balancer can tell the paths apart. -`/ready` says this process has somewhere to send a request. It is 503 on a -standby, 503 on a leader whose replicas have all failed, and 200 when there is -something to serve — which is the question a load balancer is actually asking. Both are open, because a load balancer has no key. -During a failover neither passes for about the length of the lease handover: +During a failover nothing passes for about the length of the lease handover: three seconds when the leader is stopped cleanly, up to the lease TTL when it -is not. That gap is real and there is no way to have it both ways — traffic -cannot go to a control plane that has nothing to route to. +is not. A proxy that waits that long for a healthy machine rather than +failing at once (Caddy's `lb_try_duration`) hides it from clients. + +**Caddy**, in front of three control planes — the end of +[deploy/proxy/Caddyfile](../proxy/Caddyfile) has it with the rest of the +settings: + +``` +reverse_proxy 10.0.0.157:8080 10.0.0.158:8080 10.0.0.159:8080 { + lb_policy first + health_uri /leader + health_interval 2s + lb_try_duration 15s + flush_interval -1 +} +``` + +With a floating address the address already follows the leader: give Caddy +that one address instead. Not the admin port, for a different reason: it serves pprof, and pprof can dump the process heap, where API keys sit in plaintext. diff --git a/deploy/ha/keepalived.conf.example b/deploy/ha/keepalived.conf.example index c728d40..5921fae 100644 --- a/deploy/ha/keepalived.conf.example +++ b/deploy/ha/keepalived.conf.example @@ -41,7 +41,9 @@ vrrp_instance llmgw { # MASTER on the machine you would rather serve; BACKUP on the other. state MASTER interface eth0 - # Any number both machines agree on and nothing else on the segment uses. + # Any number both machines agree on and nothing else on the segment uses: + # another cluster's keepalived with the same one reads as an invalid + # password. `llmgw server add --vip` derives one from the address. virtual_router_id 51 # The higher one wins while both are healthy. The weight above is enough # to invert this when the preferred machine's gateway is not answering. diff --git a/deploy/proxy/Caddyfile b/deploy/proxy/Caddyfile index a27034e..5bc8bae 100644 --- a/deploy/proxy/Caddyfile +++ b/deploy/proxy/Caddyfile @@ -14,6 +14,7 @@ llm.example.com { max_size 32MB } + # One control plane. With more than one, see the end of this file. reverse_proxy 127.0.0.1:8080 { # Tokens are written the moment they are generated. Without this a # streamed reply can arrive in bursts, and a client measuring time to @@ -39,3 +40,36 @@ llm.example.com { # Access log to stderr, which under systemd is the journal. log } + +# More than one control plane (server init --standbys, server add): one of +# them leads, the others stand by. Name them all and send to the one that +# leads — /leader is 200 there and 503 on a standby. Not /ready: that is 503 +# on a leader with no model running yet, and would take the portal away with +# it. A standby would pass a request on to the leader anyway, but not while +# there is none, and the hop is for nothing. +# +# With a floating address (--vip) the address already follows the leader: +# give reverse_proxy that one address instead, and keep the rest as above. +# +# llm.example.com { +# request_body { +# max_size 32MB +# } +# reverse_proxy 10.0.0.157:8080 10.0.0.158:8080 10.0.0.159:8080 { +# # The first that is healthy: the leader, whichever it is. +# lb_policy first +# health_uri /leader +# health_interval 2s +# health_timeout 2s +# # While leadership moves — a few seconds when a leader is lost — +# # none is healthy; a request waits that long for one rather than +# # failing at once. +# lb_try_duration 15s +# lb_try_interval 250ms +# flush_interval -1 +# transport http { +# response_header_timeout 15m +# } +# } +# log +# } diff --git a/docs/install.md b/docs/install.md index f7963a2..fa33a57 100644 --- a/docs/install.md +++ b/docs/install.md @@ -2394,8 +2394,12 @@ and lets it into the store; it starts there as a standby. Every node is told of it within half a minute and fails over to it without being touched. `--vip` keeps a floating address on whichever control plane leads, with keepalived on both: that is the address to give clients, or a proxy in front. -Without it, put a load balancer in front that checks `/ready` — not -`/health`, which is 200 on a standby that would refuse every request. +Without it, put a proxy in front that sends to whichever answers `/leader` +with 200 — not `/health`, which is 200 on a standby too, and not `/ready`, +which is 503 on a leader with no model yet and would take the portal away +with it. The end of [deploy/proxy/Caddyfile](../deploy/proxy/Caddyfile) has +Caddy doing that for three control planes, and +[deploy/ha/](../deploy/ha/README.md#check-leader-not-health) why. - **What a failover costs.** Measured on the reference cluster, stopping the leader: the standby led within five seconds, every node had moved to it diff --git a/internal/cli/serveradd.go b/internal/cli/serveradd.go index bc1f0bd..9b27bef 100644 --- a/internal/cli/serveradd.go +++ b/internal/cli/serveradd.go @@ -9,6 +9,7 @@ import ( "encoding/pem" "errors" "fmt" + "hash/fnv" "io" "net" "os" @@ -527,13 +528,22 @@ func (sa *serverAdd) floating(ctx context.Context) error { name string priority int }{{here, "this machine", 110}, {sa.remote, sa.name, priority}} { + // The interface on the address's network, by its route: not the + // route to the address, which is lo on the machine holding it + // already, as a run after the first finds it. out, err := m.r.Run(ctx, fmt.Sprintf(`set -e command -v keepalived >/dev/null 2>&1 || { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null; sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y keepalived >/dev/null; } -ip -o route get %s | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'`, vip), nil) +dev() { awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'; } +d=$(ip -o -4 route show %s | dev) +[ -n "$d" ] && [ "$d" != lo ] || d=$(ip -o route get %s | dev) +echo "$d"`, nw.String(), vip), nil) if err != nil { return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out))) } iface := strings.TrimSpace(string(out)) + if iface == "" || iface == "lo" { + return fmt.Errorf("on %s: no interface on %s's network", m.name, sa.vip) + } conf := keepalivedConf(iface, vip.String(), prefix, m.priority, pass) if out, err := m.r.Run(ctx, `set -e sudo install -d -m 0755 /etc/keepalived @@ -571,6 +581,17 @@ func vrrpPassword(etcDir string) (string, error) { return pass, nil } +// vrrpRouterID is the VRRP router id for a floating address: from the +// address, so two clusters on one network, each with its own address, do +// not answer each other's adverts — with one id for all, each logged the +// other's as an invalid password, and dropped its own address's +// announcements with them. +func vrrpRouterID(vip string) int { + h := fnv.New32a() + _, _ = h.Write([]byte(vip)) + return int(h.Sum32()%255) + 1 +} + func keepalivedConf(iface, vip string, prefix, priority int, pass string) string { return fmt.Sprintf(`# llmgw: the gateway's floating address, on whichever control plane leads. vrrp_script llmgw_leader { @@ -583,7 +604,7 @@ vrrp_script llmgw_leader { vrrp_instance llmgw { state BACKUP interface %s - virtual_router_id 51 + virtual_router_id %d priority %d advert_int 1 authentication { @@ -597,7 +618,7 @@ vrrp_instance llmgw { llmgw_leader } } -`, iface, priority, pass, vip, prefix) +`, iface, vrrpRouterID(vip), priority, pass, vip, prefix) } // clientCapable says a control plane's certificate can also present itself diff --git a/internal/cli/serverinit_choices_test.go b/internal/cli/serverinit_choices_test.go index e0a3ba4..6407ef3 100644 --- a/internal/cli/serverinit_choices_test.go +++ b/internal/cli/serverinit_choices_test.go @@ -166,3 +166,14 @@ func TestStandbysWantTheStoresMachines(t *testing.T) { t.Errorf("standbys with nowhere to go: %v", err) } } + +func TestTwoClustersOnOneNetworkKeepApartFloatingAddresses(t *testing.T) { + a, b := vrrpRouterID("10.0.0.161"), vrrpRouterID("10.0.0.164") + if a == b || a < 1 || a > 255 || b < 1 || b > 255 { + t.Errorf("router ids %d and %d", a, b) + } + conf := keepalivedConf("ens18", "10.0.0.164", 24, 100, "secret12") + if !strings.Contains(conf, fmt.Sprintf("virtual_router_id %d\n", b)) || !strings.Contains(conf, "interface ens18") { + t.Errorf("conf:\n%s", conf) + } +} diff --git a/internal/cli/tui_choices.go b/internal/cli/tui_choices.go index dcfe560..a520aaa 100644 --- a/internal/cli/tui_choices.go +++ b/internal/cli/tui_choices.go @@ -257,7 +257,7 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta }, } if ha { - f.starter.about += " With three machines it runs on the first node that has room for it." + f.starter.about += " Unless this machine is a node, it runs on the first node that joins with room for it." } // --- pictures and 3D --- diff --git a/internal/cli/tui_setup.go b/internal/cli/tui_setup.go index 6ddc93d..976de62 100644 --- a/internal/cli/tui_setup.go +++ b/internal/cli/tui_setup.go @@ -112,6 +112,10 @@ func newDBMachine(n int) dbMachine { if os.Getenv("SSH_AUTH_SOCK") == "" { m.auth = authKey } + // As choosing the way to log in sets them: a key file's path is not a + // secret to hide. + m.secret.Mask = m.auth == authPassword + m.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[m.auth] return m } diff --git a/internal/web/proxy/Caddyfile b/internal/web/proxy/Caddyfile index a27034e..5bc8bae 100644 --- a/internal/web/proxy/Caddyfile +++ b/internal/web/proxy/Caddyfile @@ -14,6 +14,7 @@ llm.example.com { max_size 32MB } + # One control plane. With more than one, see the end of this file. reverse_proxy 127.0.0.1:8080 { # Tokens are written the moment they are generated. Without this a # streamed reply can arrive in bursts, and a client measuring time to @@ -39,3 +40,36 @@ llm.example.com { # Access log to stderr, which under systemd is the journal. log } + +# More than one control plane (server init --standbys, server add): one of +# them leads, the others stand by. Name them all and send to the one that +# leads — /leader is 200 there and 503 on a standby. Not /ready: that is 503 +# on a leader with no model running yet, and would take the portal away with +# it. A standby would pass a request on to the leader anyway, but not while +# there is none, and the hop is for nothing. +# +# With a floating address (--vip) the address already follows the leader: +# give reverse_proxy that one address instead, and keep the rest as above. +# +# llm.example.com { +# request_body { +# max_size 32MB +# } +# reverse_proxy 10.0.0.157:8080 10.0.0.158:8080 10.0.0.159:8080 { +# # The first that is healthy: the leader, whichever it is. +# lb_policy first +# health_uri /leader +# health_interval 2s +# health_timeout 2s +# # While leadership moves — a few seconds when a leader is lost — +# # none is healthy; a request waits that long for one rather than +# # failing at once. +# lb_try_duration 15s +# lb_try_interval 250ms +# flush_interval -1 +# transport http { +# response_header_timeout 15m +# } +# } +# log +# }