A floating address keeps to its own network, and Caddy follows the leader
CI / test (push) Failing after 30m3s

Found by a three-machine install on fresh VMs. The second standby's run
asked for the floating address's interface with `ip route get <vip>`.
The machine already held the address, so that answered lo, and keepalived
would not start. The interface now comes from the route to the address's
network. Every cluster's VRRP router id was 51, so the PoC's two control
planes and the new three on one LAN each logged the other's adverts as an
invalid password; the id now comes from the address.

The Caddyfile, deploy/ha and install.md send clients to the control plane
whose /leader is 200, not /ready: /ready is 503 on a leader with no model
yet and would take the portal away with it. Setup no longer hides a key
file's path as if it were a password, and the starter's text no longer
says it runs elsewhere when this machine is a node.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 17:35:10 +02:00
1 parent 94401e21e6
commit eeaffd764d
9 files changed
+154 -19

No files matched your search

+24 -3
View File
@@ -9,6 +9,7 @@ import (
"encoding/pem"
"errors"
"fmt"
"hash/fnv"
"io"
"net"
"os"
@@ -527,13 +528,22 @@ func (sa *serverAdd) floating(ctx context.Context) error {
name string
priority int
}{{here, "this machine", 110}, {sa.remote, sa.name, priority}} {
// The interface on the address's network, by its route: not the
// route to the address, which is lo on the machine holding it
// already, as a run after the first finds it.
out, err := m.r.Run(ctx, fmt.Sprintf(`set -e
command -v keepalived >/dev/null 2>&1 || { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null; sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y keepalived >/dev/null; }
ip -o route get %s | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'`, vip), nil)
dev() { awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'; }
d=$(ip -o -4 route show %s | dev)
[ -n "$d" ] && [ "$d" != lo ] || d=$(ip -o route get %s | dev)
echo "$d"`, nw.String(), vip), nil)
if err != nil {
return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out)))
}
iface := strings.TrimSpace(string(out))
if iface == "" || iface == "lo" {
return fmt.Errorf("on %s: no interface on %s's network", m.name, sa.vip)
}
conf := keepalivedConf(iface, vip.String(), prefix, m.priority, pass)
if out, err := m.r.Run(ctx, `set -e
sudo install -d -m 0755 /etc/keepalived
@@ -571,6 +581,17 @@ func vrrpPassword(etcDir string) (string, error) {
return pass, nil
}
// vrrpRouterID is the VRRP router id for a floating address: from the
// address, so two clusters on one network, each with its own address, do
// not answer each other's adverts — with one id for all, each logged the
// other's as an invalid password, and dropped its own address's
// announcements with them.
func vrrpRouterID(vip string) int {
h := fnv.New32a()
_, _ = h.Write([]byte(vip))
return int(h.Sum32()%255) + 1
}
func keepalivedConf(iface, vip string, prefix, priority int, pass string) string {
return fmt.Sprintf(`# llmgw: the gateway's floating address, on whichever control plane leads.
vrrp_script llmgw_leader {
@@ -583,7 +604,7 @@ vrrp_script llmgw_leader {
vrrp_instance llmgw {
state BACKUP
interface %s
virtual_router_id 51
virtual_router_id %d
priority %d
advert_int 1
authentication {
@@ -597,7 +618,7 @@ vrrp_instance llmgw {
llmgw_leader
}
}
`, iface, priority, pass, vip, prefix)
`, iface, vrrpRouterID(vip), priority, pass, vip, prefix)
}
// clientCapable says a control plane's certificate can also present itself
+11
View File
@@ -166,3 +166,14 @@ func TestStandbysWantTheStoresMachines(t *testing.T) {
t.Errorf("standbys with nowhere to go: %v", err)
}
}
func TestTwoClustersOnOneNetworkKeepApartFloatingAddresses(t *testing.T) {
a, b := vrrpRouterID("10.0.0.161"), vrrpRouterID("10.0.0.164")
if a == b || a < 1 || a > 255 || b < 1 || b > 255 {
t.Errorf("router ids %d and %d", a, b)
}
conf := keepalivedConf("ens18", "10.0.0.164", 24, 100, "secret12")
if !strings.Contains(conf, fmt.Sprintf("virtual_router_id %d\n", b)) || !strings.Contains(conf, "interface ens18") {
t.Errorf("conf:\n%s", conf)
}
}
+1 -1
View File
@@ -257,7 +257,7 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta
},
}
if ha {
f.starter.about += " With three machines it runs on the first node that has room for it."
f.starter.about += " Unless this machine is a node, it runs on the first node that joins with room for it."
}
// --- pictures and 3D ---
+4
View File
@@ -112,6 +112,10 @@ func newDBMachine(n int) dbMachine {
if os.Getenv("SSH_AUTH_SOCK") == "" {
m.auth = authKey
}
// As choosing the way to log in sets them: a key file's path is not a
// secret to hide.
m.secret.Mask = m.auth == authPassword
m.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[m.auth]
return m
}
+34
View File
@@ -14,6 +14,7 @@ llm.example.com {
max_size 32MB
}
# One control plane. With more than one, see the end of this file.
reverse_proxy 127.0.0.1:8080 {
# Tokens are written the moment they are generated. Without this a
# streamed reply can arrive in bursts, and a client measuring time to
@@ -39,3 +40,36 @@ llm.example.com {
# Access log to stderr, which under systemd is the journal.
log
}
# More than one control plane (server init --standbys, server add): one of
# them leads, the others stand by. Name them all and send to the one that
# leads — /leader is 200 there and 503 on a standby. Not /ready: that is 503
# on a leader with no model running yet, and would take the portal away with
# it. A standby would pass a request on to the leader anyway, but not while
# there is none, and the hop is for nothing.
#
# With a floating address (--vip) the address already follows the leader:
# give reverse_proxy that one address instead, and keep the rest as above.
#
# llm.example.com {
# request_body {
# max_size 32MB
# }
# reverse_proxy 10.0.0.157:8080 10.0.0.158:8080 10.0.0.159:8080 {
# # The first that is healthy: the leader, whichever it is.
# lb_policy first
# health_uri /leader
# health_interval 2s
# health_timeout 2s
# # While leadership moves — a few seconds when a leader is lost —
# # none is healthy; a request waits that long for one rather than
# # failing at once.
# lb_try_duration 15s
# lb_try_interval 250ms
# flush_interval -1
# transport http {
# response_header_timeout 15m
# }
# }
# log
# }