A floating address keeps to its own network, and Caddy follows the leader
CI / test (push) Failing after 30m3s

Found by a three-machine install on fresh VMs. The second standby's run
asked for the floating address's interface with `ip route get <vip>`.
The machine already held the address, so that answered lo, and keepalived
would not start. The interface now comes from the route to the address's
network. Every cluster's VRRP router id was 51, so the PoC's two control
planes and the new three on one LAN each logged the other's adverts as an
invalid password; the id now comes from the address.

The Caddyfile, deploy/ha and install.md send clients to the control plane
whose /leader is 200, not /ready: /ready is 503 on a leader with no model
yet and would take the portal away with it. Setup no longer hides a key
file's path as if it were a password, and the starter's text no longer
says it runs elsewhere when this machine is a node.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 17:35:10 +02:00
1 parent 94401e21e6
commit eeaffd764d
9 files changed
+154 -19

No files matched your search

+24 -3
View File
@@ -9,6 +9,7 @@ import (
"encoding/pem"
"errors"
"fmt"
"hash/fnv"
"io"
"net"
"os"
@@ -527,13 +528,22 @@ func (sa *serverAdd) floating(ctx context.Context) error {
name string
priority int
}{{here, "this machine", 110}, {sa.remote, sa.name, priority}} {
// The interface on the address's network, by its route: not the
// route to the address, which is lo on the machine holding it
// already, as a run after the first finds it.
out, err := m.r.Run(ctx, fmt.Sprintf(`set -e
command -v keepalived >/dev/null 2>&1 || { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null; sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y keepalived >/dev/null; }
ip -o route get %s | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'`, vip), nil)
dev() { awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'; }
d=$(ip -o -4 route show %s | dev)
[ -n "$d" ] && [ "$d" != lo ] || d=$(ip -o route get %s | dev)
echo "$d"`, nw.String(), vip), nil)
if err != nil {
return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out)))
}
iface := strings.TrimSpace(string(out))
if iface == "" || iface == "lo" {
return fmt.Errorf("on %s: no interface on %s's network", m.name, sa.vip)
}
conf := keepalivedConf(iface, vip.String(), prefix, m.priority, pass)
if out, err := m.r.Run(ctx, `set -e
sudo install -d -m 0755 /etc/keepalived
@@ -571,6 +581,17 @@ func vrrpPassword(etcDir string) (string, error) {
return pass, nil
}
// vrrpRouterID is the VRRP router id for a floating address: from the
// address, so two clusters on one network, each with its own address, do
// not answer each other's adverts — with one id for all, each logged the
// other's as an invalid password, and dropped its own address's
// announcements with them.
func vrrpRouterID(vip string) int {
h := fnv.New32a()
_, _ = h.Write([]byte(vip))
return int(h.Sum32()%255) + 1
}
func keepalivedConf(iface, vip string, prefix, priority int, pass string) string {
return fmt.Sprintf(`# llmgw: the gateway's floating address, on whichever control plane leads.
vrrp_script llmgw_leader {
@@ -583,7 +604,7 @@ vrrp_script llmgw_leader {
vrrp_instance llmgw {
state BACKUP
interface %s
virtual_router_id 51
virtual_router_id %d
priority %d
advert_int 1
authentication {
@@ -597,7 +618,7 @@ vrrp_instance llmgw {
llmgw_leader
}
}
`, iface, priority, pass, vip, prefix)
`, iface, vrrpRouterID(vip), priority, pass, vip, prefix)
}
// clientCapable says a control plane's certificate can also present itself
+11
View File
@@ -166,3 +166,14 @@ func TestStandbysWantTheStoresMachines(t *testing.T) {
t.Errorf("standbys with nowhere to go: %v", err)
}
}
func TestTwoClustersOnOneNetworkKeepApartFloatingAddresses(t *testing.T) {
a, b := vrrpRouterID("10.0.0.161"), vrrpRouterID("10.0.0.164")
if a == b || a < 1 || a > 255 || b < 1 || b > 255 {
t.Errorf("router ids %d and %d", a, b)
}
conf := keepalivedConf("ens18", "10.0.0.164", 24, 100, "secret12")
if !strings.Contains(conf, fmt.Sprintf("virtual_router_id %d\n", b)) || !strings.Contains(conf, "interface ens18") {
t.Errorf("conf:\n%s", conf)
}
}
+1 -1
View File
@@ -257,7 +257,7 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta
},
}
if ha {
f.starter.about += " With three machines it runs on the first node that has room for it."
f.starter.about += " Unless this machine is a node, it runs on the first node that joins with room for it."
}
// --- pictures and 3D ---
+4
View File
@@ -112,6 +112,10 @@ func newDBMachine(n int) dbMachine {
if os.Getenv("SSH_AUTH_SOCK") == "" {
m.auth = authKey
}
// As choosing the way to log in sets them: a key file's path is not a
// secret to hide.
m.secret.Mask = m.auth == authPassword
m.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[m.auth]
return m
}