A floating address keeps to its own network, and Caddy follows the leader

Found by a three-machine install on fresh VMs. The second standby's run
asked for the floating address's interface with `ip route get <vip>`.
The machine already held the address, so that answered lo, and keepalived
would not start. The interface now comes from the route to the address's
network. Every cluster's VRRP router id was 51, so the PoC's two control
planes and the new three on one LAN each logged the other's adverts as an
invalid password; the id now comes from the address.

The Caddyfile, deploy/ha and install.md send clients to the control plane
whose /leader is 200, not /ready: /ready is 503 on a leader with no model
yet and would take the portal away with it. Setup no longer hides a key
file's path as if it were a password, and the starter's text no longer
says it runs elsewhere when this machine is a node.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-15 17:35:10 +02:00
1 parent 94401e21e6
commit eeaffd764d
9 files changed
+154 -19

No files matched your search

+24 -3
View File
@@ -9,6 +9,7 @@ import (
"encoding/pem"
"errors"
"fmt"
"hash/fnv"
"io"
"net"
"os"
@@ -527,13 +528,22 @@ func (sa *serverAdd) floating(ctx context.Context) error {
name string
priority int
}{{here, "this machine", 110}, {sa.remote, sa.name, priority}} {
// The interface on the address's network, by its route: not the
// route to the address, which is lo on the machine holding it
// already, as a run after the first finds it.
out, err := m.r.Run(ctx, fmt.Sprintf(`set -e
command -v keepalived >/dev/null 2>&1 || { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null; sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y keepalived >/dev/null; }
ip -o route get %s | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'`, vip), nil)
dev() { awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'; }
d=$(ip -o -4 route show %s | dev)
[ -n "$d" ] && [ "$d" != lo ] || d=$(ip -o route get %s | dev)
echo "$d"`, nw.String(), vip), nil)
if err != nil {
return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out)))
}
iface := strings.TrimSpace(string(out))
if iface == "" || iface == "lo" {
return fmt.Errorf("on %s: no interface on %s's network", m.name, sa.vip)
}
conf := keepalivedConf(iface, vip.String(), prefix, m.priority, pass)
if out, err := m.r.Run(ctx, `set -e
sudo install -d -m 0755 /etc/keepalived
@@ -571,6 +581,17 @@ func vrrpPassword(etcDir string) (string, error) {
return pass, nil
}
// vrrpRouterID is the VRRP router id for a floating address: from the
// address, so two clusters on one network, each with its own address, do
// not answer each other's adverts — with one id for all, each logged the
// other's as an invalid password, and dropped its own address's
// announcements with them.
func vrrpRouterID(vip string) int {
h := fnv.New32a()
_, _ = h.Write([]byte(vip))
return int(h.Sum32()%255) + 1
}
func keepalivedConf(iface, vip string, prefix, priority int, pass string) string {
return fmt.Sprintf(`# llmgw: the gateway's floating address, on whichever control plane leads.
vrrp_script llmgw_leader {
@@ -583,7 +604,7 @@ vrrp_script llmgw_leader {
vrrp_instance llmgw {
state BACKUP
interface %s
virtual_router_id 51
virtual_router_id %d
priority %d
advert_int 1
authentication {
@@ -597,7 +618,7 @@ vrrp_instance llmgw {
llmgw_leader
}
}
`, iface, priority, pass, vip, prefix)
`, iface, vrrpRouterID(vip), priority, pass, vip, prefix)
}
// clientCapable says a control plane's certificate can also present itself