A floating address keeps to its own network, and Caddy follows the leader
CI / test (push) Failing after 30m3s
CI / test (push) Failing after 30m3s
Found by a three-machine install on fresh VMs. The second standby's run asked for the floating address's interface with `ip route get <vip>`. The machine already held the address, so that answered lo, and keepalived would not start. The interface now comes from the route to the address's network. Every cluster's VRRP router id was 51, so the PoC's two control planes and the new three on one LAN each logged the other's adverts as an invalid password; the id now comes from the address. The Caddyfile, deploy/ha and install.md send clients to the control plane whose /leader is 200, not /ready: /ready is 503 on a leader with no model yet and would take the portal away with it. Setup no longer hides a key file's path as if it were a password, and the starter's text no longer says it runs elsewhere when this machine is a node. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
94401e21e6
commit
eeaffd764d
9 files changed
+154
-19
No files matched your search
@@ -9,6 +9,7 @@ import (
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"hash/fnv"
|
||||
"io"
|
||||
"net"
|
||||
"os"
|
||||
@@ -527,13 +528,22 @@ func (sa *serverAdd) floating(ctx context.Context) error {
|
||||
name string
|
||||
priority int
|
||||
}{{here, "this machine", 110}, {sa.remote, sa.name, priority}} {
|
||||
// The interface on the address's network, by its route: not the
|
||||
// route to the address, which is lo on the machine holding it
|
||||
// already, as a run after the first finds it.
|
||||
out, err := m.r.Run(ctx, fmt.Sprintf(`set -e
|
||||
command -v keepalived >/dev/null 2>&1 || { sudo DEBIAN_FRONTEND=noninteractive apt-get -qq update >/dev/null; sudo DEBIAN_FRONTEND=noninteractive apt-get -qq install -y keepalived >/dev/null; }
|
||||
ip -o route get %s | awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'`, vip), nil)
|
||||
dev() { awk '{for(i=1;i<=NF;i++) if($i=="dev") {print $(i+1); exit}}'; }
|
||||
d=$(ip -o -4 route show %s | dev)
|
||||
[ -n "$d" ] && [ "$d" != lo ] || d=$(ip -o route get %s | dev)
|
||||
echo "$d"`, nw.String(), vip), nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("on %s: %v: %s", m.name, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
iface := strings.TrimSpace(string(out))
|
||||
if iface == "" || iface == "lo" {
|
||||
return fmt.Errorf("on %s: no interface on %s's network", m.name, sa.vip)
|
||||
}
|
||||
conf := keepalivedConf(iface, vip.String(), prefix, m.priority, pass)
|
||||
if out, err := m.r.Run(ctx, `set -e
|
||||
sudo install -d -m 0755 /etc/keepalived
|
||||
@@ -571,6 +581,17 @@ func vrrpPassword(etcDir string) (string, error) {
|
||||
return pass, nil
|
||||
}
|
||||
|
||||
// vrrpRouterID is the VRRP router id for a floating address: from the
|
||||
// address, so two clusters on one network, each with its own address, do
|
||||
// not answer each other's adverts — with one id for all, each logged the
|
||||
// other's as an invalid password, and dropped its own address's
|
||||
// announcements with them.
|
||||
func vrrpRouterID(vip string) int {
|
||||
h := fnv.New32a()
|
||||
_, _ = h.Write([]byte(vip))
|
||||
return int(h.Sum32()%255) + 1
|
||||
}
|
||||
|
||||
func keepalivedConf(iface, vip string, prefix, priority int, pass string) string {
|
||||
return fmt.Sprintf(`# llmgw: the gateway's floating address, on whichever control plane leads.
|
||||
vrrp_script llmgw_leader {
|
||||
@@ -583,7 +604,7 @@ vrrp_script llmgw_leader {
|
||||
vrrp_instance llmgw {
|
||||
state BACKUP
|
||||
interface %s
|
||||
virtual_router_id 51
|
||||
virtual_router_id %d
|
||||
priority %d
|
||||
advert_int 1
|
||||
authentication {
|
||||
@@ -597,7 +618,7 @@ vrrp_instance llmgw {
|
||||
llmgw_leader
|
||||
}
|
||||
}
|
||||
`, iface, priority, pass, vip, prefix)
|
||||
`, iface, vrrpRouterID(vip), priority, pass, vip, prefix)
|
||||
}
|
||||
|
||||
// clientCapable says a control plane's certificate can also present itself
|
||||
|
||||
@@ -166,3 +166,14 @@ func TestStandbysWantTheStoresMachines(t *testing.T) {
|
||||
t.Errorf("standbys with nowhere to go: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTwoClustersOnOneNetworkKeepApartFloatingAddresses(t *testing.T) {
|
||||
a, b := vrrpRouterID("10.0.0.161"), vrrpRouterID("10.0.0.164")
|
||||
if a == b || a < 1 || a > 255 || b < 1 || b > 255 {
|
||||
t.Errorf("router ids %d and %d", a, b)
|
||||
}
|
||||
conf := keepalivedConf("ens18", "10.0.0.164", 24, 100, "secret12")
|
||||
if !strings.Contains(conf, fmt.Sprintf("virtual_router_id %d\n", b)) || !strings.Contains(conf, "interface ens18") {
|
||||
t.Errorf("conf:\n%s", conf)
|
||||
}
|
||||
}
|
||||
@@ -257,7 +257,7 @@ func newInstallForm(in *serverInit, host string, found []string, ha bool) *insta
|
||||
},
|
||||
}
|
||||
if ha {
|
||||
f.starter.about += " With three machines it runs on the first node that has room for it."
|
||||
f.starter.about += " Unless this machine is a node, it runs on the first node that joins with room for it."
|
||||
}
|
||||
|
||||
// --- pictures and 3D ---
|
||||
|
||||
@@ -112,6 +112,10 @@ func newDBMachine(n int) dbMachine {
|
||||
if os.Getenv("SSH_AUTH_SOCK") == "" {
|
||||
m.auth = authKey
|
||||
}
|
||||
// As choosing the way to log in sets them: a key file's path is not a
|
||||
// secret to hide.
|
||||
m.secret.Mask = m.auth == authPassword
|
||||
m.secret.Placeholder = map[int]string{authKey: "/root/.ssh/id_ed25519", authPassword: "the user's ssh password"}[m.auth]
|
||||
return m
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user