From ad04a7834301064064894a7cbcc0d068528a02b5 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 6 Oct 2026 09:36:09 +0200 Subject: [PATCH] The mirror's address is one the nodes reach, and node add checks it server init and mirror init took the first non-loopback address for the mirror's --base, Docker's 172.17.0.1 on a machine with Docker; nodes asked their own bridge for every model and were refused. primaryAddr takes the default route's source, else the first address on an interface that is not a container's, bridge's or overlay's. node add has the node reach the mirror's files during the checks and fails, naming the address and the fix, when it cannot. Co-Authored-By: Claude Opus 5.5 (1M context) --- CHANGELOG.md | 8 +++++ internal/cli/mirrorinit.go | 5 +-- internal/cli/node.go | 11 ++++++ internal/cli/serverinit.go | 55 ++++++++++++++++++++++++++++++ internal/cli/serverinit_cluster.go | 4 +-- internal/cli/serverinit_test.go | 19 +++++++++++ internal/onboard/onboard.go | 40 +++++++++++++++++++++- internal/onboard/onboard_test.go | 31 +++++++++++++++++ 8 files changed, 166 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 20630c05..1cfa1051 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -56,6 +56,14 @@ APIs change between minors without a deprecation period. ### Fixed +- **The mirror is given an address the nodes reach**: `server init` and + `mirror init` took the machine's first address for the mirror's `--base`, + which on a machine with Docker could be 172.17.0.1 — a node then asked its + own bridge for every model and was refused. They now take the source of + the default route, else the first address not on a container's or a + bridge's interface. And `node add` has the node reach the mirror's files + before it is added: one that cannot fails the checks, naming the address + and what to change. - **A standby that becomes the leader reads the store again**: models added or removed, engine builds chosen and domains held through the leader before it, rather than placing on what it read when it started. diff --git a/internal/cli/mirrorinit.go b/internal/cli/mirrorinit.go index 7a433f45..63614c12 100644 --- a/internal/cli/mirrorinit.go +++ b/internal/cli/mirrorinit.go @@ -89,10 +89,7 @@ func mirrorInit(ctx context.Context, out io.Writer, dir, base string, controlPla if err := in.chown(dir, true); err != nil { return err } - addr := "" - if ips := localAddrs(); len(ips) > 0 { - addr = ips[0].String() - } + addr := primaryAddr() if base == "" { if addr == "" { return errors.New("this machine has no address to give nodes; give one with --base") diff --git a/internal/cli/node.go b/internal/cli/node.go index 8f35330f..049cceb2 100644 --- a/internal/cli/node.go +++ b/internal/cli/node.go @@ -18,6 +18,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" "gitea.mynodes.xyz/ilker/llmgw/internal/firewall" + "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/onboard" "gitea.mynodes.xyz/ilker/llmgw/internal/pgha" "gitea.mynodes.xyz/ilker/llmgw/internal/release" @@ -183,6 +184,16 @@ func (na *nodeAdd) prepare() error { o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter} o.FirewallAllow, _ = cfg.FirewallAllow() } + // Where the mirror serves model files, for the node to be asked to reach. + if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && cfg.Mirror.API != "" { + if mc, err := mirror.NewClient(cfg.Mirror.API, cfg.Mirror.TLS, 5*time.Second); err == nil { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + if list, err := mc.List(ctx); err == nil { + o.MirrorBase = list.Base + } + cancel() + } + } // Where the control plane upgrades nodes from, so a node joins on the // same release from the same place: an offline cluster's mirror. if o.Releases == "" { diff --git a/internal/cli/serverinit.go b/internal/cli/serverinit.go index 92f114a8..76583978 100644 --- a/internal/cli/serverinit.go +++ b/internal/cli/serverinit.go @@ -821,3 +821,58 @@ Before people outside this network use it: # in server.yaml, and copy them off this machine `) } + +// primaryAddr is the address other machines most likely reach this one on: +// the source of its default route, else the first address on an interface +// that is not a container's or a bridge's. The first address of all is not +// it on a machine with Docker, whose 172.17.0.1 a node takes for its own. +func primaryAddr() string { + if out, err := exec.Command("ip", "-4", "route", "get", "1.1.1.1").Output(); err == nil { + if src := routeSource(string(out)); src != "" { + return src + } + } + ifaces, err := net.Interfaces() + if err == nil { + for _, ifc := range ifaces { + if ifc.Flags&net.FlagUp == 0 || virtualInterface(ifc.Name) { + continue + } + addrs, _ := ifc.Addrs() + for _, a := range addrs { + if n, ok := a.(*net.IPNet); ok && n.IP.To4() != nil && n.IP.IsGlobalUnicast() { + return n.IP.String() + } + } + } + } + if ips := localAddrs(); len(ips) > 0 { + return ips[0].String() + } + return "" +} + +// routeSource reads the src of `ip route get`. +func routeSource(out string) string { + f := strings.Fields(out) + for i := 0; i+1 < len(f); i++ { + if f[i] == "src" { + if ip := net.ParseIP(f[i+1]); ip != nil && !ip.IsLoopback() { + return ip.String() + } + } + } + return "" +} + +// virtualInterface is an interface containers, bridges or overlays made, +// whose address no other machine reaches this one on. +func virtualInterface(name string) bool { + for _, p := range []string{"docker", "br-", "virbr", "veth", "cni", "flannel", "cali", "vxlan", + "kube", "lxc", "lxd", "podman", "cilium", "weave", "tun", "tap"} { + if strings.HasPrefix(name, p) { + return true + } + } + return false +} diff --git a/internal/cli/serverinit_cluster.go b/internal/cli/serverinit_cluster.go index efaef21b..27cf3c54 100644 --- a/internal/cli/serverinit_cluster.go +++ b/internal/cli/serverinit_cluster.go @@ -86,8 +86,8 @@ func (in *serverInit) reachableAddr() string { if len(in.hosts) > 0 { return in.hosts[0] } - if ips := localAddrs(); len(ips) > 0 { - return ips[0].String() + if a := primaryAddr(); a != "" { + return a } name, _ := os.Hostname() return name diff --git a/internal/cli/serverinit_test.go b/internal/cli/serverinit_test.go index 35323042..bd202f14 100644 --- a/internal/cli/serverinit_test.go +++ b/internal/cli/serverinit_test.go @@ -172,3 +172,22 @@ func TestInitKeepsCertificatesFromAnAuthorityElsewhere(t *testing.T) { t.Error("--host cannot be honoured without the authority, and should say so") } } + +func TestTheAddressNodesReachIsNotAContainersBridge(t *testing.T) { + if got := routeSource("1.1.1.1 via 10.104.130.129 dev ens192 src 10.104.130.140 uid 0\n cache\n"); got != "10.104.130.140" { + t.Errorf("route source %q", got) + } + if got := routeSource("unreachable"); got != "" { + t.Errorf("no route: %q", got) + } + for _, name := range []string{"docker0", "br-3f2a", "virbr0", "veth12ab", "cni0", "flannel.1"} { + if !virtualInterface(name) { + t.Errorf("%s taken for a real interface", name) + } + } + for _, name := range []string{"eth0", "ens192", "enp3s0", "bond0", "ib0"} { + if virtualInterface(name) { + t.Errorf("%s taken for a virtual one", name) + } + } +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index 3ef7a04c..e0559c5a 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -69,7 +69,11 @@ type Options struct { // NoFirewall leaves the machine's firewall as it is. Otherwise its llmgw // ports — FirewallPorts — answer only the networks it reaches the // control planes on, and FirewallAllow. - NoFirewall bool + NoFirewall bool + // MirrorBase is where the mirror serves model files, http://host:8081: + // the node is asked to reach it before it is added, since a node that + // cannot fetches no model. + MirrorBase string FirewallPorts []firewall.Port FirewallAllow []netip.Prefix @@ -461,6 +465,9 @@ func (r *Runner) check(ctx context.Context, res *Result) error { // A missing firewall is what this run is about to write. rep.Findings = slices.DeleteFunc(rep.Findings, func(f doctor.Finding) bool { return f.Area == "firewall" }) } + if f, ok := r.mirrorReach(ctx); ok { + rep.Findings = append(rep.Findings, f) + } res.Facts, res.Findings = rep.Facts, rep.Findings fails := 0 @@ -502,6 +509,37 @@ func (r *Runner) check(ctx context.Context, res *Result) error { return nil } +// mirrorReach asks the node to reach the mirror's files: a base address the +// node cannot reach (another machine's interface, Docker's bridge, a port +// closed to it) otherwise shows up as every model failing to download. +func (r *Runner) mirrorReach(ctx context.Context) (doctor.Finding, bool) { + if r.Opts.MirrorBase == "" { + return doctor.Finding{}, false + } + u, err := url.Parse(r.Opts.MirrorBase) + if err != nil || u.Hostname() == "" { + return doctor.Finding{}, false + } + port := u.Port() + if port == "" { + port = "80" + } + base := strings.TrimRight(r.Opts.MirrorBase, "/") + "/" + out, _ := r.Remote.Run(ctx, `if command -v curl >/dev/null 2>&1; then + curl -s -o /dev/null -w '%{http_code}' --max-time 5 `+shellQuote(base)+` || true +else + timeout 5 bash -c `+shellQuote("/dev/null && echo 200 || echo 000 +fi`, nil) + code := strings.TrimSpace(lastLine(out)) + if code != "" && code != "000" { + return doctor.Finding{Level: doctor.OK, Area: "mirror", Message: "reaches the mirror's files at " + base}, true + } + return doctor.Finding{Level: doctor.Fail, Area: "mirror", + Message: "cannot reach the mirror's files at " + base + ": every model it is given would fail to download", + Fix: "on the mirror's machine, give llmgw-mirror a --base this node reaches (systemctl cat llmgw-mirror), " + + "and let this node through to port " + port}, true +} + // suggestFabrics groups the active adapters by the fabric they report being // on: the subnet manager they see, or the network of their RoCE address. // Names are placeholders for the operator: a name is what ties one node's diff --git a/internal/onboard/onboard_test.go b/internal/onboard/onboard_test.go index 95e0a192..29dd213e 100644 --- a/internal/onboard/onboard_test.go +++ b/internal/onboard/onboard_test.go @@ -32,6 +32,8 @@ type fakeMachine struct { files map[string][]byte scripts []string download error + // mirror is what curl prints for the mirror's address: 000 unreachable. + mirror string } type doctorReport struct { @@ -64,6 +66,8 @@ func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byt return nil, nil case strings.Contains(script, "systemctl restart llmgw-agent"): return []byte("active\n"), nil + case strings.Contains(script, "curl -s -o /dev/null -w"): + return []byte(f.mirror), nil } return nil, nil } @@ -389,3 +393,30 @@ func TestTheSudoGrantRemovesItselfWithinTheHour(t *testing.T) { t.Errorf("not shell: %v %s\n%s", err, out, inner) } } + +func TestANodeThatCannotReachTheMirrorIsNotAdded(t *testing.T) { + m := newFake() + m.mirror = "000" + r, events := runner(t, m, Options{Addr: "10.0.0.9", MirrorBase: "http://172.17.0.1:8081"}) + var res Result + err := r.check(t.Context(), &res) + if !errors.Is(err, ErrChecks) { + t.Fatalf("a node that cannot reach the mirror was let through: %v", err) + } + found := false + for _, e := range *events { + if e.Level == Fail && strings.Contains(e.Detail, "http://172.17.0.1:8081/") && strings.Contains(e.Fix, "--base") { + found = true + } + } + if !found { + t.Errorf("no failure naming the address and the fix: %+v", *events) + } + + ok := newFake() + ok.mirror = "404" + r, _ = runner(t, ok, Options{Addr: "10.0.0.9", MirrorBase: "http://10.0.0.146:8081"}) + if err := r.check(t.Context(), &res); err != nil { + t.Errorf("a node that reaches the mirror: %v", err) + } +}