The mirror's address is one the nodes reach, and node add checks it

server init and mirror init took the first non-loopback address for the
mirror's --base, Docker's 172.17.0.1 on a machine with Docker; nodes asked
their own bridge for every model and were refused. primaryAddr takes the
default route's source, else the first address on an interface that is not
a container's, bridge's or overlay's. node add has the node reach the
mirror's files during the checks and fails, naming the address and the fix,
when it cannot.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-06 09:36:09 +02:00
1 parent 32c3942040
commit ad04a78343
8 files changed
+166 -7

No files matched your search

+8
View File
@@ -56,6 +56,14 @@ APIs change between minors without a deprecation period.
### Fixed
- **The mirror is given an address the nodes reach**: `server init` and
`mirror init` took the machine's first address for the mirror's `--base`,
which on a machine with Docker could be 172.17.0.1 — a node then asked its
own bridge for every model and was refused. They now take the source of
the default route, else the first address not on a container's or a
bridge's interface. And `node add` has the node reach the mirror's files
before it is added: one that cannot fails the checks, naming the address
and what to change.
- **A standby that becomes the leader reads the store again**: models added
or removed, engine builds chosen and domains held through the leader before
it, rather than placing on what it read when it started.
+1 -4
View File
@@ -89,10 +89,7 @@ func mirrorInit(ctx context.Context, out io.Writer, dir, base string, controlPla
if err := in.chown(dir, true); err != nil {
return err
}
addr := ""
if ips := localAddrs(); len(ips) > 0 {
addr = ips[0].String()
}
addr := primaryAddr()
if base == "" {
if addr == "" {
return errors.New("this machine has no address to give nodes; give one with --base")
+11
View File
@@ -18,6 +18,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
@@ -183,6 +184,16 @@ func (na *nodeAdd) prepare() error {
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
o.FirewallAllow, _ = cfg.FirewallAllow()
}
// Where the mirror serves model files, for the node to be asked to reach.
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && cfg.Mirror.API != "" {
if mc, err := mirror.NewClient(cfg.Mirror.API, cfg.Mirror.TLS, 5*time.Second); err == nil {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if list, err := mc.List(ctx); err == nil {
o.MirrorBase = list.Base
}
cancel()
}
}
// Where the control plane upgrades nodes from, so a node joins on the
// same release from the same place: an offline cluster's mirror.
if o.Releases == "" {
+55
View File
@@ -821,3 +821,58 @@ Before people outside this network use it:
# in server.yaml, and copy them off this machine
`)
}
// primaryAddr is the address other machines most likely reach this one on:
// the source of its default route, else the first address on an interface
// that is not a container's or a bridge's. The first address of all is not
// it on a machine with Docker, whose 172.17.0.1 a node takes for its own.
func primaryAddr() string {
if out, err := exec.Command("ip", "-4", "route", "get", "1.1.1.1").Output(); err == nil {
if src := routeSource(string(out)); src != "" {
return src
}
}
ifaces, err := net.Interfaces()
if err == nil {
for _, ifc := range ifaces {
if ifc.Flags&net.FlagUp == 0 || virtualInterface(ifc.Name) {
continue
}
addrs, _ := ifc.Addrs()
for _, a := range addrs {
if n, ok := a.(*net.IPNet); ok && n.IP.To4() != nil && n.IP.IsGlobalUnicast() {
return n.IP.String()
}
}
}
}
if ips := localAddrs(); len(ips) > 0 {
return ips[0].String()
}
return ""
}
// routeSource reads the src of `ip route get`.
func routeSource(out string) string {
f := strings.Fields(out)
for i := 0; i+1 < len(f); i++ {
if f[i] == "src" {
if ip := net.ParseIP(f[i+1]); ip != nil && !ip.IsLoopback() {
return ip.String()
}
}
}
return ""
}
// virtualInterface is an interface containers, bridges or overlays made,
// whose address no other machine reaches this one on.
func virtualInterface(name string) bool {
for _, p := range []string{"docker", "br-", "virbr", "veth", "cni", "flannel", "cali", "vxlan",
"kube", "lxc", "lxd", "podman", "cilium", "weave", "tun", "tap"} {
if strings.HasPrefix(name, p) {
return true
}
}
return false
}
+2 -2
View File
@@ -86,8 +86,8 @@ func (in *serverInit) reachableAddr() string {
if len(in.hosts) > 0 {
return in.hosts[0]
}
if ips := localAddrs(); len(ips) > 0 {
return ips[0].String()
if a := primaryAddr(); a != "" {
return a
}
name, _ := os.Hostname()
return name
+19
View File
@@ -172,3 +172,22 @@ func TestInitKeepsCertificatesFromAnAuthorityElsewhere(t *testing.T) {
t.Error("--host cannot be honoured without the authority, and should say so")
}
}
func TestTheAddressNodesReachIsNotAContainersBridge(t *testing.T) {
if got := routeSource("1.1.1.1 via 10.104.130.129 dev ens192 src 10.104.130.140 uid 0\n cache\n"); got != "10.104.130.140" {
t.Errorf("route source %q", got)
}
if got := routeSource("unreachable"); got != "" {
t.Errorf("no route: %q", got)
}
for _, name := range []string{"docker0", "br-3f2a", "virbr0", "veth12ab", "cni0", "flannel.1"} {
if !virtualInterface(name) {
t.Errorf("%s taken for a real interface", name)
}
}
for _, name := range []string{"eth0", "ens192", "enp3s0", "bond0", "ib0"} {
if virtualInterface(name) {
t.Errorf("%s taken for a virtual one", name)
}
}
}
+39 -1
View File
@@ -69,7 +69,11 @@ type Options struct {
// NoFirewall leaves the machine's firewall as it is. Otherwise its llmgw
// ports — FirewallPorts — answer only the networks it reaches the
// control planes on, and FirewallAllow.
NoFirewall bool
NoFirewall bool
// MirrorBase is where the mirror serves model files, http://host:8081:
// the node is asked to reach it before it is added, since a node that
// cannot fetches no model.
MirrorBase string
FirewallPorts []firewall.Port
FirewallAllow []netip.Prefix
@@ -461,6 +465,9 @@ func (r *Runner) check(ctx context.Context, res *Result) error {
// A missing firewall is what this run is about to write.
rep.Findings = slices.DeleteFunc(rep.Findings, func(f doctor.Finding) bool { return f.Area == "firewall" })
}
if f, ok := r.mirrorReach(ctx); ok {
rep.Findings = append(rep.Findings, f)
}
res.Facts, res.Findings = rep.Facts, rep.Findings
fails := 0
@@ -502,6 +509,37 @@ func (r *Runner) check(ctx context.Context, res *Result) error {
return nil
}
// mirrorReach asks the node to reach the mirror's files: a base address the
// node cannot reach (another machine's interface, Docker's bridge, a port
// closed to it) otherwise shows up as every model failing to download.
func (r *Runner) mirrorReach(ctx context.Context) (doctor.Finding, bool) {
if r.Opts.MirrorBase == "" {
return doctor.Finding{}, false
}
u, err := url.Parse(r.Opts.MirrorBase)
if err != nil || u.Hostname() == "" {
return doctor.Finding{}, false
}
port := u.Port()
if port == "" {
port = "80"
}
base := strings.TrimRight(r.Opts.MirrorBase, "/") + "/"
out, _ := r.Remote.Run(ctx, `if command -v curl >/dev/null 2>&1; then
curl -s -o /dev/null -w '%{http_code}' --max-time 5 `+shellQuote(base)+` || true
else
timeout 5 bash -c `+shellQuote("</dev/tcp/"+u.Hostname()+"/"+port)+` 2>/dev/null && echo 200 || echo 000
fi`, nil)
code := strings.TrimSpace(lastLine(out))
if code != "" && code != "000" {
return doctor.Finding{Level: doctor.OK, Area: "mirror", Message: "reaches the mirror's files at " + base}, true
}
return doctor.Finding{Level: doctor.Fail, Area: "mirror",
Message: "cannot reach the mirror's files at " + base + ": every model it is given would fail to download",
Fix: "on the mirror's machine, give llmgw-mirror a --base this node reaches (systemctl cat llmgw-mirror), " +
"and let this node through to port " + port}, true
}
// suggestFabrics groups the active adapters by the fabric they report being
// on: the subnet manager they see, or the network of their RoCE address.
// Names are placeholders for the operator: a name is what ties one node's
+31
View File
@@ -32,6 +32,8 @@ type fakeMachine struct {
files map[string][]byte
scripts []string
download error
// mirror is what curl prints for the mirror's address: 000 unreachable.
mirror string
}
type doctorReport struct {
@@ -64,6 +66,8 @@ func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byt
return nil, nil
case strings.Contains(script, "systemctl restart llmgw-agent"):
return []byte("active\n"), nil
case strings.Contains(script, "curl -s -o /dev/null -w"):
return []byte(f.mirror), nil
}
return nil, nil
}
@@ -389,3 +393,30 @@ func TestTheSudoGrantRemovesItselfWithinTheHour(t *testing.T) {
t.Errorf("not shell: %v %s\n%s", err, out, inner)
}
}
func TestANodeThatCannotReachTheMirrorIsNotAdded(t *testing.T) {
m := newFake()
m.mirror = "000"
r, events := runner(t, m, Options{Addr: "10.0.0.9", MirrorBase: "http://172.17.0.1:8081"})
var res Result
err := r.check(t.Context(), &res)
if !errors.Is(err, ErrChecks) {
t.Fatalf("a node that cannot reach the mirror was let through: %v", err)
}
found := false
for _, e := range *events {
if e.Level == Fail && strings.Contains(e.Detail, "http://172.17.0.1:8081/") && strings.Contains(e.Fix, "--base") {
found = true
}
}
if !found {
t.Errorf("no failure naming the address and the fix: %+v", *events)
}
ok := newFake()
ok.mirror = "404"
r, _ = runner(t, ok, Options{Addr: "10.0.0.9", MirrorBase: "http://10.0.0.146:8081"})
if err := r.check(t.Context(), &res); err != nil {
t.Errorf("a node that reaches the mirror: %v", err)
}
}