The mirror's address is one the nodes reach, and node add checks it

server init and mirror init took the first non-loopback address for the
mirror's --base, Docker's 172.17.0.1 on a machine with Docker; nodes asked
their own bridge for every model and were refused. primaryAddr takes the
default route's source, else the first address on an interface that is not
a container's, bridge's or overlay's. node add has the node reach the
mirror's files during the checks and fails, naming the address and the fix,
when it cannot.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-06 09:36:09 +02:00
1 parent 32c3942040
commit ad04a78343
8 files changed
+166 -7

No files matched your search

+1 -4
View File
@@ -89,10 +89,7 @@ func mirrorInit(ctx context.Context, out io.Writer, dir, base string, controlPla
if err := in.chown(dir, true); err != nil {
return err
}
addr := ""
if ips := localAddrs(); len(ips) > 0 {
addr = ips[0].String()
}
addr := primaryAddr()
if base == "" {
if addr == "" {
return errors.New("this machine has no address to give nodes; give one with --base")
+11
View File
@@ -18,6 +18,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
@@ -183,6 +184,16 @@ func (na *nodeAdd) prepare() error {
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
o.FirewallAllow, _ = cfg.FirewallAllow()
}
// Where the mirror serves model files, for the node to be asked to reach.
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && cfg.Mirror.API != "" {
if mc, err := mirror.NewClient(cfg.Mirror.API, cfg.Mirror.TLS, 5*time.Second); err == nil {
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
if list, err := mc.List(ctx); err == nil {
o.MirrorBase = list.Base
}
cancel()
}
}
// Where the control plane upgrades nodes from, so a node joins on the
// same release from the same place: an offline cluster's mirror.
if o.Releases == "" {
+55
View File
@@ -821,3 +821,58 @@ Before people outside this network use it:
# in server.yaml, and copy them off this machine
`)
}
// primaryAddr is the address other machines most likely reach this one on:
// the source of its default route, else the first address on an interface
// that is not a container's or a bridge's. The first address of all is not
// it on a machine with Docker, whose 172.17.0.1 a node takes for its own.
func primaryAddr() string {
if out, err := exec.Command("ip", "-4", "route", "get", "1.1.1.1").Output(); err == nil {
if src := routeSource(string(out)); src != "" {
return src
}
}
ifaces, err := net.Interfaces()
if err == nil {
for _, ifc := range ifaces {
if ifc.Flags&net.FlagUp == 0 || virtualInterface(ifc.Name) {
continue
}
addrs, _ := ifc.Addrs()
for _, a := range addrs {
if n, ok := a.(*net.IPNet); ok && n.IP.To4() != nil && n.IP.IsGlobalUnicast() {
return n.IP.String()
}
}
}
}
if ips := localAddrs(); len(ips) > 0 {
return ips[0].String()
}
return ""
}
// routeSource reads the src of `ip route get`.
func routeSource(out string) string {
f := strings.Fields(out)
for i := 0; i+1 < len(f); i++ {
if f[i] == "src" {
if ip := net.ParseIP(f[i+1]); ip != nil && !ip.IsLoopback() {
return ip.String()
}
}
}
return ""
}
// virtualInterface is an interface containers, bridges or overlays made,
// whose address no other machine reaches this one on.
func virtualInterface(name string) bool {
for _, p := range []string{"docker", "br-", "virbr", "veth", "cni", "flannel", "cali", "vxlan",
"kube", "lxc", "lxd", "podman", "cilium", "weave", "tun", "tap"} {
if strings.HasPrefix(name, p) {
return true
}
}
return false
}
+2 -2
View File
@@ -86,8 +86,8 @@ func (in *serverInit) reachableAddr() string {
if len(in.hosts) > 0 {
return in.hosts[0]
}
if ips := localAddrs(); len(ips) > 0 {
return ips[0].String()
if a := primaryAddr(); a != "" {
return a
}
name, _ := os.Hostname()
return name
+19
View File
@@ -172,3 +172,22 @@ func TestInitKeepsCertificatesFromAnAuthorityElsewhere(t *testing.T) {
t.Error("--host cannot be honoured without the authority, and should say so")
}
}
func TestTheAddressNodesReachIsNotAContainersBridge(t *testing.T) {
if got := routeSource("1.1.1.1 via 10.104.130.129 dev ens192 src 10.104.130.140 uid 0\n cache\n"); got != "10.104.130.140" {
t.Errorf("route source %q", got)
}
if got := routeSource("unreachable"); got != "" {
t.Errorf("no route: %q", got)
}
for _, name := range []string{"docker0", "br-3f2a", "virbr0", "veth12ab", "cni0", "flannel.1"} {
if !virtualInterface(name) {
t.Errorf("%s taken for a real interface", name)
}
}
for _, name := range []string{"eth0", "ens192", "enp3s0", "bond0", "ib0"} {
if virtualInterface(name) {
t.Errorf("%s taken for a virtual one", name)
}
}
}