The mirror's address is one the nodes reach, and node add checks it
server init and mirror init took the first non-loopback address for the mirror's --base, Docker's 172.17.0.1 on a machine with Docker; nodes asked their own bridge for every model and were refused. primaryAddr takes the default route's source, else the first address on an interface that is not a container's, bridge's or overlay's. node add has the node reach the mirror's files during the checks and fails, naming the address and the fix, when it cannot. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
32c3942040
commit
ad04a78343
8 files changed
+166
-7
No files matched your search
@@ -89,10 +89,7 @@ func mirrorInit(ctx context.Context, out io.Writer, dir, base string, controlPla
|
||||
if err := in.chown(dir, true); err != nil {
|
||||
return err
|
||||
}
|
||||
addr := ""
|
||||
if ips := localAddrs(); len(ips) > 0 {
|
||||
addr = ips[0].String()
|
||||
}
|
||||
addr := primaryAddr()
|
||||
if base == "" {
|
||||
if addr == "" {
|
||||
return errors.New("this machine has no address to give nodes; give one with --base")
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
@@ -183,6 +184,16 @@ func (na *nodeAdd) prepare() error {
|
||||
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
|
||||
o.FirewallAllow, _ = cfg.FirewallAllow()
|
||||
}
|
||||
// Where the mirror serves model files, for the node to be asked to reach.
|
||||
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && cfg.Mirror.API != "" {
|
||||
if mc, err := mirror.NewClient(cfg.Mirror.API, cfg.Mirror.TLS, 5*time.Second); err == nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if list, err := mc.List(ctx); err == nil {
|
||||
o.MirrorBase = list.Base
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
// Where the control plane upgrades nodes from, so a node joins on the
|
||||
// same release from the same place: an offline cluster's mirror.
|
||||
if o.Releases == "" {
|
||||
|
||||
@@ -821,3 +821,58 @@ Before people outside this network use it:
|
||||
# in server.yaml, and copy them off this machine
|
||||
`)
|
||||
}
|
||||
|
||||
// primaryAddr is the address other machines most likely reach this one on:
|
||||
// the source of its default route, else the first address on an interface
|
||||
// that is not a container's or a bridge's. The first address of all is not
|
||||
// it on a machine with Docker, whose 172.17.0.1 a node takes for its own.
|
||||
func primaryAddr() string {
|
||||
if out, err := exec.Command("ip", "-4", "route", "get", "1.1.1.1").Output(); err == nil {
|
||||
if src := routeSource(string(out)); src != "" {
|
||||
return src
|
||||
}
|
||||
}
|
||||
ifaces, err := net.Interfaces()
|
||||
if err == nil {
|
||||
for _, ifc := range ifaces {
|
||||
if ifc.Flags&net.FlagUp == 0 || virtualInterface(ifc.Name) {
|
||||
continue
|
||||
}
|
||||
addrs, _ := ifc.Addrs()
|
||||
for _, a := range addrs {
|
||||
if n, ok := a.(*net.IPNet); ok && n.IP.To4() != nil && n.IP.IsGlobalUnicast() {
|
||||
return n.IP.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ips := localAddrs(); len(ips) > 0 {
|
||||
return ips[0].String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// routeSource reads the src of `ip route get`.
|
||||
func routeSource(out string) string {
|
||||
f := strings.Fields(out)
|
||||
for i := 0; i+1 < len(f); i++ {
|
||||
if f[i] == "src" {
|
||||
if ip := net.ParseIP(f[i+1]); ip != nil && !ip.IsLoopback() {
|
||||
return ip.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// virtualInterface is an interface containers, bridges or overlays made,
|
||||
// whose address no other machine reaches this one on.
|
||||
func virtualInterface(name string) bool {
|
||||
for _, p := range []string{"docker", "br-", "virbr", "veth", "cni", "flannel", "cali", "vxlan",
|
||||
"kube", "lxc", "lxd", "podman", "cilium", "weave", "tun", "tap"} {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -86,8 +86,8 @@ func (in *serverInit) reachableAddr() string {
|
||||
if len(in.hosts) > 0 {
|
||||
return in.hosts[0]
|
||||
}
|
||||
if ips := localAddrs(); len(ips) > 0 {
|
||||
return ips[0].String()
|
||||
if a := primaryAddr(); a != "" {
|
||||
return a
|
||||
}
|
||||
name, _ := os.Hostname()
|
||||
return name
|
||||
|
||||
@@ -172,3 +172,22 @@ func TestInitKeepsCertificatesFromAnAuthorityElsewhere(t *testing.T) {
|
||||
t.Error("--host cannot be honoured without the authority, and should say so")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAddressNodesReachIsNotAContainersBridge(t *testing.T) {
|
||||
if got := routeSource("1.1.1.1 via 10.104.130.129 dev ens192 src 10.104.130.140 uid 0\n cache\n"); got != "10.104.130.140" {
|
||||
t.Errorf("route source %q", got)
|
||||
}
|
||||
if got := routeSource("unreachable"); got != "" {
|
||||
t.Errorf("no route: %q", got)
|
||||
}
|
||||
for _, name := range []string{"docker0", "br-3f2a", "virbr0", "veth12ab", "cni0", "flannel.1"} {
|
||||
if !virtualInterface(name) {
|
||||
t.Errorf("%s taken for a real interface", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"eth0", "ens192", "enp3s0", "bond0", "ib0"} {
|
||||
if virtualInterface(name) {
|
||||
t.Errorf("%s taken for a virtual one", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user