The mirror's address is one the nodes reach, and node add checks it
server init and mirror init took the first non-loopback address for the mirror's --base, Docker's 172.17.0.1 on a machine with Docker; nodes asked their own bridge for every model and were refused. primaryAddr takes the default route's source, else the first address on an interface that is not a container's, bridge's or overlay's. node add has the node reach the mirror's files during the checks and fails, naming the address and the fix, when it cannot. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
32c3942040
commit
ad04a78343
8 files changed
+166
-7
No files matched your search
@@ -56,6 +56,14 @@ APIs change between minors without a deprecation period.
|
||||
|
||||
### Fixed
|
||||
|
||||
- **The mirror is given an address the nodes reach**: `server init` and
|
||||
`mirror init` took the machine's first address for the mirror's `--base`,
|
||||
which on a machine with Docker could be 172.17.0.1 — a node then asked its
|
||||
own bridge for every model and was refused. They now take the source of
|
||||
the default route, else the first address not on a container's or a
|
||||
bridge's interface. And `node add` has the node reach the mirror's files
|
||||
before it is added: one that cannot fails the checks, naming the address
|
||||
and what to change.
|
||||
- **A standby that becomes the leader reads the store again**: models added
|
||||
or removed, engine builds chosen and domains held through the leader before
|
||||
it, rather than placing on what it read when it started.
|
||||
|
||||
@@ -89,10 +89,7 @@ func mirrorInit(ctx context.Context, out io.Writer, dir, base string, controlPla
|
||||
if err := in.chown(dir, true); err != nil {
|
||||
return err
|
||||
}
|
||||
addr := ""
|
||||
if ips := localAddrs(); len(ips) > 0 {
|
||||
addr = ips[0].String()
|
||||
}
|
||||
addr := primaryAddr()
|
||||
if base == "" {
|
||||
if addr == "" {
|
||||
return errors.New("this machine has no address to give nodes; give one with --base")
|
||||
|
||||
@@ -18,6 +18,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/firewall"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
@@ -183,6 +184,16 @@ func (na *nodeAdd) prepare() error {
|
||||
o.FirewallPorts = []firewall.Port{firewall.Engines(from, to), firewall.NodeExporter}
|
||||
o.FirewallAllow, _ = cfg.FirewallAllow()
|
||||
}
|
||||
// Where the mirror serves model files, for the node to be asked to reach.
|
||||
if cfg, err := loadEffectiveConfig(na.serverCfg); err == nil && cfg.Mirror.API != "" {
|
||||
if mc, err := mirror.NewClient(cfg.Mirror.API, cfg.Mirror.TLS, 5*time.Second); err == nil {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||
if list, err := mc.List(ctx); err == nil {
|
||||
o.MirrorBase = list.Base
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
// Where the control plane upgrades nodes from, so a node joins on the
|
||||
// same release from the same place: an offline cluster's mirror.
|
||||
if o.Releases == "" {
|
||||
|
||||
@@ -821,3 +821,58 @@ Before people outside this network use it:
|
||||
# in server.yaml, and copy them off this machine
|
||||
`)
|
||||
}
|
||||
|
||||
// primaryAddr is the address other machines most likely reach this one on:
|
||||
// the source of its default route, else the first address on an interface
|
||||
// that is not a container's or a bridge's. The first address of all is not
|
||||
// it on a machine with Docker, whose 172.17.0.1 a node takes for its own.
|
||||
func primaryAddr() string {
|
||||
if out, err := exec.Command("ip", "-4", "route", "get", "1.1.1.1").Output(); err == nil {
|
||||
if src := routeSource(string(out)); src != "" {
|
||||
return src
|
||||
}
|
||||
}
|
||||
ifaces, err := net.Interfaces()
|
||||
if err == nil {
|
||||
for _, ifc := range ifaces {
|
||||
if ifc.Flags&net.FlagUp == 0 || virtualInterface(ifc.Name) {
|
||||
continue
|
||||
}
|
||||
addrs, _ := ifc.Addrs()
|
||||
for _, a := range addrs {
|
||||
if n, ok := a.(*net.IPNet); ok && n.IP.To4() != nil && n.IP.IsGlobalUnicast() {
|
||||
return n.IP.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if ips := localAddrs(); len(ips) > 0 {
|
||||
return ips[0].String()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// routeSource reads the src of `ip route get`.
|
||||
func routeSource(out string) string {
|
||||
f := strings.Fields(out)
|
||||
for i := 0; i+1 < len(f); i++ {
|
||||
if f[i] == "src" {
|
||||
if ip := net.ParseIP(f[i+1]); ip != nil && !ip.IsLoopback() {
|
||||
return ip.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// virtualInterface is an interface containers, bridges or overlays made,
|
||||
// whose address no other machine reaches this one on.
|
||||
func virtualInterface(name string) bool {
|
||||
for _, p := range []string{"docker", "br-", "virbr", "veth", "cni", "flannel", "cali", "vxlan",
|
||||
"kube", "lxc", "lxd", "podman", "cilium", "weave", "tun", "tap"} {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -86,8 +86,8 @@ func (in *serverInit) reachableAddr() string {
|
||||
if len(in.hosts) > 0 {
|
||||
return in.hosts[0]
|
||||
}
|
||||
if ips := localAddrs(); len(ips) > 0 {
|
||||
return ips[0].String()
|
||||
if a := primaryAddr(); a != "" {
|
||||
return a
|
||||
}
|
||||
name, _ := os.Hostname()
|
||||
return name
|
||||
|
||||
@@ -172,3 +172,22 @@ func TestInitKeepsCertificatesFromAnAuthorityElsewhere(t *testing.T) {
|
||||
t.Error("--host cannot be honoured without the authority, and should say so")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheAddressNodesReachIsNotAContainersBridge(t *testing.T) {
|
||||
if got := routeSource("1.1.1.1 via 10.104.130.129 dev ens192 src 10.104.130.140 uid 0\n cache\n"); got != "10.104.130.140" {
|
||||
t.Errorf("route source %q", got)
|
||||
}
|
||||
if got := routeSource("unreachable"); got != "" {
|
||||
t.Errorf("no route: %q", got)
|
||||
}
|
||||
for _, name := range []string{"docker0", "br-3f2a", "virbr0", "veth12ab", "cni0", "flannel.1"} {
|
||||
if !virtualInterface(name) {
|
||||
t.Errorf("%s taken for a real interface", name)
|
||||
}
|
||||
}
|
||||
for _, name := range []string{"eth0", "ens192", "enp3s0", "bond0", "ib0"} {
|
||||
if virtualInterface(name) {
|
||||
t.Errorf("%s taken for a virtual one", name)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -69,7 +69,11 @@ type Options struct {
|
||||
// NoFirewall leaves the machine's firewall as it is. Otherwise its llmgw
|
||||
// ports — FirewallPorts — answer only the networks it reaches the
|
||||
// control planes on, and FirewallAllow.
|
||||
NoFirewall bool
|
||||
NoFirewall bool
|
||||
// MirrorBase is where the mirror serves model files, http://host:8081:
|
||||
// the node is asked to reach it before it is added, since a node that
|
||||
// cannot fetches no model.
|
||||
MirrorBase string
|
||||
FirewallPorts []firewall.Port
|
||||
FirewallAllow []netip.Prefix
|
||||
|
||||
@@ -461,6 +465,9 @@ func (r *Runner) check(ctx context.Context, res *Result) error {
|
||||
// A missing firewall is what this run is about to write.
|
||||
rep.Findings = slices.DeleteFunc(rep.Findings, func(f doctor.Finding) bool { return f.Area == "firewall" })
|
||||
}
|
||||
if f, ok := r.mirrorReach(ctx); ok {
|
||||
rep.Findings = append(rep.Findings, f)
|
||||
}
|
||||
res.Facts, res.Findings = rep.Facts, rep.Findings
|
||||
|
||||
fails := 0
|
||||
@@ -502,6 +509,37 @@ func (r *Runner) check(ctx context.Context, res *Result) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// mirrorReach asks the node to reach the mirror's files: a base address the
|
||||
// node cannot reach (another machine's interface, Docker's bridge, a port
|
||||
// closed to it) otherwise shows up as every model failing to download.
|
||||
func (r *Runner) mirrorReach(ctx context.Context) (doctor.Finding, bool) {
|
||||
if r.Opts.MirrorBase == "" {
|
||||
return doctor.Finding{}, false
|
||||
}
|
||||
u, err := url.Parse(r.Opts.MirrorBase)
|
||||
if err != nil || u.Hostname() == "" {
|
||||
return doctor.Finding{}, false
|
||||
}
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "80"
|
||||
}
|
||||
base := strings.TrimRight(r.Opts.MirrorBase, "/") + "/"
|
||||
out, _ := r.Remote.Run(ctx, `if command -v curl >/dev/null 2>&1; then
|
||||
curl -s -o /dev/null -w '%{http_code}' --max-time 5 `+shellQuote(base)+` || true
|
||||
else
|
||||
timeout 5 bash -c `+shellQuote("</dev/tcp/"+u.Hostname()+"/"+port)+` 2>/dev/null && echo 200 || echo 000
|
||||
fi`, nil)
|
||||
code := strings.TrimSpace(lastLine(out))
|
||||
if code != "" && code != "000" {
|
||||
return doctor.Finding{Level: doctor.OK, Area: "mirror", Message: "reaches the mirror's files at " + base}, true
|
||||
}
|
||||
return doctor.Finding{Level: doctor.Fail, Area: "mirror",
|
||||
Message: "cannot reach the mirror's files at " + base + ": every model it is given would fail to download",
|
||||
Fix: "on the mirror's machine, give llmgw-mirror a --base this node reaches (systemctl cat llmgw-mirror), " +
|
||||
"and let this node through to port " + port}, true
|
||||
}
|
||||
|
||||
// suggestFabrics groups the active adapters by the fabric they report being
|
||||
// on: the subnet manager they see, or the network of their RoCE address.
|
||||
// Names are placeholders for the operator: a name is what ties one node's
|
||||
|
||||
@@ -32,6 +32,8 @@ type fakeMachine struct {
|
||||
files map[string][]byte
|
||||
scripts []string
|
||||
download error
|
||||
// mirror is what curl prints for the mirror's address: 000 unreachable.
|
||||
mirror string
|
||||
}
|
||||
|
||||
type doctorReport struct {
|
||||
@@ -64,6 +66,8 @@ func (f *fakeMachine) Run(_ context.Context, script string, stdin []byte) ([]byt
|
||||
return nil, nil
|
||||
case strings.Contains(script, "systemctl restart llmgw-agent"):
|
||||
return []byte("active\n"), nil
|
||||
case strings.Contains(script, "curl -s -o /dev/null -w"):
|
||||
return []byte(f.mirror), nil
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
@@ -389,3 +393,30 @@ func TestTheSudoGrantRemovesItselfWithinTheHour(t *testing.T) {
|
||||
t.Errorf("not shell: %v %s\n%s", err, out, inner)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeThatCannotReachTheMirrorIsNotAdded(t *testing.T) {
|
||||
m := newFake()
|
||||
m.mirror = "000"
|
||||
r, events := runner(t, m, Options{Addr: "10.0.0.9", MirrorBase: "http://172.17.0.1:8081"})
|
||||
var res Result
|
||||
err := r.check(t.Context(), &res)
|
||||
if !errors.Is(err, ErrChecks) {
|
||||
t.Fatalf("a node that cannot reach the mirror was let through: %v", err)
|
||||
}
|
||||
found := false
|
||||
for _, e := range *events {
|
||||
if e.Level == Fail && strings.Contains(e.Detail, "http://172.17.0.1:8081/") && strings.Contains(e.Fix, "--base") {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Errorf("no failure naming the address and the fix: %+v", *events)
|
||||
}
|
||||
|
||||
ok := newFake()
|
||||
ok.mirror = "404"
|
||||
r, _ = runner(t, ok, Options{Addr: "10.0.0.9", MirrorBase: "http://10.0.0.146:8081"})
|
||||
if err := r.check(t.Context(), &res); err != nil {
|
||||
t.Errorf("a node that reaches the mirror: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user