A node's own readiness and its multi-node readiness are kept apart

doctor sums findings up as two readinesses - the node's, which decides
whether it joins, and multi-node, its fabrics for replicas spanning nodes -
prints fabric findings in a section of their own, and gives both in --json.
node add says the multi-node one. NodeView.multi_node says whether a node's
declared fabrics are up, naming adapters that are down, and ctl nodes shows
it beside the fabric.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

docs/admin-api.openapi.json

@@ -5038,6 +5038,10 @@
             },
             "description": "EngineVersions are the installed engines with their versions: llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute capabilities, 7.0 for a V100, so an engine build's support can be read against them."
           },
+          "multi_node": {
+            "type": "string",
+            "description": "MultiNode is whether it can take part in replicas spanning nodes: \"ready\", \"down: mlx5_1\" naming the declared adapters whose ports are not active, or empty for a node on no fabric. Its own replicas run either way."
+          },
           "compute_capabilities": {
             "type": "array",
             "items": {

internal/cli/ctl.go

@@ -233,7 +233,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
 					// at when a model will not place and the hardware looks
 					// fine.
 					enginesColumn(n),
-					classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
+					classesColumn(n), fabricsColumn(n), n.Replicas, n.LastSeenAgo)
 			}
 			_ = w.Flush()
 
@@ -596,6 +596,16 @@ func emitJSON(v any) error {
 // more than one or MIG instances among them: a card cut into two 3g.40gb
 // instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where
 // the GPUS column's 3 alone would read as three of something.
+// fabricsColumn is the node's fabrics, and when they are not all up, which
+// adapters are down: what keeps it out of replicas spanning nodes.
+func fabricsColumn(n adminapi.NodeView) string {
+	s := join(n.Fabrics)
+	if n.MultiNode != "" && n.MultiNode != "ready" {
+		s += " (" + n.MultiNode + ")"
+	}
+	return s
+}
+
 // enginesColumn is each engine with its version: llamacpp b10991.
 func enginesColumn(n adminapi.NodeView) string {
 	if len(n.EngineVersions) == 0 {

internal/cli/doctor.go

@@ -63,9 +63,10 @@ that file when the report is not enough to go on.`,
 				enc := json.NewEncoder(os.Stdout)
 				enc.SetIndent("", "  ")
 				return enc.Encode(struct {
-					Findings []doctor.Finding `json:"findings"`
-					Facts    doctor.Facts     `json:"facts"`
-				}{findings, facts})
+					Findings  []doctor.Finding `json:"findings"`
+					Readiness doctor.Readiness `json:"readiness"`
+					Facts     doctor.Facts     `json:"facts"`
+				}{findings, doctor.ReadinessOf(findings), facts})
 			}
 			printDoctor(os.Stdout, facts, findings, verbose)
 			for _, f := range findings {
@@ -101,12 +102,26 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo
 	fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion,
 		f.Host.Arch, orDash(f.Host.NVIDIADriver))
 
-	for _, x := range findings {
-		fmt.Fprintf(out, "%-4s  %-6s  %s\n", x.Level, x.Area, x.Message)
-		if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
-			fmt.Fprintf(out, "              → %s\n", x.Fix)
+	// The node first; then what replicas spanning nodes would need, which
+	// keeps a node out only when a fabric was declared for it.
+	multi := false
+	for _, pass := range []bool{false, true} {
+		for _, x := range findings {
+			if x.MultiNode() != pass {
+				continue
+			}
+			if pass && !multi {
+				fmt.Fprintln(out, "\nMulti-node (fabrics, RDMA) — for replicas spanning nodes only:")
+				multi = true
+			}
+			fmt.Fprintf(out, "%-4s  %-6s  %s\n", x.Level, x.Area, x.Message)
+			if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
+				fmt.Fprintf(out, "              → %s\n", x.Fix)
+			}
 		}
 	}
+	r := doctor.ReadinessOf(findings)
+	fmt.Fprintf(out, "\nThis node:  %s\nMulti-node: %s\n", r.Node, r.MultiNode)
 
 	w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
 	if len(f.GPUs) > 0 {

internal/control/adminapi/api.go

@@ -233,8 +233,13 @@ type NodeView struct {
 	// llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute
 	// capabilities, 7.0 for a V100, so an engine build's support can be read
 	// against them.
-	EngineVersions      map[string]string `json:"engine_versions,omitempty"`
-	ComputeCapabilities []string          `json:"compute_capabilities,omitempty"`
+	EngineVersions map[string]string `json:"engine_versions,omitempty"`
+	// MultiNode is whether it can take part in replicas spanning nodes:
+	// "ready", "down: mlx5_1" naming the declared adapters whose ports are
+	// not active, or empty for a node on no fabric. Its own replicas run
+	// either way.
+	MultiNode           string   `json:"multi_node,omitempty"`
+	ComputeCapabilities []string `json:"compute_capabilities,omitempty"`
 	// Provision is where installing each wanted engine stands. An engine is in
 	// Engines only once its installation here is ready.
 	Provision []inventory.EngineInstall `json:"provision,omitempty"`
@@ -663,6 +668,7 @@ func (a *API) Nodes() []NodeView {
 			}
 		}
 		slices.Sort(v.ComputeCapabilities)
+		v.MultiNode = multiNodeReadiness(n)
 		v.Provision = n.Provision
 		if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 {
 			chosen := ch.For(n.ID)
@@ -2026,3 +2032,32 @@ func (a *API) SetNodeEngine(ctx context.Context, node, engine, build, by string)
 	}
 	return ch.Set(ctx, node, engine, build, by)
 }
+
+// multiNodeReadiness is whether a node's declared fabrics are up: every
+// adapter they name with an active port. Empty for a node on no fabric.
+func multiNodeReadiness(n inventory.Node) string {
+	if len(n.Fabrics) == 0 {
+		return ""
+	}
+	if len(n.IBPorts) == 0 {
+		return "unknown: no port reported yet"
+	}
+	var down []string
+	for _, f := range n.Fabrics {
+		for _, hca := range f.HCAs {
+			up := false
+			for _, p := range n.IBPorts {
+				if p.HCA == hca && p.Active() {
+					up = true
+				}
+			}
+			if !up && !slices.Contains(down, hca) {
+				down = append(down, hca)
+			}
+		}
+	}
+	if len(down) > 0 {
+		return "down: " + strings.Join(down, ", ")
+	}
+	return "ready"
+}

internal/control/adminapi/judges_test.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
 )
 
@@ -104,3 +105,21 @@ func TestANodeSaysItsEnginesVersionsAndCapabilities(t *testing.T) {
 		t.Errorf("%+v", nodes)
 	}
 }
+
+func TestANodeSaysWhetherItsFabricsAreUp(t *testing.T) {
+	n := gpuNode("ib1", 2, "ib-a")
+	if got := multiNodeReadiness(n); !strings.HasPrefix(got, "unknown") {
+		t.Errorf("no port yet: %q", got)
+	}
+	n.IBPorts = []inventory.IBPort{{HCA: "mlx5_0", State: "DOWN"}}
+	if got := multiNodeReadiness(n); got != "down: mlx5_0" {
+		t.Errorf("down: %q", got)
+	}
+	n.IBPorts[0].State = "ACTIVE"
+	if got := multiNodeReadiness(n); got != "ready" {
+		t.Errorf("up: %q", got)
+	}
+	if got := multiNodeReadiness(gpuNode("plain", 1)); got != "" {
+		t.Errorf("no fabric: %q", got)
+	}
+}

internal/doctor/doctor.go

@@ -819,3 +819,45 @@ func sortedKeys[V any](m map[string]V) []string {
 	sort.Strings(out)
 	return out
 }
+
+// MultiNode is whether a finding is about the fabrics replicas spanning
+// nodes use, rather than about the node itself.
+func (f Finding) MultiNode() bool { return f.Area == "fabric" }
+
+// Readiness is a node's two readinesses, kept apart: whether it can join and
+// serve replicas of its own, and whether it can also take part in replicas
+// spanning nodes over a fabric. The second never keeps a node out unless a
+// fabric was declared for it, whose problems are then the node's.
+type Readiness struct {
+	// Node is "ready", or how many problems keep it out.
+	Node string `json:"node"`
+	// MultiNode is "ready", "not used" (no fabric adapter), or "not ready:"
+	// and the first reason.
+	MultiNode string `json:"multi_node"`
+}
+
+// ReadinessOf sums findings up as the two readinesses.
+func ReadinessOf(fs []Finding) Readiness {
+	r := Readiness{Node: "ready", MultiNode: "not used"}
+	fails := 0
+	for _, f := range fs {
+		if f.Level == Fail {
+			fails++
+		}
+	}
+	if fails > 0 {
+		r.Node = fmt.Sprintf("%d problem(s) keep it out", fails)
+	}
+	for _, f := range fs {
+		if !f.MultiNode() {
+			continue
+		}
+		if r.MultiNode == "not used" {
+			r.MultiNode = "ready"
+		}
+		if (f.Level == Fail || f.Level == Warn) && !strings.HasPrefix(r.MultiNode, "not ready") {
+			r.MultiNode = "not ready: " + f.Message
+		}
+	}
+	return r
+}

internal/doctor/doctor_test.go

@@ -259,3 +259,20 @@ func TestAMissingFirewallIsSaidOnASetUpNode(t *testing.T) {
 		}
 	}
 }
+
+func TestTheNodeAndItsFabricsAreReadySeparately(t *testing.T) {
+	r := ReadinessOf([]Finding{
+		{Level: OK, Area: "gpu", Message: "2 GPUs"},
+		{Level: Warn, Area: "fabric", Message: "mlx5_1 is not active"},
+	})
+	if r.Node != "ready" || r.MultiNode != "not ready: mlx5_1 is not active" {
+		t.Errorf("%+v", r)
+	}
+	r = ReadinessOf([]Finding{{Level: Fail, Area: "gpu", Message: "no driver"}})
+	if !strings.Contains(r.Node, "1 problem") || r.MultiNode != "not used" {
+		t.Errorf("%+v", r)
+	}
+	if r := ReadinessOf([]Finding{{Level: OK, Area: "fabric", Message: "mlx5_0 active"}}); r.MultiNode != "ready" {
+		t.Errorf("%+v", r)
+	}
+}

internal/onboard/onboard.go

@@ -505,6 +505,9 @@ func (r *Runner) check(ctx context.Context, res *Result) error {
 	if fails > 0 && !goOn {
 		return fmt.Errorf("%w: doctor found %d; fix them and run this again, or add --ignore-checks", ErrChecks, fails)
 	}
+	if rd := doctor.ReadinessOf(rep.Findings); rd.MultiNode != "not used" {
+		r.emit(StepCheck, Info, "multi-node: "+rd.MultiNode+"; replicas of its own run either way", "")
+	}
 	r.emit(StepCheck, Done, fmt.Sprintf("%d problem(s), %d warning(s)", fails, countLevel(rep.Findings, doctor.Warn)), "")
 	return nil
 }