diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 2df4bd12..b4d6ebae 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -5038,6 +5038,10 @@ }, "description": "EngineVersions are the installed engines with their versions: llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute capabilities, 7.0 for a V100, so an engine build's support can be read against them." }, + "multi_node": { + "type": "string", + "description": "MultiNode is whether it can take part in replicas spanning nodes: \"ready\", \"down: mlx5_1\" naming the declared adapters whose ports are not active, or empty for a node on no fabric. Its own replicas run either way." + }, "compute_capabilities": { "type": "array", "items": { diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index d1e02975..00b48103 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -233,7 +233,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { // at when a model will not place and the hardware looks // fine. enginesColumn(n), - classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo) + classesColumn(n), fabricsColumn(n), n.Replicas, n.LastSeenAgo) } _ = w.Flush() @@ -596,6 +596,16 @@ func emitJSON(v any) error { // more than one or MIG instances among them: a card cut into two 3g.40gb // instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where // the GPUS column's 3 alone would read as three of something. +// fabricsColumn is the node's fabrics, and when they are not all up, which +// adapters are down: what keeps it out of replicas spanning nodes. +func fabricsColumn(n adminapi.NodeView) string { + s := join(n.Fabrics) + if n.MultiNode != "" && n.MultiNode != "ready" { + s += " (" + n.MultiNode + ")" + } + return s +} + // enginesColumn is each engine with its version: llamacpp b10991. func enginesColumn(n adminapi.NodeView) string { if len(n.EngineVersions) == 0 { diff --git a/internal/cli/doctor.go b/internal/cli/doctor.go index 13d30fb7..2999483c 100644 --- a/internal/cli/doctor.go +++ b/internal/cli/doctor.go @@ -63,9 +63,10 @@ that file when the report is not enough to go on.`, enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") return enc.Encode(struct { - Findings []doctor.Finding `json:"findings"` - Facts doctor.Facts `json:"facts"` - }{findings, facts}) + Findings []doctor.Finding `json:"findings"` + Readiness doctor.Readiness `json:"readiness"` + Facts doctor.Facts `json:"facts"` + }{findings, doctor.ReadinessOf(findings), facts}) } printDoctor(os.Stdout, facts, findings, verbose) for _, f := range findings { @@ -101,12 +102,26 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion, f.Host.Arch, orDash(f.Host.NVIDIADriver)) - for _, x := range findings { - fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message) - if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) { - fmt.Fprintf(out, " → %s\n", x.Fix) + // The node first; then what replicas spanning nodes would need, which + // keeps a node out only when a fabric was declared for it. + multi := false + for _, pass := range []bool{false, true} { + for _, x := range findings { + if x.MultiNode() != pass { + continue + } + if pass && !multi { + fmt.Fprintln(out, "\nMulti-node (fabrics, RDMA) — for replicas spanning nodes only:") + multi = true + } + fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message) + if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) { + fmt.Fprintf(out, " → %s\n", x.Fix) + } } } + r := doctor.ReadinessOf(findings) + fmt.Fprintf(out, "\nThis node: %s\nMulti-node: %s\n", r.Node, r.MultiNode) w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0) if len(f.GPUs) > 0 { diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 87fe5e43..ac217263 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -233,8 +233,13 @@ type NodeView struct { // llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute // capabilities, 7.0 for a V100, so an engine build's support can be read // against them. - EngineVersions map[string]string `json:"engine_versions,omitempty"` - ComputeCapabilities []string `json:"compute_capabilities,omitempty"` + EngineVersions map[string]string `json:"engine_versions,omitempty"` + // MultiNode is whether it can take part in replicas spanning nodes: + // "ready", "down: mlx5_1" naming the declared adapters whose ports are + // not active, or empty for a node on no fabric. Its own replicas run + // either way. + MultiNode string `json:"multi_node,omitempty"` + ComputeCapabilities []string `json:"compute_capabilities,omitempty"` // Provision is where installing each wanted engine stands. An engine is in // Engines only once its installation here is ready. Provision []inventory.EngineInstall `json:"provision,omitempty"` @@ -663,6 +668,7 @@ func (a *API) Nodes() []NodeView { } } slices.Sort(v.ComputeCapabilities) + v.MultiNode = multiNodeReadiness(n) v.Provision = n.Provision if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 { chosen := ch.For(n.ID) @@ -2026,3 +2032,32 @@ func (a *API) SetNodeEngine(ctx context.Context, node, engine, build, by string) } return ch.Set(ctx, node, engine, build, by) } + +// multiNodeReadiness is whether a node's declared fabrics are up: every +// adapter they name with an active port. Empty for a node on no fabric. +func multiNodeReadiness(n inventory.Node) string { + if len(n.Fabrics) == 0 { + return "" + } + if len(n.IBPorts) == 0 { + return "unknown: no port reported yet" + } + var down []string + for _, f := range n.Fabrics { + for _, hca := range f.HCAs { + up := false + for _, p := range n.IBPorts { + if p.HCA == hca && p.Active() { + up = true + } + } + if !up && !slices.Contains(down, hca) { + down = append(down, hca) + } + } + } + if len(down) > 0 { + return "down: " + strings.Join(down, ", ") + } + return "ready" +} diff --git a/internal/control/adminapi/judges_test.go b/internal/control/adminapi/judges_test.go index 6d8fa097..116fa45a 100644 --- a/internal/control/adminapi/judges_test.go +++ b/internal/control/adminapi/judges_test.go @@ -7,6 +7,7 @@ import ( "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" ) @@ -104,3 +105,21 @@ func TestANodeSaysItsEnginesVersionsAndCapabilities(t *testing.T) { t.Errorf("%+v", nodes) } } + +func TestANodeSaysWhetherItsFabricsAreUp(t *testing.T) { + n := gpuNode("ib1", 2, "ib-a") + if got := multiNodeReadiness(n); !strings.HasPrefix(got, "unknown") { + t.Errorf("no port yet: %q", got) + } + n.IBPorts = []inventory.IBPort{{HCA: "mlx5_0", State: "DOWN"}} + if got := multiNodeReadiness(n); got != "down: mlx5_0" { + t.Errorf("down: %q", got) + } + n.IBPorts[0].State = "ACTIVE" + if got := multiNodeReadiness(n); got != "ready" { + t.Errorf("up: %q", got) + } + if got := multiNodeReadiness(gpuNode("plain", 1)); got != "" { + t.Errorf("no fabric: %q", got) + } +} diff --git a/internal/doctor/doctor.go b/internal/doctor/doctor.go index b32b4f4a..e4e16164 100644 --- a/internal/doctor/doctor.go +++ b/internal/doctor/doctor.go @@ -819,3 +819,45 @@ func sortedKeys[V any](m map[string]V) []string { sort.Strings(out) return out } + +// MultiNode is whether a finding is about the fabrics replicas spanning +// nodes use, rather than about the node itself. +func (f Finding) MultiNode() bool { return f.Area == "fabric" } + +// Readiness is a node's two readinesses, kept apart: whether it can join and +// serve replicas of its own, and whether it can also take part in replicas +// spanning nodes over a fabric. The second never keeps a node out unless a +// fabric was declared for it, whose problems are then the node's. +type Readiness struct { + // Node is "ready", or how many problems keep it out. + Node string `json:"node"` + // MultiNode is "ready", "not used" (no fabric adapter), or "not ready:" + // and the first reason. + MultiNode string `json:"multi_node"` +} + +// ReadinessOf sums findings up as the two readinesses. +func ReadinessOf(fs []Finding) Readiness { + r := Readiness{Node: "ready", MultiNode: "not used"} + fails := 0 + for _, f := range fs { + if f.Level == Fail { + fails++ + } + } + if fails > 0 { + r.Node = fmt.Sprintf("%d problem(s) keep it out", fails) + } + for _, f := range fs { + if !f.MultiNode() { + continue + } + if r.MultiNode == "not used" { + r.MultiNode = "ready" + } + if (f.Level == Fail || f.Level == Warn) && !strings.HasPrefix(r.MultiNode, "not ready") { + r.MultiNode = "not ready: " + f.Message + } + } + return r +} diff --git a/internal/doctor/doctor_test.go b/internal/doctor/doctor_test.go index f093d81c..896fa2c9 100644 --- a/internal/doctor/doctor_test.go +++ b/internal/doctor/doctor_test.go @@ -259,3 +259,20 @@ func TestAMissingFirewallIsSaidOnASetUpNode(t *testing.T) { } } } + +func TestTheNodeAndItsFabricsAreReadySeparately(t *testing.T) { + r := ReadinessOf([]Finding{ + {Level: OK, Area: "gpu", Message: "2 GPUs"}, + {Level: Warn, Area: "fabric", Message: "mlx5_1 is not active"}, + }) + if r.Node != "ready" || r.MultiNode != "not ready: mlx5_1 is not active" { + t.Errorf("%+v", r) + } + r = ReadinessOf([]Finding{{Level: Fail, Area: "gpu", Message: "no driver"}}) + if !strings.Contains(r.Node, "1 problem") || r.MultiNode != "not used" { + t.Errorf("%+v", r) + } + if r := ReadinessOf([]Finding{{Level: OK, Area: "fabric", Message: "mlx5_0 active"}}); r.MultiNode != "ready" { + t.Errorf("%+v", r) + } +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index e0559c5a..9faa07e5 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -505,6 +505,9 @@ func (r *Runner) check(ctx context.Context, res *Result) error { if fails > 0 && !goOn { return fmt.Errorf("%w: doctor found %d; fix them and run this again, or add --ignore-checks", ErrChecks, fails) } + if rd := doctor.ReadinessOf(rep.Findings); rd.MultiNode != "not used" { + r.emit(StepCheck, Info, "multi-node: "+rd.MultiNode+"; replicas of its own run either way", "") + } r.emit(StepCheck, Done, fmt.Sprintf("%d problem(s), %d warning(s)", fails, countLevel(rep.Findings, doctor.Warn)), "") return nil }