A node's own readiness and its multi-node readiness are kept apart
doctor sums findings up as two readinesses - the node's, which decides whether it joins, and multi-node, its fabrics for replicas spanning nodes - prints fabric findings in a section of their own, and gives both in --json. node add says the multi-node one. NodeView.multi_node says whether a node's declared fabrics are up, naming adapters that are down, and ctl nodes shows it beside the fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5ae870e1b2
commit
f2b2e24e1f
8 files changed
+155
-10
No files matched your search
+11
-1
@@ -233,7 +233,7 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
// at when a model will not place and the hardware looks
|
||||
// fine.
|
||||
enginesColumn(n),
|
||||
classesColumn(n), join(n.Fabrics), n.Replicas, n.LastSeenAgo)
|
||||
classesColumn(n), fabricsColumn(n), n.Replicas, n.LastSeenAgo)
|
||||
}
|
||||
_ = w.Flush()
|
||||
|
||||
@@ -596,6 +596,16 @@ func emitJSON(v any) error {
|
||||
// more than one or MIG instances among them: a card cut into two 3g.40gb
|
||||
// instances beside a whole card is "a100-80g-mig-3g.40gb×2,a100-80g×1", where
|
||||
// the GPUS column's 3 alone would read as three of something.
|
||||
// fabricsColumn is the node's fabrics, and when they are not all up, which
|
||||
// adapters are down: what keeps it out of replicas spanning nodes.
|
||||
func fabricsColumn(n adminapi.NodeView) string {
|
||||
s := join(n.Fabrics)
|
||||
if n.MultiNode != "" && n.MultiNode != "ready" {
|
||||
s += " (" + n.MultiNode + ")"
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// enginesColumn is each engine with its version: llamacpp b10991.
|
||||
func enginesColumn(n adminapi.NodeView) string {
|
||||
if len(n.EngineVersions) == 0 {
|
||||
|
||||
+22
-7
@@ -63,9 +63,10 @@ that file when the report is not enough to go on.`,
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(struct {
|
||||
Findings []doctor.Finding `json:"findings"`
|
||||
Facts doctor.Facts `json:"facts"`
|
||||
}{findings, facts})
|
||||
Findings []doctor.Finding `json:"findings"`
|
||||
Readiness doctor.Readiness `json:"readiness"`
|
||||
Facts doctor.Facts `json:"facts"`
|
||||
}{findings, doctor.ReadinessOf(findings), facts})
|
||||
}
|
||||
printDoctor(os.Stdout, facts, findings, verbose)
|
||||
for _, f := range findings {
|
||||
@@ -101,12 +102,26 @@ func printDoctor(out io.Writer, f doctor.Facts, findings []doctor.Finding, verbo
|
||||
fmt.Fprintf(out, "%s · %s %s %s · driver %s\n\n", f.Hostname, f.Host.OSID, f.Host.OSVersion,
|
||||
f.Host.Arch, orDash(f.Host.NVIDIADriver))
|
||||
|
||||
for _, x := range findings {
|
||||
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
|
||||
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
|
||||
fmt.Fprintf(out, " → %s\n", x.Fix)
|
||||
// The node first; then what replicas spanning nodes would need, which
|
||||
// keeps a node out only when a fabric was declared for it.
|
||||
multi := false
|
||||
for _, pass := range []bool{false, true} {
|
||||
for _, x := range findings {
|
||||
if x.MultiNode() != pass {
|
||||
continue
|
||||
}
|
||||
if pass && !multi {
|
||||
fmt.Fprintln(out, "\nMulti-node (fabrics, RDMA) — for replicas spanning nodes only:")
|
||||
multi = true
|
||||
}
|
||||
fmt.Fprintf(out, "%-4s %-6s %s\n", x.Level, x.Area, x.Message)
|
||||
if x.Fix != "" && (x.Level == doctor.Fail || x.Level == doctor.Warn) {
|
||||
fmt.Fprintf(out, " → %s\n", x.Fix)
|
||||
}
|
||||
}
|
||||
}
|
||||
r := doctor.ReadinessOf(findings)
|
||||
fmt.Fprintf(out, "\nThis node: %s\nMulti-node: %s\n", r.Node, r.MultiNode)
|
||||
|
||||
w := tabwriter.NewWriter(out, 0, 0, 2, ' ', 0)
|
||||
if len(f.GPUs) > 0 {
|
||||
|
||||
Reference in new issue
Block a user