A node's own readiness and its multi-node readiness are kept apart
doctor sums findings up as two readinesses - the node's, which decides whether it joins, and multi-node, its fabrics for replicas spanning nodes - prints fabric findings in a section of their own, and gives both in --json. node add says the multi-node one. NodeView.multi_node says whether a node's declared fabrics are up, naming adapters that are down, and ctl nodes shows it beside the fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
5ae870e1b2
commit
f2b2e24e1f
8 files changed
+155
-10
No files matched your search
@@ -233,8 +233,13 @@ type NodeView struct {
|
||||
// llamacpp → b10991. ComputeCapabilities are its GPUs' CUDA compute
|
||||
// capabilities, 7.0 for a V100, so an engine build's support can be read
|
||||
// against them.
|
||||
EngineVersions map[string]string `json:"engine_versions,omitempty"`
|
||||
ComputeCapabilities []string `json:"compute_capabilities,omitempty"`
|
||||
EngineVersions map[string]string `json:"engine_versions,omitempty"`
|
||||
// MultiNode is whether it can take part in replicas spanning nodes:
|
||||
// "ready", "down: mlx5_1" naming the declared adapters whose ports are
|
||||
// not active, or empty for a node on no fabric. Its own replicas run
|
||||
// either way.
|
||||
MultiNode string `json:"multi_node,omitempty"`
|
||||
ComputeCapabilities []string `json:"compute_capabilities,omitempty"`
|
||||
// Provision is where installing each wanted engine stands. An engine is in
|
||||
// Engines only once its installation here is ready.
|
||||
Provision []inventory.EngineInstall `json:"provision,omitempty"`
|
||||
@@ -663,6 +668,7 @@ func (a *API) Nodes() []NodeView {
|
||||
}
|
||||
}
|
||||
slices.Sort(v.ComputeCapabilities)
|
||||
v.MultiNode = multiNodeReadiness(n)
|
||||
v.Provision = n.Provision
|
||||
if ch := a.ctl.EngineChoices(); ch != nil && len(n.GPUs) > 0 {
|
||||
chosen := ch.For(n.ID)
|
||||
@@ -2026,3 +2032,32 @@ func (a *API) SetNodeEngine(ctx context.Context, node, engine, build, by string)
|
||||
}
|
||||
return ch.Set(ctx, node, engine, build, by)
|
||||
}
|
||||
|
||||
// multiNodeReadiness is whether a node's declared fabrics are up: every
|
||||
// adapter they name with an active port. Empty for a node on no fabric.
|
||||
func multiNodeReadiness(n inventory.Node) string {
|
||||
if len(n.Fabrics) == 0 {
|
||||
return ""
|
||||
}
|
||||
if len(n.IBPorts) == 0 {
|
||||
return "unknown: no port reported yet"
|
||||
}
|
||||
var down []string
|
||||
for _, f := range n.Fabrics {
|
||||
for _, hca := range f.HCAs {
|
||||
up := false
|
||||
for _, p := range n.IBPorts {
|
||||
if p.HCA == hca && p.Active() {
|
||||
up = true
|
||||
}
|
||||
}
|
||||
if !up && !slices.Contains(down, hca) {
|
||||
down = append(down, hca)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(down) > 0 {
|
||||
return "down: " + strings.Join(down, ", ")
|
||||
}
|
||||
return "ready"
|
||||
}
|
||||
@@ -7,6 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
)
|
||||
|
||||
@@ -104,3 +105,21 @@ func TestANodeSaysItsEnginesVersionsAndCapabilities(t *testing.T) {
|
||||
t.Errorf("%+v", nodes)
|
||||
}
|
||||
}
|
||||
|
||||
func TestANodeSaysWhetherItsFabricsAreUp(t *testing.T) {
|
||||
n := gpuNode("ib1", 2, "ib-a")
|
||||
if got := multiNodeReadiness(n); !strings.HasPrefix(got, "unknown") {
|
||||
t.Errorf("no port yet: %q", got)
|
||||
}
|
||||
n.IBPorts = []inventory.IBPort{{HCA: "mlx5_0", State: "DOWN"}}
|
||||
if got := multiNodeReadiness(n); got != "down: mlx5_0" {
|
||||
t.Errorf("down: %q", got)
|
||||
}
|
||||
n.IBPorts[0].State = "ACTIVE"
|
||||
if got := multiNodeReadiness(n); got != "ready" {
|
||||
t.Errorf("up: %q", got)
|
||||
}
|
||||
if got := multiNodeReadiness(gpuNode("plain", 1)); got != "" {
|
||||
t.Errorf("no fabric: %q", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user