Ilker ManapandClaude Opus 5.5
93bd16f00e
A port's state was read once, at registration: one that went down later left its node on the fabric, offered multi-node work that would hang on its first collective, until somebody restarted the agent. The agent now re-reads every IB and RoCE port on each heartbeat - state, physical state, rate and the error and data counters - and when the fabrics it can carry traffic on change, says so once; the control plane replaces the node's fabrics, which is what placement reads. The ports are exported as llmgw_ib_port_active (with state and phys_state, so an unplugged cable reads differently from a missing subnet manager), llmgw_ib_port_rate_gbps and llmgw_ib_port_counter_total, and a node's page lists them with the fabric each carries and any error counter that is not zero. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
104 lines
3.4 KiB
Go
104 lines
3.4 KiB
Go
package web
|
|
|
|
import (
|
|
"net/http"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/promq"
|
|
)
|
|
|
|
// nodeMux mounts the page where it lives, so the path carries the node.
|
|
func nodeMux(src Source, h History) http.Handler {
|
|
mux := http.NewServeMux()
|
|
if h == nil {
|
|
mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, nil, time.Hour, false, "v1", nil))
|
|
} else {
|
|
mux.HandleFunc("GET /admin/nodes/{id}", NodePage(src, h, time.Hour, false, "v1", nil))
|
|
}
|
|
return mux
|
|
}
|
|
|
|
// busySource is fakeSource with a node that has been measured.
|
|
type busySource struct{ fakeSource }
|
|
|
|
func (busySource) Nodes() []adminapi.NodeView {
|
|
n := fakeSource{}.Nodes()[0]
|
|
n.Load = &adminapi.LoadView{CPUPercent: 62, CPUs: 8, Load1: 3.25, Load5: 2, Load15: 1,
|
|
MemUsed: 6 << 30, MemTotal: 8 << 30, MemPercent: 75}
|
|
n.IBPorts = []adminapi.IBPortView{
|
|
{HCA: "mlx5_0", Port: 1, State: "ACTIVE", PhysState: "LinkUp", RateGbps: 400, Active: true,
|
|
Fabric: "ib-a", TxBytes: 4 << 30},
|
|
{HCA: "mlx5_1", Port: 1, State: "DOWN", PhysState: "Polling", Errors: map[string]uint64{"link_downed": 7}},
|
|
}
|
|
return []adminapi.NodeView{n}
|
|
}
|
|
|
|
func (busySource) Replicas(string) []adminapi.ReplicaView {
|
|
r := fakeSource{}.Replicas("")
|
|
r[0].Shards[0].CPUCores, r[0].Shards[0].MemoryBytes = 4, 5<<30
|
|
return r
|
|
}
|
|
|
|
func TestANodesPageShowsHowBusyItIsAndWhatIsUsingIt(t *testing.T) {
|
|
h := &fakeHistory{byExpr: map[string][]promq.Series{
|
|
`llmgw_node_cpu_utilization_percent{node="llmgw-node01"} / 100`: {ts("llmgw-node01", 0.2, 0.6)},
|
|
}}
|
|
code, body := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/llmgw-node01", nil)
|
|
if code != http.StatusOK {
|
|
t.Fatalf("status %d", code)
|
|
}
|
|
for _, want := range []string{
|
|
"62%", "CPU · 8 cores", "RAM in use", "3.25",
|
|
// The engine and its share: four of eight cores is half the machine.
|
|
"<td>served</td>", "4.0 cores", "width: 50%",
|
|
`hx-get="/admin/nodes/llmgw-node01"`,
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("missing %q", want)
|
|
}
|
|
}
|
|
// Every chart is about this node, and nothing else.
|
|
for _, q := range h.asked {
|
|
if !strings.Contains(q, `"llmgw-node01"`) && !strings.Contains(q, `served-1`) {
|
|
t.Errorf("a query not scoped to the node: %s", q)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAnUnknownNodeIsNotFound(t *testing.T) {
|
|
h := &fakeHistory{}
|
|
code, _ := get(t, nodeMux(busySource{}, h), "http://x/admin/nodes/nope", nil)
|
|
if code != http.StatusNotFound {
|
|
t.Errorf("status %d", code)
|
|
}
|
|
// Nothing was asked of the metrics store for a name nobody knows.
|
|
if len(h.asked) != 0 {
|
|
t.Errorf("queried for an unknown node: %v", h.asked)
|
|
}
|
|
}
|
|
|
|
func TestANodeNotMeasuredLatelySaysSoRatherThanZero(t *testing.T) {
|
|
_, body := get(t, nodeMux(fakeSource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
|
if !strings.Contains(body, "not measured lately") {
|
|
t.Error("an unmeasured node should say so")
|
|
}
|
|
if !strings.Contains(body, "No metrics store is configured") {
|
|
t.Error("without a metrics store the page should say where history comes from")
|
|
}
|
|
}
|
|
|
|
func TestANodesPageShowsItsFabricPorts(t *testing.T) {
|
|
_, body := get(t, nodeMux(busySource{}, nil), "http://x/admin/nodes/llmgw-node01", nil)
|
|
for _, want := range []string{
|
|
"<h2>Fabric</h2>", "On ib-a.", "<code>mlx5_0/1</code>", "400 Gb/s", "4.0 GiB",
|
|
`<td class="bad">DOWN <span class="note">Polling</span></td>`, "link_downed 7",
|
|
} {
|
|
if !strings.Contains(body, want) {
|
|
t.Errorf("missing %q", want)
|
|
}
|
|
}
|
|
}
|