Ilker ManapandClaude Opus 5.5
7bb360fc78
doctor checks a node alone, and every check can pass while two nodes still cannot reach each other. ctl fabric-test A B asks B's agent for an ib_write_bw server and A's for the client, adapter by adapter as NCCL pairs them, one rail at a time, and grades each against its link's rate. It goes through the agents' streams, so there is no SSH between nodes; the agent installs perftest the first time, and names the RoCE v2 GID itself. The result is read from perftest's table by its heading, not a column number. Agents now list the requests they answer beyond the usual (Register.features), and the control plane refuses a test on one too old to answer instead of waiting minutes for it. The forwarding dispatcher carries the new capability, and its test says so: a missing forward would have made the command quietly unavailable in production. Not yet run over real InfiniBand or RoCE; this cluster has neither. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
41 lines
1.6 KiB
Go
41 lines
1.6 KiB
Go
package cli
|
|
|
|
import (
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
|
)
|
|
|
|
func TestTheForwardingDispatcherCarriesEveryCapability(t *testing.T) {
|
|
// dispatcherRef exists to break a construction cycle, and it satisfies the
|
|
// dispatcher interface by forwarding each method by hand. That makes it a
|
|
// place where capabilities go missing silently: the controller reaches
|
|
// past the interface by type assertion, and an assertion against this type
|
|
// fails for anything nobody remembered to forward.
|
|
//
|
|
// It happened. `ctl weights --collect` returned "this control plane cannot
|
|
// ask nodes to delete files" against a real cluster, while every unit test
|
|
// passed — because the test dispatchers do not implement the capability
|
|
// either, so its absence looked exactly like the expected case.
|
|
var ref any = &dispatcherRef{}
|
|
if _, ok := ref.(control.WeightRemover); !ok {
|
|
t.Error("dispatcherRef drops WeightRemover, so the control plane cannot reclaim disk")
|
|
}
|
|
|
|
if _, ok := ref.(fabrictest.Asker); !ok {
|
|
t.Error("dispatcherRef drops the bandwidth test, so ctl fabric-test cannot reach the agents")
|
|
}
|
|
|
|
// And the thing it forwards to must have it, or the forwarding has nothing
|
|
// to reach.
|
|
var svc any = &agentsvc.Service{}
|
|
if _, ok := svc.(control.WeightRemover); !ok {
|
|
t.Error("the agent service does not implement WeightRemover")
|
|
}
|
|
if _, ok := svc.(fabrictest.Asker); !ok {
|
|
t.Error("the agent service does not implement the bandwidth test")
|
|
}
|
|
}
|