From 8e57171c4f570be2bf6b5f53888864404d331b58 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Sun, 27 Sep 2026 07:30:51 +0200 Subject: [PATCH] Let ctl ask a question instead of printing the whole fleet MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five nodes fit in a table. Fifty do not, and this is built for people who already have racks of GPUs — so the flags are how a question gets asked: which nodes on this fabric, which of this GPU class, which are not ready, which replicas have a shard on this machine. Filtered here rather than in the API, so --json is narrowed the same way the table is. A server-side filter would be two implementations of one predicate, and the JSON one is the one nobody would notice had drifted. An empty result says which case it is. A filter that matched nothing and an empty fleet call for opposite next steps — one is a cluster to look at, the other is a word to correct — and printing "no nodes" for both makes a mistyped fabric name read as an outage. A state matches on prefix because states are few and distinct and somebody scanning a fleet types quickly. A pool matches either half, because it is fabric/class and both are worth narrowing by. A replica is found from any of its nodes, since one spanning two machines is on both. --label takes a bare key as "carries this at all", which is how you ask for every node with a role without caring which. The flag's help said so before the parser did; the test caught them disagreeing. Co-Authored-By: Claude Opus 5 (1M context) --- internal/cli/ctl.go | 34 +++++++- internal/cli/filter.go | 170 ++++++++++++++++++++++++++++++++++++ internal/cli/filter_test.go | 155 ++++++++++++++++++++++++++++++++ 3 files changed, 358 insertions(+), 1 deletion(-) create mode 100644 internal/cli/filter.go create mode 100644 internal/cli/filter_test.go diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 1a9e637..d76e9ba 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -133,18 +133,34 @@ func newCtlCmd() *cobra.Command { } func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { - return &cobra.Command{ + var f nodeFilter + c := &cobra.Command{ Use: "nodes", Short: "List nodes and their capacity", + Long: "On a fleet too large to read a whole table of, the filters are how a\n" + + "question gets asked: which nodes on this fabric, which of this GPU class,\n" + + "which are not ready. They combine, and they apply to --json too.", RunE: func(cmd *cobra.Command, _ []string) error { var nodes []adminapi.NodeView if err := client().get(cmd.Context(), "/api/v1/nodes", &nodes); err != nil { return err } + total := len(nodes) + nodes, err := f.apply(nodes) + if err != nil { + return err + } if *asJSON { return emitJSON(nodes) } if len(nodes) == 0 { + // A filter that matched nothing and an empty fleet call for + // opposite next steps: one is a cluster to look at, the other + // is a word to correct. + if !f.empty() && total > 0 { + fmt.Println(noMatches("nodes", f.describe(), total)) + return nil + } fmt.Println("No nodes have registered.") return nil } @@ -172,6 +188,12 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { return nil }, } + c.Flags().StringVar(&f.fabric, "fabric", "", "only nodes attached to this fabric") + c.Flags().StringVar(&f.class, "class", "", "only nodes with this GPU class") + c.Flags().StringVar(&f.state, "state", "", "only nodes in this state (prefix match)") + c.Flags().StringArrayVar(&f.labels, "label", nil, + "only nodes carrying key=value; repeatable, and key alone matches any value") + return c } // modelStatus says what is happening to a model, in the words that decide what @@ -275,6 +297,7 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { var model string + var f replicaFilter c := &cobra.Command{ Use: "replicas", Short: "List replicas and the shards they are made of", @@ -287,10 +310,16 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { if err := client().get(cmd.Context(), path, &reps); err != nil { return err } + total := len(reps) + reps = f.apply(reps) if *asJSON { return emitJSON(reps) } if len(reps) == 0 { + if !f.empty() && total > 0 { + fmt.Println(noMatches("replicas", f.describe(), total)) + return nil + } fmt.Println("No replicas are running.") return nil } @@ -323,6 +352,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { }, } c.Flags().StringVar(&model, "model", "", "only this model's replicas") + c.Flags().StringVar(&f.node, "node", "", "only replicas with a shard on this node") + c.Flags().StringVar(&f.state, "state", "", "only replicas in this state (prefix match)") + c.Flags().StringVar(&f.pool, "pool", "", "only this pool; matches either half of fabric/class") return c } diff --git a/internal/cli/filter.go b/internal/cli/filter.go new file mode 100644 index 0000000..bab10fc --- /dev/null +++ b/internal/cli/filter.go @@ -0,0 +1,170 @@ +package cli + +import ( + "fmt" + "slices" + "strings" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +// nodeFilter narrows a node listing. +// +// Client-side on the whole list rather than as query parameters, because the +// list is small even on a large fleet — a few hundred rows — and doing it here +// means --json is filtered the same way the table is. A server-side filter +// would be two implementations of the same predicate, and the JSON one is the +// one nobody would notice had drifted. +type nodeFilter struct { + fabric string + class string + state string + labels []string +} + +// empty reports whether this filter would narrow anything. +// +// The difference matters: an empty fleet and a filter that matched nothing are +// different situations with different next steps, and "no nodes" for both +// makes a mistyped fabric look like an outage. +func (f nodeFilter) empty() bool { + return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0 +} + +func (f nodeFilter) describe() string { + var parts []string + if f.fabric != "" { + parts = append(parts, "fabric "+f.fabric) + } + if f.class != "" { + parts = append(parts, "class "+f.class) + } + if f.state != "" { + parts = append(parts, "state "+f.state) + } + for _, l := range f.labels { + parts = append(parts, "label "+l) + } + return strings.Join(parts, ", ") +} + +func (f nodeFilter) apply(in []adminapi.NodeView) ([]adminapi.NodeView, error) { + want := map[string]string{} + for _, l := range f.labels { + // A bare key is allowed and means "carries this at all", which is how + // an operator asks for every node with a role without caring which. + // An empty key is not: "=value" is a typo, and matching everything + // would look like the filter having worked. + k, v, _ := strings.Cut(l, "=") + if k == "" { + return nil, fmt.Errorf("--label needs a key: %q has none", l) + } + want[k] = v + } + + out := make([]adminapi.NodeView, 0, len(in)) + for _, n := range in { + if f.fabric != "" && !slices.Contains(n.Fabrics, f.fabric) { + continue + } + if f.class != "" && !slices.Contains(n.Classes, f.class) { + continue + } + // Prefix rather than equality, so --state ready matches and --state re + // does too. Node states are few and distinct, and an operator scanning + // a fleet is typing quickly. + if f.state != "" && !strings.HasPrefix(n.State, strings.ToLower(f.state)) { + continue + } + if !hasLabels(n.Labels, want) { + continue + } + out = append(out, n) + } + return out, nil +} + +// hasLabels reports whether a node carries every wanted label. +// +// An empty value means "has this key at all", which is how an operator asks +// for every node that carries a role without caring which. +func hasLabels(have map[string]string, want map[string]string) bool { + for k, v := range want { + got, ok := have[k] + if !ok { + return false + } + if v != "" && got != v { + return false + } + } + return true +} + +// replicaFilter narrows a replica listing. +type replicaFilter struct { + node string + state string + pool string +} + +func (f replicaFilter) empty() bool { + return f.node == "" && f.state == "" && f.pool == "" +} + +func (f replicaFilter) describe() string { + var parts []string + if f.node != "" { + parts = append(parts, "node "+f.node) + } + if f.state != "" { + parts = append(parts, "state "+f.state) + } + if f.pool != "" { + parts = append(parts, "pool "+f.pool) + } + return strings.Join(parts, ", ") +} + +func (f replicaFilter) apply(in []adminapi.ReplicaView) []adminapi.ReplicaView { + out := make([]adminapi.ReplicaView, 0, len(in)) + for _, r := range in { + // Through the shards, because that is where the nodes are: a replica + // spanning two machines is on both, and asking "what is on node05" + // has to find it from either side. + if f.node != "" && !onNode(r, f.node) { + continue + } + if f.state != "" && !strings.HasPrefix(strings.ToLower(r.State), strings.ToLower(f.state)) { + continue + } + // Substring, because a pool is "fabric/class" and both halves are + // things an operator narrows by: --pool ib-a and --pool h100-80g are + // both questions worth asking. + if f.pool != "" && !strings.Contains(r.Pool, f.pool) { + continue + } + out = append(out, r) + } + return out +} + +// onNode reports whether any shard of a replica runs on a node. +func onNode(r adminapi.ReplicaView, node string) bool { + for _, sh := range r.Shards { + if sh.Node == node { + return true + } + } + return false +} + +// noMatches is what to print when a filter excluded everything. +// +// Distinct from "there are none", because they call for opposite next steps: +// one is a cluster to look at and the other is a word to correct. A mistyped +// fabric name otherwise reads as an outage. +func noMatches(kind, what string, total int) string { + return fmt.Sprintf("No %s match %s. %d %s in total; run without the filter to see them.", + kind, what, total, kind) +} diff --git a/internal/cli/filter_test.go b/internal/cli/filter_test.go new file mode 100644 index 0000000..18fa8ee --- /dev/null +++ b/internal/cli/filter_test.go @@ -0,0 +1,155 @@ +package cli + +import ( + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" +) + +func fleet() []adminapi.NodeView { + return []adminapi.NodeView{ + {ID: "n1", State: "ready", Fabrics: []string{"ib-a"}, Classes: []string{"h100-80g"}, + Labels: map[string]string{"role": "cpu-large", "rack": "r1"}}, + {ID: "n2", State: "ready", Fabrics: []string{"ib-b"}, Classes: []string{"h100-80g"}, + Labels: map[string]string{"role": "cpu-large"}}, + {ID: "n3", State: "cordoned", Fabrics: []string{"ib-a"}, Classes: []string{"l40s-45g"}}, + {ID: "n4", State: "ready"}, + } +} + +func ids(in []adminapi.NodeView) string { + var out []string + for _, n := range in { + out = append(out, n.ID) + } + return strings.Join(out, ",") +} + +func mustApply(t *testing.T, f nodeFilter) []adminapi.NodeView { + t.Helper() + got, err := f.apply(fleet()) + if err != nil { + t.Fatal(err) + } + return got +} + +func TestFilteringByFabricClassAndState(t *testing.T) { + if got := ids(mustApply(t, nodeFilter{fabric: "ib-a"})); got != "n1,n3" { + t.Errorf("--fabric ib-a = %s", got) + } + if got := ids(mustApply(t, nodeFilter{class: "l40s-45g"})); got != "n3" { + t.Errorf("--class l40s-45g = %s", got) + } + if got := ids(mustApply(t, nodeFilter{state: "cordoned"})); got != "n3" { + t.Errorf("--state cordoned = %s", got) + } +} + +func TestFiltersCombine(t *testing.T) { + // Which is the question worth asking on a large fleet: not "what is on + // this fabric" but "what on this fabric is not ready". + if got := ids(mustApply(t, nodeFilter{fabric: "ib-a", state: "ready"})); got != "n1" { + t.Errorf("combined = %s", got) + } +} + +func TestAStatePrefixIsEnough(t *testing.T) { + // Node states are few and distinct, and somebody scanning a fleet types + // quickly. + if got := ids(mustApply(t, nodeFilter{state: "cord"})); got != "n3" { + t.Errorf("--state cord = %s", got) + } +} + +func TestALabelKeyAloneMatchesAnyValue(t *testing.T) { + // How an operator asks for every node carrying a role without caring which. + if got := ids(mustApply(t, nodeFilter{labels: []string{"role"}})); got != "n1,n2" { + t.Errorf("--label role = %s", got) + } + if got := ids(mustApply(t, nodeFilter{labels: []string{"role=cpu-large", "rack=r1"}})); got != "n1" { + t.Errorf("two labels = %s", got) + } + if got := ids(mustApply(t, nodeFilter{labels: []string{"rack=r9"}})); got != "" { + t.Errorf("a label nothing carries matched %s", got) + } +} + +func TestAMalformedLabelIsRefusedRatherThanIgnored(t *testing.T) { + // Silently ignoring it would show the whole fleet, which reads as the + // filter having matched everything rather than as a typo. + _, err := nodeFilter{labels: []string{"=value"}}.apply(fleet()) + if err == nil { + t.Error("a label with no key was accepted") + } +} + +func TestNoFilterIsRecognisedAsNone(t *testing.T) { + // It decides which of two messages an empty result gets. + if !(nodeFilter{}).empty() { + t.Error("an unset filter did not report itself as empty") + } + if (nodeFilter{state: "ready"}).empty() { + t.Error("a set filter reported itself as empty") + } +} + +func TestTheEmptyMessageSaysWhichCaseItIs(t *testing.T) { + // A mistyped fabric name otherwise reads as an outage. + msg := noMatches("nodes", "fabric ib-c", 12) + for _, want := range []string{"fabric ib-c", "12"} { + if !strings.Contains(msg, want) { + t.Errorf("the message does not mention %q: %s", want, msg) + } + } +} + +func replicas() []adminapi.ReplicaView { + return []adminapi.ReplicaView{ + {ID: "r1", Model: "big", State: "ready", Pool: "ib-a/h100-80g", + Shards: []adminapi.ShardView{{Node: "n1"}, {Node: "n2"}}}, + {ID: "r2", Model: "small", State: "starting", Pool: "none/cpu", + Shards: []adminapi.ShardView{{Node: "n5"}}}, + {ID: "r3", Model: "small", State: "ready", Pool: "ib-b/h100-80g", + Shards: []adminapi.ShardView{{Node: "n2"}}}, + } +} + +func repIDs(in []adminapi.ReplicaView) string { + var out []string + for _, r := range in { + out = append(out, r.ID) + } + return strings.Join(out, ",") +} + +func TestAReplicaIsFoundFromEitherOfItsNodes(t *testing.T) { + // A replica spanning two machines is on both, and "what is running on n2" + // has to find it from either side. + if got := repIDs(replicaFilter{node: "n1"}.apply(replicas())); got != "r1" { + t.Errorf("--node n1 = %s", got) + } + if got := repIDs(replicaFilter{node: "n2"}.apply(replicas())); got != "r1,r3" { + t.Errorf("--node n2 = %s", got) + } +} + +func TestAPoolMatchesEitherHalf(t *testing.T) { + // A pool is fabric/class and both are things worth narrowing by. + if got := repIDs(replicaFilter{pool: "ib-a"}.apply(replicas())); got != "r1" { + t.Errorf("--pool ib-a = %s", got) + } + if got := repIDs(replicaFilter{pool: "h100-80g"}.apply(replicas())); got != "r1,r3" { + t.Errorf("--pool h100-80g = %s", got) + } +} + +func TestReplicaStateFilters(t *testing.T) { + if got := repIDs(replicaFilter{state: "start"}.apply(replicas())); got != "r2" { + t.Errorf("--state start = %s", got) + } + if got := repIDs(replicaFilter{node: "n2", state: "ready"}.apply(replicas())); got != "r1,r3" { + t.Errorf("combined = %s", got) + } +}