Let ctl ask a question instead of printing the whole fleet
CI / test (push) Successful in 1m42s

Five nodes fit in a table. Fifty do not, and this is built for people who
already have racks of GPUs — so the flags are how a question gets asked:
which nodes on this fabric, which of this GPU class, which are not ready,
which replicas have a shard on this machine.

Filtered here rather than in the API, so --json is narrowed the same way the
table is. A server-side filter would be two implementations of one predicate,
and the JSON one is the one nobody would notice had drifted.

An empty result says which case it is. A filter that matched nothing and an
empty fleet call for opposite next steps — one is a cluster to look at, the
other is a word to correct — and printing "no nodes" for both makes a mistyped
fabric name read as an outage.

A state matches on prefix because states are few and distinct and somebody
scanning a fleet types quickly. A pool matches either half, because it is
fabric/class and both are worth narrowing by. A replica is found from any of
its nodes, since one spanning two machines is on both.

--label takes a bare key as "carries this at all", which is how you ask for
every node with a role without caring which. The flag's help said so before
the parser did; the test caught them disagreeing.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-27 07:30:51 +02:00
1 parent 3a4260c8c5
commit 8e57171c4f
3 files changed
+358 -1

No files matched your search

+33 -1
View File
@@ -133,18 +133,34 @@ func newCtlCmd() *cobra.Command {
}
func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return &cobra.Command{
var f nodeFilter
c := &cobra.Command{
Use: "nodes",
Short: "List nodes and their capacity",
Long: "On a fleet too large to read a whole table of, the filters are how a\n" +
"question gets asked: which nodes on this fabric, which of this GPU class,\n" +
"which are not ready. They combine, and they apply to --json too.",
RunE: func(cmd *cobra.Command, _ []string) error {
var nodes []adminapi.NodeView
if err := client().get(cmd.Context(), "/api/v1/nodes", &nodes); err != nil {
return err
}
total := len(nodes)
nodes, err := f.apply(nodes)
if err != nil {
return err
}
if *asJSON {
return emitJSON(nodes)
}
if len(nodes) == 0 {
// A filter that matched nothing and an empty fleet call for
// opposite next steps: one is a cluster to look at, the other
// is a word to correct.
if !f.empty() && total > 0 {
fmt.Println(noMatches("nodes", f.describe(), total))
return nil
}
fmt.Println("No nodes have registered.")
return nil
}
@@ -172,6 +188,12 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
return nil
},
}
c.Flags().StringVar(&f.fabric, "fabric", "", "only nodes attached to this fabric")
c.Flags().StringVar(&f.class, "class", "", "only nodes with this GPU class")
c.Flags().StringVar(&f.state, "state", "", "only nodes in this state (prefix match)")
c.Flags().StringArrayVar(&f.labels, "label", nil,
"only nodes carrying key=value; repeatable, and key alone matches any value")
return c
}
// modelStatus says what is happening to a model, in the words that decide what
@@ -275,6 +297,7 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
var model string
var f replicaFilter
c := &cobra.Command{
Use: "replicas",
Short: "List replicas and the shards they are made of",
@@ -287,10 +310,16 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
if err := client().get(cmd.Context(), path, &reps); err != nil {
return err
}
total := len(reps)
reps = f.apply(reps)
if *asJSON {
return emitJSON(reps)
}
if len(reps) == 0 {
if !f.empty() && total > 0 {
fmt.Println(noMatches("replicas", f.describe(), total))
return nil
}
fmt.Println("No replicas are running.")
return nil
}
@@ -323,6 +352,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
},
}
c.Flags().StringVar(&model, "model", "", "only this model's replicas")
c.Flags().StringVar(&f.node, "node", "", "only replicas with a shard on this node")
c.Flags().StringVar(&f.state, "state", "", "only replicas in this state (prefix match)")
c.Flags().StringVar(&f.pool, "pool", "", "only this pool; matches either half of fabric/class")
return c
}
+170
View File
@@ -0,0 +1,170 @@
package cli
import (
"fmt"
"slices"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
// nodeFilter narrows a node listing.
//
// Client-side on the whole list rather than as query parameters, because the
// list is small even on a large fleet — a few hundred rows — and doing it here
// means --json is filtered the same way the table is. A server-side filter
// would be two implementations of the same predicate, and the JSON one is the
// one nobody would notice had drifted.
type nodeFilter struct {
fabric string
class string
state string
labels []string
}
// empty reports whether this filter would narrow anything.
//
// The difference matters: an empty fleet and a filter that matched nothing are
// different situations with different next steps, and "no nodes" for both
// makes a mistyped fabric look like an outage.
func (f nodeFilter) empty() bool {
return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0
}
func (f nodeFilter) describe() string {
var parts []string
if f.fabric != "" {
parts = append(parts, "fabric "+f.fabric)
}
if f.class != "" {
parts = append(parts, "class "+f.class)
}
if f.state != "" {
parts = append(parts, "state "+f.state)
}
for _, l := range f.labels {
parts = append(parts, "label "+l)
}
return strings.Join(parts, ", ")
}
func (f nodeFilter) apply(in []adminapi.NodeView) ([]adminapi.NodeView, error) {
want := map[string]string{}
for _, l := range f.labels {
// A bare key is allowed and means "carries this at all", which is how
// an operator asks for every node with a role without caring which.
// An empty key is not: "=value" is a typo, and matching everything
// would look like the filter having worked.
k, v, _ := strings.Cut(l, "=")
if k == "" {
return nil, fmt.Errorf("--label needs a key: %q has none", l)
}
want[k] = v
}
out := make([]adminapi.NodeView, 0, len(in))
for _, n := range in {
if f.fabric != "" && !slices.Contains(n.Fabrics, f.fabric) {
continue
}
if f.class != "" && !slices.Contains(n.Classes, f.class) {
continue
}
// Prefix rather than equality, so --state ready matches and --state re
// does too. Node states are few and distinct, and an operator scanning
// a fleet is typing quickly.
if f.state != "" && !strings.HasPrefix(n.State, strings.ToLower(f.state)) {
continue
}
if !hasLabels(n.Labels, want) {
continue
}
out = append(out, n)
}
return out, nil
}
// hasLabels reports whether a node carries every wanted label.
//
// An empty value means "has this key at all", which is how an operator asks
// for every node that carries a role without caring which.
func hasLabels(have map[string]string, want map[string]string) bool {
for k, v := range want {
got, ok := have[k]
if !ok {
return false
}
if v != "" && got != v {
return false
}
}
return true
}
// replicaFilter narrows a replica listing.
type replicaFilter struct {
node string
state string
pool string
}
func (f replicaFilter) empty() bool {
return f.node == "" && f.state == "" && f.pool == ""
}
func (f replicaFilter) describe() string {
var parts []string
if f.node != "" {
parts = append(parts, "node "+f.node)
}
if f.state != "" {
parts = append(parts, "state "+f.state)
}
if f.pool != "" {
parts = append(parts, "pool "+f.pool)
}
return strings.Join(parts, ", ")
}
func (f replicaFilter) apply(in []adminapi.ReplicaView) []adminapi.ReplicaView {
out := make([]adminapi.ReplicaView, 0, len(in))
for _, r := range in {
// Through the shards, because that is where the nodes are: a replica
// spanning two machines is on both, and asking "what is on node05"
// has to find it from either side.
if f.node != "" && !onNode(r, f.node) {
continue
}
if f.state != "" && !strings.HasPrefix(strings.ToLower(r.State), strings.ToLower(f.state)) {
continue
}
// Substring, because a pool is "fabric/class" and both halves are
// things an operator narrows by: --pool ib-a and --pool h100-80g are
// both questions worth asking.
if f.pool != "" && !strings.Contains(r.Pool, f.pool) {
continue
}
out = append(out, r)
}
return out
}
// onNode reports whether any shard of a replica runs on a node.
func onNode(r adminapi.ReplicaView, node string) bool {
for _, sh := range r.Shards {
if sh.Node == node {
return true
}
}
return false
}
// noMatches is what to print when a filter excluded everything.
//
// Distinct from "there are none", because they call for opposite next steps:
// one is a cluster to look at and the other is a word to correct. A mistyped
// fabric name otherwise reads as an outage.
func noMatches(kind, what string, total int) string {
return fmt.Sprintf("No %s match %s. %d %s in total; run without the filter to see them.",
kind, what, total, kind)
}
+155
View File
@@ -0,0 +1,155 @@
package cli
import (
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)
func fleet() []adminapi.NodeView {
return []adminapi.NodeView{
{ID: "n1", State: "ready", Fabrics: []string{"ib-a"}, Classes: []string{"h100-80g"},
Labels: map[string]string{"role": "cpu-large", "rack": "r1"}},
{ID: "n2", State: "ready", Fabrics: []string{"ib-b"}, Classes: []string{"h100-80g"},
Labels: map[string]string{"role": "cpu-large"}},
{ID: "n3", State: "cordoned", Fabrics: []string{"ib-a"}, Classes: []string{"l40s-45g"}},
{ID: "n4", State: "ready"},
}
}
func ids(in []adminapi.NodeView) string {
var out []string
for _, n := range in {
out = append(out, n.ID)
}
return strings.Join(out, ",")
}
func mustApply(t *testing.T, f nodeFilter) []adminapi.NodeView {
t.Helper()
got, err := f.apply(fleet())
if err != nil {
t.Fatal(err)
}
return got
}
func TestFilteringByFabricClassAndState(t *testing.T) {
if got := ids(mustApply(t, nodeFilter{fabric: "ib-a"})); got != "n1,n3" {
t.Errorf("--fabric ib-a = %s", got)
}
if got := ids(mustApply(t, nodeFilter{class: "l40s-45g"})); got != "n3" {
t.Errorf("--class l40s-45g = %s", got)
}
if got := ids(mustApply(t, nodeFilter{state: "cordoned"})); got != "n3" {
t.Errorf("--state cordoned = %s", got)
}
}
func TestFiltersCombine(t *testing.T) {
// Which is the question worth asking on a large fleet: not "what is on
// this fabric" but "what on this fabric is not ready".
if got := ids(mustApply(t, nodeFilter{fabric: "ib-a", state: "ready"})); got != "n1" {
t.Errorf("combined = %s", got)
}
}
func TestAStatePrefixIsEnough(t *testing.T) {
// Node states are few and distinct, and somebody scanning a fleet types
// quickly.
if got := ids(mustApply(t, nodeFilter{state: "cord"})); got != "n3" {
t.Errorf("--state cord = %s", got)
}
}
func TestALabelKeyAloneMatchesAnyValue(t *testing.T) {
// How an operator asks for every node carrying a role without caring which.
if got := ids(mustApply(t, nodeFilter{labels: []string{"role"}})); got != "n1,n2" {
t.Errorf("--label role = %s", got)
}
if got := ids(mustApply(t, nodeFilter{labels: []string{"role=cpu-large", "rack=r1"}})); got != "n1" {
t.Errorf("two labels = %s", got)
}
if got := ids(mustApply(t, nodeFilter{labels: []string{"rack=r9"}})); got != "" {
t.Errorf("a label nothing carries matched %s", got)
}
}
func TestAMalformedLabelIsRefusedRatherThanIgnored(t *testing.T) {
// Silently ignoring it would show the whole fleet, which reads as the
// filter having matched everything rather than as a typo.
_, err := nodeFilter{labels: []string{"=value"}}.apply(fleet())
if err == nil {
t.Error("a label with no key was accepted")
}
}
func TestNoFilterIsRecognisedAsNone(t *testing.T) {
// It decides which of two messages an empty result gets.
if !(nodeFilter{}).empty() {
t.Error("an unset filter did not report itself as empty")
}
if (nodeFilter{state: "ready"}).empty() {
t.Error("a set filter reported itself as empty")
}
}
func TestTheEmptyMessageSaysWhichCaseItIs(t *testing.T) {
// A mistyped fabric name otherwise reads as an outage.
msg := noMatches("nodes", "fabric ib-c", 12)
for _, want := range []string{"fabric ib-c", "12"} {
if !strings.Contains(msg, want) {
t.Errorf("the message does not mention %q: %s", want, msg)
}
}
}
func replicas() []adminapi.ReplicaView {
return []adminapi.ReplicaView{
{ID: "r1", Model: "big", State: "ready", Pool: "ib-a/h100-80g",
Shards: []adminapi.ShardView{{Node: "n1"}, {Node: "n2"}}},
{ID: "r2", Model: "small", State: "starting", Pool: "none/cpu",
Shards: []adminapi.ShardView{{Node: "n5"}}},
{ID: "r3", Model: "small", State: "ready", Pool: "ib-b/h100-80g",
Shards: []adminapi.ShardView{{Node: "n2"}}},
}
}
func repIDs(in []adminapi.ReplicaView) string {
var out []string
for _, r := range in {
out = append(out, r.ID)
}
return strings.Join(out, ",")
}
func TestAReplicaIsFoundFromEitherOfItsNodes(t *testing.T) {
// A replica spanning two machines is on both, and "what is running on n2"
// has to find it from either side.
if got := repIDs(replicaFilter{node: "n1"}.apply(replicas())); got != "r1" {
t.Errorf("--node n1 = %s", got)
}
if got := repIDs(replicaFilter{node: "n2"}.apply(replicas())); got != "r1,r3" {
t.Errorf("--node n2 = %s", got)
}
}
func TestAPoolMatchesEitherHalf(t *testing.T) {
// A pool is fabric/class and both are things worth narrowing by.
if got := repIDs(replicaFilter{pool: "ib-a"}.apply(replicas())); got != "r1" {
t.Errorf("--pool ib-a = %s", got)
}
if got := repIDs(replicaFilter{pool: "h100-80g"}.apply(replicas())); got != "r1,r3" {
t.Errorf("--pool h100-80g = %s", got)
}
}
func TestReplicaStateFilters(t *testing.T) {
if got := repIDs(replicaFilter{state: "start"}.apply(replicas())); got != "r2" {
t.Errorf("--state start = %s", got)
}
if got := repIDs(replicaFilter{node: "n2", state: "ready"}.apply(replicas())); got != "r1,r3" {
t.Errorf("combined = %s", got)
}
}