Five nodes fit in a table. Fifty do not, and this is built for people who already have racks of GPUs — so the flags are how a question gets asked: which nodes on this fabric, which of this GPU class, which are not ready, which replicas have a shard on this machine. Filtered here rather than in the API, so --json is narrowed the same way the table is. A server-side filter would be two implementations of one predicate, and the JSON one is the one nobody would notice had drifted. An empty result says which case it is. A filter that matched nothing and an empty fleet call for opposite next steps — one is a cluster to look at, the other is a word to correct — and printing "no nodes" for both makes a mistyped fabric name read as an outage. A state matches on prefix because states are few and distinct and somebody scanning a fleet types quickly. A pool matches either half, because it is fabric/class and both are worth narrowing by. A replica is found from any of its nodes, since one spanning two machines is on both. --label takes a bare key as "carries this at all", which is how you ask for every node with a role without caring which. The flag's help said so before the parser did; the test caught them disagreeing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
3a4260c8c5
commit
8e57171c4f
3 files changed
+358
-1
No files matched your search
+33
-1
@@ -133,18 +133,34 @@ func newCtlCmd() *cobra.Command {
|
||||
}
|
||||
|
||||
func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
var f nodeFilter
|
||||
c := &cobra.Command{
|
||||
Use: "nodes",
|
||||
Short: "List nodes and their capacity",
|
||||
Long: "On a fleet too large to read a whole table of, the filters are how a\n" +
|
||||
"question gets asked: which nodes on this fabric, which of this GPU class,\n" +
|
||||
"which are not ready. They combine, and they apply to --json too.",
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var nodes []adminapi.NodeView
|
||||
if err := client().get(cmd.Context(), "/api/v1/nodes", &nodes); err != nil {
|
||||
return err
|
||||
}
|
||||
total := len(nodes)
|
||||
nodes, err := f.apply(nodes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(nodes)
|
||||
}
|
||||
if len(nodes) == 0 {
|
||||
// A filter that matched nothing and an empty fleet call for
|
||||
// opposite next steps: one is a cluster to look at, the other
|
||||
// is a word to correct.
|
||||
if !f.empty() && total > 0 {
|
||||
fmt.Println(noMatches("nodes", f.describe(), total))
|
||||
return nil
|
||||
}
|
||||
fmt.Println("No nodes have registered.")
|
||||
return nil
|
||||
}
|
||||
@@ -172,6 +188,12 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&f.fabric, "fabric", "", "only nodes attached to this fabric")
|
||||
c.Flags().StringVar(&f.class, "class", "", "only nodes with this GPU class")
|
||||
c.Flags().StringVar(&f.state, "state", "", "only nodes in this state (prefix match)")
|
||||
c.Flags().StringArrayVar(&f.labels, "label", nil,
|
||||
"only nodes carrying key=value; repeatable, and key alone matches any value")
|
||||
return c
|
||||
}
|
||||
|
||||
// modelStatus says what is happening to a model, in the words that decide what
|
||||
@@ -275,6 +297,7 @@ func newModelsCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
|
||||
func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
var model string
|
||||
var f replicaFilter
|
||||
c := &cobra.Command{
|
||||
Use: "replicas",
|
||||
Short: "List replicas and the shards they are made of",
|
||||
@@ -287,10 +310,16 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
if err := client().get(cmd.Context(), path, &reps); err != nil {
|
||||
return err
|
||||
}
|
||||
total := len(reps)
|
||||
reps = f.apply(reps)
|
||||
if *asJSON {
|
||||
return emitJSON(reps)
|
||||
}
|
||||
if len(reps) == 0 {
|
||||
if !f.empty() && total > 0 {
|
||||
fmt.Println(noMatches("replicas", f.describe(), total))
|
||||
return nil
|
||||
}
|
||||
fmt.Println("No replicas are running.")
|
||||
return nil
|
||||
}
|
||||
@@ -323,6 +352,9 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&model, "model", "", "only this model's replicas")
|
||||
c.Flags().StringVar(&f.node, "node", "", "only replicas with a shard on this node")
|
||||
c.Flags().StringVar(&f.state, "state", "", "only replicas in this state (prefix match)")
|
||||
c.Flags().StringVar(&f.pool, "pool", "", "only this pool; matches either half of fabric/class")
|
||||
return c
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,170 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// nodeFilter narrows a node listing.
|
||||
//
|
||||
// Client-side on the whole list rather than as query parameters, because the
|
||||
// list is small even on a large fleet — a few hundred rows — and doing it here
|
||||
// means --json is filtered the same way the table is. A server-side filter
|
||||
// would be two implementations of the same predicate, and the JSON one is the
|
||||
// one nobody would notice had drifted.
|
||||
type nodeFilter struct {
|
||||
fabric string
|
||||
class string
|
||||
state string
|
||||
labels []string
|
||||
}
|
||||
|
||||
// empty reports whether this filter would narrow anything.
|
||||
//
|
||||
// The difference matters: an empty fleet and a filter that matched nothing are
|
||||
// different situations with different next steps, and "no nodes" for both
|
||||
// makes a mistyped fabric look like an outage.
|
||||
func (f nodeFilter) empty() bool {
|
||||
return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0
|
||||
}
|
||||
|
||||
func (f nodeFilter) describe() string {
|
||||
var parts []string
|
||||
if f.fabric != "" {
|
||||
parts = append(parts, "fabric "+f.fabric)
|
||||
}
|
||||
if f.class != "" {
|
||||
parts = append(parts, "class "+f.class)
|
||||
}
|
||||
if f.state != "" {
|
||||
parts = append(parts, "state "+f.state)
|
||||
}
|
||||
for _, l := range f.labels {
|
||||
parts = append(parts, "label "+l)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
func (f nodeFilter) apply(in []adminapi.NodeView) ([]adminapi.NodeView, error) {
|
||||
want := map[string]string{}
|
||||
for _, l := range f.labels {
|
||||
// A bare key is allowed and means "carries this at all", which is how
|
||||
// an operator asks for every node with a role without caring which.
|
||||
// An empty key is not: "=value" is a typo, and matching everything
|
||||
// would look like the filter having worked.
|
||||
k, v, _ := strings.Cut(l, "=")
|
||||
if k == "" {
|
||||
return nil, fmt.Errorf("--label needs a key: %q has none", l)
|
||||
}
|
||||
want[k] = v
|
||||
}
|
||||
|
||||
out := make([]adminapi.NodeView, 0, len(in))
|
||||
for _, n := range in {
|
||||
if f.fabric != "" && !slices.Contains(n.Fabrics, f.fabric) {
|
||||
continue
|
||||
}
|
||||
if f.class != "" && !slices.Contains(n.Classes, f.class) {
|
||||
continue
|
||||
}
|
||||
// Prefix rather than equality, so --state ready matches and --state re
|
||||
// does too. Node states are few and distinct, and an operator scanning
|
||||
// a fleet is typing quickly.
|
||||
if f.state != "" && !strings.HasPrefix(n.State, strings.ToLower(f.state)) {
|
||||
continue
|
||||
}
|
||||
if !hasLabels(n.Labels, want) {
|
||||
continue
|
||||
}
|
||||
out = append(out, n)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// hasLabels reports whether a node carries every wanted label.
|
||||
//
|
||||
// An empty value means "has this key at all", which is how an operator asks
|
||||
// for every node that carries a role without caring which.
|
||||
func hasLabels(have map[string]string, want map[string]string) bool {
|
||||
for k, v := range want {
|
||||
got, ok := have[k]
|
||||
if !ok {
|
||||
return false
|
||||
}
|
||||
if v != "" && got != v {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// replicaFilter narrows a replica listing.
|
||||
type replicaFilter struct {
|
||||
node string
|
||||
state string
|
||||
pool string
|
||||
}
|
||||
|
||||
func (f replicaFilter) empty() bool {
|
||||
return f.node == "" && f.state == "" && f.pool == ""
|
||||
}
|
||||
|
||||
func (f replicaFilter) describe() string {
|
||||
var parts []string
|
||||
if f.node != "" {
|
||||
parts = append(parts, "node "+f.node)
|
||||
}
|
||||
if f.state != "" {
|
||||
parts = append(parts, "state "+f.state)
|
||||
}
|
||||
if f.pool != "" {
|
||||
parts = append(parts, "pool "+f.pool)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
func (f replicaFilter) apply(in []adminapi.ReplicaView) []adminapi.ReplicaView {
|
||||
out := make([]adminapi.ReplicaView, 0, len(in))
|
||||
for _, r := range in {
|
||||
// Through the shards, because that is where the nodes are: a replica
|
||||
// spanning two machines is on both, and asking "what is on node05"
|
||||
// has to find it from either side.
|
||||
if f.node != "" && !onNode(r, f.node) {
|
||||
continue
|
||||
}
|
||||
if f.state != "" && !strings.HasPrefix(strings.ToLower(r.State), strings.ToLower(f.state)) {
|
||||
continue
|
||||
}
|
||||
// Substring, because a pool is "fabric/class" and both halves are
|
||||
// things an operator narrows by: --pool ib-a and --pool h100-80g are
|
||||
// both questions worth asking.
|
||||
if f.pool != "" && !strings.Contains(r.Pool, f.pool) {
|
||||
continue
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// onNode reports whether any shard of a replica runs on a node.
|
||||
func onNode(r adminapi.ReplicaView, node string) bool {
|
||||
for _, sh := range r.Shards {
|
||||
if sh.Node == node {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// noMatches is what to print when a filter excluded everything.
|
||||
//
|
||||
// Distinct from "there are none", because they call for opposite next steps:
|
||||
// one is a cluster to look at and the other is a word to correct. A mistyped
|
||||
// fabric name otherwise reads as an outage.
|
||||
func noMatches(kind, what string, total int) string {
|
||||
return fmt.Sprintf("No %s match %s. %d %s in total; run without the filter to see them.",
|
||||
kind, what, total, kind)
|
||||
}
|
||||
@@ -0,0 +1,155 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
func fleet() []adminapi.NodeView {
|
||||
return []adminapi.NodeView{
|
||||
{ID: "n1", State: "ready", Fabrics: []string{"ib-a"}, Classes: []string{"h100-80g"},
|
||||
Labels: map[string]string{"role": "cpu-large", "rack": "r1"}},
|
||||
{ID: "n2", State: "ready", Fabrics: []string{"ib-b"}, Classes: []string{"h100-80g"},
|
||||
Labels: map[string]string{"role": "cpu-large"}},
|
||||
{ID: "n3", State: "cordoned", Fabrics: []string{"ib-a"}, Classes: []string{"l40s-45g"}},
|
||||
{ID: "n4", State: "ready"},
|
||||
}
|
||||
}
|
||||
|
||||
func ids(in []adminapi.NodeView) string {
|
||||
var out []string
|
||||
for _, n := range in {
|
||||
out = append(out, n.ID)
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
func mustApply(t *testing.T, f nodeFilter) []adminapi.NodeView {
|
||||
t.Helper()
|
||||
got, err := f.apply(fleet())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return got
|
||||
}
|
||||
|
||||
func TestFilteringByFabricClassAndState(t *testing.T) {
|
||||
if got := ids(mustApply(t, nodeFilter{fabric: "ib-a"})); got != "n1,n3" {
|
||||
t.Errorf("--fabric ib-a = %s", got)
|
||||
}
|
||||
if got := ids(mustApply(t, nodeFilter{class: "l40s-45g"})); got != "n3" {
|
||||
t.Errorf("--class l40s-45g = %s", got)
|
||||
}
|
||||
if got := ids(mustApply(t, nodeFilter{state: "cordoned"})); got != "n3" {
|
||||
t.Errorf("--state cordoned = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFiltersCombine(t *testing.T) {
|
||||
// Which is the question worth asking on a large fleet: not "what is on
|
||||
// this fabric" but "what on this fabric is not ready".
|
||||
if got := ids(mustApply(t, nodeFilter{fabric: "ib-a", state: "ready"})); got != "n1" {
|
||||
t.Errorf("combined = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStatePrefixIsEnough(t *testing.T) {
|
||||
// Node states are few and distinct, and somebody scanning a fleet types
|
||||
// quickly.
|
||||
if got := ids(mustApply(t, nodeFilter{state: "cord"})); got != "n3" {
|
||||
t.Errorf("--state cord = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestALabelKeyAloneMatchesAnyValue(t *testing.T) {
|
||||
// How an operator asks for every node carrying a role without caring which.
|
||||
if got := ids(mustApply(t, nodeFilter{labels: []string{"role"}})); got != "n1,n2" {
|
||||
t.Errorf("--label role = %s", got)
|
||||
}
|
||||
if got := ids(mustApply(t, nodeFilter{labels: []string{"role=cpu-large", "rack=r1"}})); got != "n1" {
|
||||
t.Errorf("two labels = %s", got)
|
||||
}
|
||||
if got := ids(mustApply(t, nodeFilter{labels: []string{"rack=r9"}})); got != "" {
|
||||
t.Errorf("a label nothing carries matched %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAMalformedLabelIsRefusedRatherThanIgnored(t *testing.T) {
|
||||
// Silently ignoring it would show the whole fleet, which reads as the
|
||||
// filter having matched everything rather than as a typo.
|
||||
_, err := nodeFilter{labels: []string{"=value"}}.apply(fleet())
|
||||
if err == nil {
|
||||
t.Error("a label with no key was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoFilterIsRecognisedAsNone(t *testing.T) {
|
||||
// It decides which of two messages an empty result gets.
|
||||
if !(nodeFilter{}).empty() {
|
||||
t.Error("an unset filter did not report itself as empty")
|
||||
}
|
||||
if (nodeFilter{state: "ready"}).empty() {
|
||||
t.Error("a set filter reported itself as empty")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheEmptyMessageSaysWhichCaseItIs(t *testing.T) {
|
||||
// A mistyped fabric name otherwise reads as an outage.
|
||||
msg := noMatches("nodes", "fabric ib-c", 12)
|
||||
for _, want := range []string{"fabric ib-c", "12"} {
|
||||
if !strings.Contains(msg, want) {
|
||||
t.Errorf("the message does not mention %q: %s", want, msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func replicas() []adminapi.ReplicaView {
|
||||
return []adminapi.ReplicaView{
|
||||
{ID: "r1", Model: "big", State: "ready", Pool: "ib-a/h100-80g",
|
||||
Shards: []adminapi.ShardView{{Node: "n1"}, {Node: "n2"}}},
|
||||
{ID: "r2", Model: "small", State: "starting", Pool: "none/cpu",
|
||||
Shards: []adminapi.ShardView{{Node: "n5"}}},
|
||||
{ID: "r3", Model: "small", State: "ready", Pool: "ib-b/h100-80g",
|
||||
Shards: []adminapi.ShardView{{Node: "n2"}}},
|
||||
}
|
||||
}
|
||||
|
||||
func repIDs(in []adminapi.ReplicaView) string {
|
||||
var out []string
|
||||
for _, r := range in {
|
||||
out = append(out, r.ID)
|
||||
}
|
||||
return strings.Join(out, ",")
|
||||
}
|
||||
|
||||
func TestAReplicaIsFoundFromEitherOfItsNodes(t *testing.T) {
|
||||
// A replica spanning two machines is on both, and "what is running on n2"
|
||||
// has to find it from either side.
|
||||
if got := repIDs(replicaFilter{node: "n1"}.apply(replicas())); got != "r1" {
|
||||
t.Errorf("--node n1 = %s", got)
|
||||
}
|
||||
if got := repIDs(replicaFilter{node: "n2"}.apply(replicas())); got != "r1,r3" {
|
||||
t.Errorf("--node n2 = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPoolMatchesEitherHalf(t *testing.T) {
|
||||
// A pool is fabric/class and both are things worth narrowing by.
|
||||
if got := repIDs(replicaFilter{pool: "ib-a"}.apply(replicas())); got != "r1" {
|
||||
t.Errorf("--pool ib-a = %s", got)
|
||||
}
|
||||
if got := repIDs(replicaFilter{pool: "h100-80g"}.apply(replicas())); got != "r1,r3" {
|
||||
t.Errorf("--pool h100-80g = %s", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestReplicaStateFilters(t *testing.T) {
|
||||
if got := repIDs(replicaFilter{state: "start"}.apply(replicas())); got != "r2" {
|
||||
t.Errorf("--state start = %s", got)
|
||||
}
|
||||
if got := repIDs(replicaFilter{node: "n2", state: "ready"}.apply(replicas())); got != "r1,r3" {
|
||||
t.Errorf("combined = %s", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user