CI / test (push) Successful in 1m42s
Five nodes fit in a table. Fifty do not, and this is built for people who already have racks of GPUs — so the flags are how a question gets asked: which nodes on this fabric, which of this GPU class, which are not ready, which replicas have a shard on this machine. Filtered here rather than in the API, so --json is narrowed the same way the table is. A server-side filter would be two implementations of one predicate, and the JSON one is the one nobody would notice had drifted. An empty result says which case it is. A filter that matched nothing and an empty fleet call for opposite next steps — one is a cluster to look at, the other is a word to correct — and printing "no nodes" for both makes a mistyped fabric name read as an outage. A state matches on prefix because states are few and distinct and somebody scanning a fleet types quickly. A pool matches either half, because it is fabric/class and both are worth narrowing by. A replica is found from any of its nodes, since one spanning two machines is on both. --label takes a bare key as "carries this at all", which is how you ask for every node with a role without caring which. The flag's help said so before the parser did; the test caught them disagreeing. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
171 lines
4.7 KiB
Go
171 lines
4.7 KiB
Go
package cli
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
)
|
|
|
|
// nodeFilter narrows a node listing.
|
|
//
|
|
// Client-side on the whole list rather than as query parameters, because the
|
|
// list is small even on a large fleet — a few hundred rows — and doing it here
|
|
// means --json is filtered the same way the table is. A server-side filter
|
|
// would be two implementations of the same predicate, and the JSON one is the
|
|
// one nobody would notice had drifted.
|
|
type nodeFilter struct {
|
|
fabric string
|
|
class string
|
|
state string
|
|
labels []string
|
|
}
|
|
|
|
// empty reports whether this filter would narrow anything.
|
|
//
|
|
// The difference matters: an empty fleet and a filter that matched nothing are
|
|
// different situations with different next steps, and "no nodes" for both
|
|
// makes a mistyped fabric look like an outage.
|
|
func (f nodeFilter) empty() bool {
|
|
return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0
|
|
}
|
|
|
|
func (f nodeFilter) describe() string {
|
|
var parts []string
|
|
if f.fabric != "" {
|
|
parts = append(parts, "fabric "+f.fabric)
|
|
}
|
|
if f.class != "" {
|
|
parts = append(parts, "class "+f.class)
|
|
}
|
|
if f.state != "" {
|
|
parts = append(parts, "state "+f.state)
|
|
}
|
|
for _, l := range f.labels {
|
|
parts = append(parts, "label "+l)
|
|
}
|
|
return strings.Join(parts, ", ")
|
|
}
|
|
|
|
func (f nodeFilter) apply(in []adminapi.NodeView) ([]adminapi.NodeView, error) {
|
|
want := map[string]string{}
|
|
for _, l := range f.labels {
|
|
// A bare key is allowed and means "carries this at all", which is how
|
|
// an operator asks for every node with a role without caring which.
|
|
// An empty key is not: "=value" is a typo, and matching everything
|
|
// would look like the filter having worked.
|
|
k, v, _ := strings.Cut(l, "=")
|
|
if k == "" {
|
|
return nil, fmt.Errorf("--label needs a key: %q has none", l)
|
|
}
|
|
want[k] = v
|
|
}
|
|
|
|
out := make([]adminapi.NodeView, 0, len(in))
|
|
for _, n := range in {
|
|
if f.fabric != "" && !slices.Contains(n.Fabrics, f.fabric) {
|
|
continue
|
|
}
|
|
if f.class != "" && !slices.Contains(n.Classes, f.class) {
|
|
continue
|
|
}
|
|
// Prefix rather than equality, so --state ready matches and --state re
|
|
// does too. Node states are few and distinct, and an operator scanning
|
|
// a fleet is typing quickly.
|
|
if f.state != "" && !strings.HasPrefix(n.State, strings.ToLower(f.state)) {
|
|
continue
|
|
}
|
|
if !hasLabels(n.Labels, want) {
|
|
continue
|
|
}
|
|
out = append(out, n)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// hasLabels reports whether a node carries every wanted label.
|
|
//
|
|
// An empty value means "has this key at all", which is how an operator asks
|
|
// for every node that carries a role without caring which.
|
|
func hasLabels(have map[string]string, want map[string]string) bool {
|
|
for k, v := range want {
|
|
got, ok := have[k]
|
|
if !ok {
|
|
return false
|
|
}
|
|
if v != "" && got != v {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// replicaFilter narrows a replica listing.
|
|
type replicaFilter struct {
|
|
node string
|
|
state string
|
|
pool string
|
|
}
|
|
|
|
func (f replicaFilter) empty() bool {
|
|
return f.node == "" && f.state == "" && f.pool == ""
|
|
}
|
|
|
|
func (f replicaFilter) describe() string {
|
|
var parts []string
|
|
if f.node != "" {
|
|
parts = append(parts, "node "+f.node)
|
|
}
|
|
if f.state != "" {
|
|
parts = append(parts, "state "+f.state)
|
|
}
|
|
if f.pool != "" {
|
|
parts = append(parts, "pool "+f.pool)
|
|
}
|
|
return strings.Join(parts, ", ")
|
|
}
|
|
|
|
func (f replicaFilter) apply(in []adminapi.ReplicaView) []adminapi.ReplicaView {
|
|
out := make([]adminapi.ReplicaView, 0, len(in))
|
|
for _, r := range in {
|
|
// Through the shards, because that is where the nodes are: a replica
|
|
// spanning two machines is on both, and asking "what is on node05"
|
|
// has to find it from either side.
|
|
if f.node != "" && !onNode(r, f.node) {
|
|
continue
|
|
}
|
|
if f.state != "" && !strings.HasPrefix(strings.ToLower(r.State), strings.ToLower(f.state)) {
|
|
continue
|
|
}
|
|
// Substring, because a pool is "fabric/class" and both halves are
|
|
// things an operator narrows by: --pool ib-a and --pool h100-80g are
|
|
// both questions worth asking.
|
|
if f.pool != "" && !strings.Contains(r.Pool, f.pool) {
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// onNode reports whether any shard of a replica runs on a node.
|
|
func onNode(r adminapi.ReplicaView, node string) bool {
|
|
for _, sh := range r.Shards {
|
|
if sh.Node == node {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// noMatches is what to print when a filter excluded everything.
|
|
//
|
|
// Distinct from "there are none", because they call for opposite next steps:
|
|
// one is a cluster to look at and the other is a word to correct. A mistyped
|
|
// fabric name otherwise reads as an outage.
|
|
func noMatches(kind, what string, total int) string {
|
|
return fmt.Sprintf("No %s match %s. %d %s in total; run without the filter to see them.",
|
|
kind, what, total, kind)
|
|
}
|