Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
189 lines
5.2 KiB
Go
189 lines
5.2 KiB
Go
package cli
|
|
|
|
import (
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
|
)
|
|
|
|
// nodeFilter narrows a node listing.
|
|
//
|
|
// Client-side on the whole list rather than as query parameters, because the
|
|
// list is small even on a large fleet — a few hundred rows — and doing it here
|
|
// means --json is filtered the same way the table is. A server-side filter
|
|
// would be two implementations of the same predicate, and the JSON one is the
|
|
// one nobody would notice had drifted.
|
|
type nodeFilter struct {
|
|
fabric string
|
|
class string
|
|
state string
|
|
labels []string
|
|
// domain is level:name, rack:r12-07.
|
|
domain string
|
|
}
|
|
|
|
// empty reports whether this filter would narrow anything.
|
|
//
|
|
// The difference matters: an empty fleet and a filter that matched nothing are
|
|
// different situations with different next steps, and "no nodes" for both
|
|
// makes a mistyped fabric look like an outage.
|
|
func (f nodeFilter) empty() bool {
|
|
return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0 && f.domain == ""
|
|
}
|
|
|
|
func (f nodeFilter) describe() string {
|
|
var parts []string
|
|
if f.fabric != "" {
|
|
parts = append(parts, "fabric "+f.fabric)
|
|
}
|
|
if f.class != "" {
|
|
parts = append(parts, "class "+f.class)
|
|
}
|
|
if f.state != "" {
|
|
parts = append(parts, "state "+f.state)
|
|
}
|
|
for _, l := range f.labels {
|
|
parts = append(parts, "label "+l)
|
|
}
|
|
if f.domain != "" {
|
|
parts = append(parts, "domain "+f.domain)
|
|
}
|
|
return strings.Join(parts, ", ")
|
|
}
|
|
|
|
func (f nodeFilter) apply(in []adminapi.NodeView) ([]adminapi.NodeView, error) {
|
|
want := map[string]string{}
|
|
for _, l := range f.labels {
|
|
// A bare key is allowed and means "carries this at all", which is how
|
|
// an operator asks for every node with a role without caring which.
|
|
// An empty key is not: "=value" is a typo, and matching everything
|
|
// would look like the filter having worked.
|
|
k, v, _ := strings.Cut(l, "=")
|
|
if k == "" {
|
|
return nil, fmt.Errorf("--label needs a key: %q has none", l)
|
|
}
|
|
want[k] = v
|
|
}
|
|
|
|
var domain facility.Domain
|
|
if f.domain != "" {
|
|
d, err := facility.ParseDomain(f.domain)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("--domain: %w", err)
|
|
}
|
|
domain = d
|
|
}
|
|
|
|
out := make([]adminapi.NodeView, 0, len(in))
|
|
for _, n := range in {
|
|
if f.fabric != "" && !slices.Contains(n.Fabrics, f.fabric) {
|
|
continue
|
|
}
|
|
if domain != "" && (n.Location == nil || !slices.Contains(n.Location.Domains(), domain)) {
|
|
continue
|
|
}
|
|
if f.class != "" && !slices.Contains(n.Classes, f.class) {
|
|
continue
|
|
}
|
|
// Prefix rather than equality, so --state ready matches and --state re
|
|
// does too. Node states are few and distinct, and an operator scanning
|
|
// a fleet is typing quickly.
|
|
if f.state != "" && !strings.HasPrefix(n.State, strings.ToLower(f.state)) {
|
|
continue
|
|
}
|
|
if !hasLabels(n.Labels, want) {
|
|
continue
|
|
}
|
|
out = append(out, n)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// hasLabels reports whether a node carries every wanted label.
|
|
//
|
|
// An empty value means "has this key at all", which is how an operator asks
|
|
// for every node that carries a role without caring which.
|
|
func hasLabels(have map[string]string, want map[string]string) bool {
|
|
for k, v := range want {
|
|
got, ok := have[k]
|
|
if !ok {
|
|
return false
|
|
}
|
|
if v != "" && got != v {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
|
|
// replicaFilter narrows a replica listing.
|
|
type replicaFilter struct {
|
|
node string
|
|
state string
|
|
pool string
|
|
}
|
|
|
|
func (f replicaFilter) empty() bool {
|
|
return f.node == "" && f.state == "" && f.pool == ""
|
|
}
|
|
|
|
func (f replicaFilter) describe() string {
|
|
var parts []string
|
|
if f.node != "" {
|
|
parts = append(parts, "node "+f.node)
|
|
}
|
|
if f.state != "" {
|
|
parts = append(parts, "state "+f.state)
|
|
}
|
|
if f.pool != "" {
|
|
parts = append(parts, "pool "+f.pool)
|
|
}
|
|
return strings.Join(parts, ", ")
|
|
}
|
|
|
|
func (f replicaFilter) apply(in []adminapi.ReplicaView) []adminapi.ReplicaView {
|
|
out := make([]adminapi.ReplicaView, 0, len(in))
|
|
for _, r := range in {
|
|
// Through the shards, because that is where the nodes are: a replica
|
|
// spanning two machines is on both, and asking "what is on node05"
|
|
// has to find it from either side.
|
|
if f.node != "" && !onNode(r, f.node) {
|
|
continue
|
|
}
|
|
if f.state != "" && !strings.HasPrefix(strings.ToLower(r.State), strings.ToLower(f.state)) {
|
|
continue
|
|
}
|
|
// Substring, because a pool is "fabric/class" and both halves are
|
|
// things an operator narrows by: --pool ib-a and --pool h100-80g are
|
|
// both questions worth asking.
|
|
if f.pool != "" && !strings.Contains(r.Pool, f.pool) {
|
|
continue
|
|
}
|
|
out = append(out, r)
|
|
}
|
|
return out
|
|
}
|
|
|
|
// onNode reports whether any shard of a replica runs on a node.
|
|
func onNode(r adminapi.ReplicaView, node string) bool {
|
|
for _, sh := range r.Shards {
|
|
if sh.Node == node {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
// noMatches is what to print when a filter excluded everything.
|
|
//
|
|
// Distinct from "there are none", because they call for opposite next steps:
|
|
// one is a cluster to look at and the other is a word to correct. A mistyped
|
|
// fabric name otherwise reads as an outage.
|
|
func noMatches(kind, what string, total int) string {
|
|
return fmt.Sprintf("No %s match %s. %d %s in total; run without the filter to see them.",
|
|
kind, what, total, kind)
|
|
}
|