Track which replicas are actually being used, and summarise the fleet
CI / test (push) Successful in 1m28s
CI / test (push) Successful in 1m28s
Two things a cluster with many nodes and many GPUs needs that were missing, and they are the same thing seen from two distances. Idle tracking. The control plane knew what was placed and whether it was healthy, never whether anyone wanted it — so on a fleet with more models than cards there was no way to answer the first question anyone asks, which is what can I stop. The router is the only thing that knows, so it now reports it: a timestamp written in Acquire, where a request already meets an endpoint under a lock already held, and collected once a tick as a map. That shape is the point. Per-request notification would be a firehose carrying one bit — the newest timestamp wins — and this costs the same on four replicas as on four hundred. It travels through an interface in the direction opposite EndpointSink, for the same reason that one exists: the control plane decides what should run, the router knows whether anyone wants it, and neither should need to be in the other's process to say so. Unknown is kept distinct from idle throughout. A router that has just restarted knows nothing, and a zero timestamp would be the oldest idle replica in every comparison — so one restart would nominate an entire fleet for eviction. Absent from the map, "-" in the table, negative in the JSON, and IdleFor returns a second value rather than a duration anything could act on by mistake. ctl status. A per-node listing is right for four nodes and wrong for two hundred: how full am I, what is broken, what is holding cards nobody is using are aggregates, and reading them off a long table is a chore, which means it gets skipped. Capacity per GPU class with usage bars, free whole nodes per fabric — the number that decides whether a multi-node replica can land, since free cards on the wrong fabric are not capacity for a model that spans nodes — replica and model counts, models that cannot be placed, and what has gone unasked-for. Derived from the same state the tables show, so it cannot disagree with them. Unhealthy GPUs count as neither used nor free, because counting them free is how a cluster looks like it has room it does not. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1431426db0
commit
9d8cd5e284
15 files changed
+928
-20
No files matched your search
+18
-3
@@ -109,6 +109,7 @@ func newCtlCmd() *cobra.Command {
|
||||
// cert works on files rather than the admin API, so --server does not
|
||||
// apply to it; it lives here because this is where operators look.
|
||||
newBenchCmd(&asJSON),
|
||||
newStatusCmd(client, &asJSON),
|
||||
newEngineCheckCmd(),
|
||||
newToolCheckCmd(),
|
||||
newCertCmd(),
|
||||
@@ -157,6 +158,19 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
}
|
||||
|
||||
// idleColumn shows how long a replica has gone unused.
|
||||
//
|
||||
// "-" for a replica nothing has reported use of, which is emphatically not the
|
||||
// same as idle forever: it is what every replica looks like before its first
|
||||
// request, and what they all look like after a router restart. Showing that as
|
||||
// a very large number would make an untouched fleet read as an abandoned one.
|
||||
func idleColumn(sec int64) string {
|
||||
if sec < 0 {
|
||||
return "-"
|
||||
}
|
||||
return (time.Duration(sec) * time.Second).Round(time.Second).String()
|
||||
}
|
||||
|
||||
// memColumn shows a node's memory, distinguishing "did not say" from a number.
|
||||
// An older agent reports nothing, and that is not the same as a small machine.
|
||||
func memColumn(gb int) string {
|
||||
@@ -221,15 +235,16 @@ func newReplicasCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tNODES\tAGE\tENDPOINT")
|
||||
fmt.Fprintln(w, "REPLICA\tMODEL\tSTATE\tPOOL\tNODES\tAGE\tIDLE\tENDPOINT")
|
||||
for _, r := range reps {
|
||||
var nodes []string
|
||||
for _, s := range r.Shards {
|
||||
nodes = append(nodes, fmt.Sprintf("%s:%v", s.Node, s.GPUs))
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n",
|
||||
r.ID, r.Model, r.State, r.Pool, join(nodes),
|
||||
(time.Duration(r.AgeSec) * time.Second).String(), r.Endpoint)
|
||||
(time.Duration(r.AgeSec) * time.Second).String(),
|
||||
idleColumn(r.IdleSec), r.Endpoint)
|
||||
}
|
||||
_ = w.Flush()
|
||||
|
||||
|
||||
@@ -257,6 +257,12 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Cata
|
||||
// the service once it exists.
|
||||
fwd := &dispatcherRef{}
|
||||
ctl := control.New(ccfg, cat, fwd, sink, log)
|
||||
// The router is both where endpoints go and where activity comes back
|
||||
// from. It only reports usage when it is the thing serving requests, which
|
||||
// is why this is a type assertion rather than a parameter.
|
||||
if u, ok := sink.(control.UsageSource); ok {
|
||||
ctl.SetUsageSource(u)
|
||||
}
|
||||
svc := agentsvc.New(ctl.Inventory(), ctl.Reconciler(), requireIdentity, log)
|
||||
fwd.set(svc)
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
)
|
||||
|
||||
// newStatusCmd prints the whole fleet in one screen.
|
||||
//
|
||||
// `ctl nodes` is the right tool for four nodes and the wrong one for two
|
||||
// hundred. The questions that matter at that size are aggregates — how full am
|
||||
// I, what is broken, what is holding cards nobody is using — and reading them
|
||||
// off a long table is a chore, which means it gets skipped, which means nobody
|
||||
// notices until something fails.
|
||||
func newStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "status",
|
||||
Short: "The whole fleet in one answer: capacity, health, and what is idle",
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var s adminapi.Summary
|
||||
if err := client().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(s)
|
||||
}
|
||||
printSummary(cmd, s)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
func printSummary(cmd *cobra.Command, s adminapi.Summary) {
|
||||
out := cmd.OutOrStdout()
|
||||
|
||||
fmt.Fprintf(out, "NODES %d ready", s.Nodes.Ready)
|
||||
for _, x := range []struct {
|
||||
n int
|
||||
label string
|
||||
}{
|
||||
{s.Nodes.Cordoned, "cordoned"},
|
||||
{s.Nodes.Draining, "draining"},
|
||||
{s.Nodes.Unreachable, "unreachable"},
|
||||
} {
|
||||
if x.n > 0 {
|
||||
fmt.Fprintf(out, ", %d %s", x.n, x.label)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(out, " (of %d)\n", s.Nodes.Total)
|
||||
|
||||
if len(s.GPUs) > 0 {
|
||||
fmt.Fprintln(out, "\nGPUS")
|
||||
w := tab()
|
||||
fmt.Fprintln(w, " CLASS\tIN USE\tFREE\tUNHEALTHY\tUSAGE")
|
||||
for _, g := range s.GPUs {
|
||||
fmt.Fprintf(w, " %s\t%d/%d\t%d\t%s\t%s\n",
|
||||
g.Class, g.Used, g.Total, g.Free(), unhealthyCell(g.Unhealthy), bar(g.Used, g.Total))
|
||||
}
|
||||
_ = w.Flush()
|
||||
}
|
||||
|
||||
if len(s.Fabrics) > 0 {
|
||||
fmt.Fprintln(out, "\nFABRICS")
|
||||
w := tab()
|
||||
// The free-block column is the one that decides whether a multi-node
|
||||
// replica can land: free cards on the wrong fabric are not capacity for
|
||||
// a model that spans nodes, because a replica may never cross one.
|
||||
fmt.Fprintln(w, " FABRIC\tNODES\tFULLY FREE")
|
||||
for _, f := range s.Fabrics {
|
||||
fmt.Fprintf(w, " %s\t%d\t%d\n", f.Fabric, f.Nodes, f.LargestFreeBlock)
|
||||
}
|
||||
_ = w.Flush()
|
||||
}
|
||||
|
||||
fmt.Fprintf(out, "\nREPLICAS %d ready", s.Replicas.Ready)
|
||||
for _, x := range []struct {
|
||||
n int
|
||||
label string
|
||||
}{
|
||||
{s.Replicas.Starting, "starting"},
|
||||
{s.Replicas.Draining, "draining"},
|
||||
{s.Replicas.Failed, "failed"},
|
||||
} {
|
||||
if x.n > 0 {
|
||||
fmt.Fprintf(out, ", %d %s", x.n, x.label)
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n",
|
||||
s.Models.Satisfied, s.Models.Total)
|
||||
|
||||
if len(s.Unplaceable) > 0 {
|
||||
fmt.Fprintln(out, "\nNOT PLACED")
|
||||
for _, m := range s.Unplaceable {
|
||||
fmt.Fprintf(out, " %s\n", m)
|
||||
}
|
||||
fmt.Fprintln(out, "\n Why, per model: llmgw ctl explain-placement <model>")
|
||||
}
|
||||
|
||||
if len(s.IdleReplicas) > 0 {
|
||||
fmt.Fprintln(out, "\nIDLE")
|
||||
w := tab()
|
||||
fmt.Fprintln(w, " REPLICA\tMODEL\tGPUS\tUNUSED FOR")
|
||||
for _, r := range s.IdleReplicas {
|
||||
fmt.Fprintf(w, " %s\t%s\t%d\t%s\n", r.ID, r.Model, r.GPUs,
|
||||
(time.Duration(r.IdleSec) * time.Second).Round(time.Minute))
|
||||
}
|
||||
_ = w.Flush()
|
||||
fmt.Fprintln(out, "\n These hold hardware nothing is asking for.")
|
||||
}
|
||||
}
|
||||
|
||||
// bar draws usage so a column of classes can be read at a glance rather than
|
||||
// compared digit by digit.
|
||||
func bar(used, total int) string {
|
||||
const width = 20
|
||||
if total <= 0 {
|
||||
return ""
|
||||
}
|
||||
filled := used * width / total
|
||||
if filled > width {
|
||||
filled = width
|
||||
}
|
||||
b := make([]rune, width)
|
||||
for i := range b {
|
||||
if i < filled {
|
||||
b[i] = '#'
|
||||
} else {
|
||||
b[i] = '.'
|
||||
}
|
||||
}
|
||||
return fmt.Sprintf("%s %3d%%", string(b), used*100/total)
|
||||
}
|
||||
|
||||
// unhealthyCell keeps a zero quiet. A column of noughts trains the eye to skip
|
||||
// the column, which is the opposite of what it is for.
|
||||
func unhealthyCell(n int) string {
|
||||
if n == 0 {
|
||||
return "-"
|
||||
}
|
||||
return fmt.Sprintf("%d", n)
|
||||
}
|
||||
@@ -0,0 +1,35 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestBarStaysWithinItsWidth(t *testing.T) {
|
||||
// It is drawn from counts that can disagree briefly — a GPU freed between
|
||||
// two reads — and a bar that overruns its column wrecks the table it is in.
|
||||
for _, tc := range [][2]int{{0, 0}, {0, 8}, {8, 8}, {9, 8}, {100, 8}} {
|
||||
got := bar(tc[0], tc[1])
|
||||
if tc[1] == 0 {
|
||||
if got != "" {
|
||||
t.Errorf("bar(%d,%d) = %q, want empty for an empty class", tc[0], tc[1], got)
|
||||
}
|
||||
continue
|
||||
}
|
||||
if n := strings.Count(got, "#") + strings.Count(got, "."); n != 20 {
|
||||
t.Errorf("bar(%d,%d) drew %d cells: %q", tc[0], tc[1], n, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdleColumnDistinguishesUnknownFromIdle(t *testing.T) {
|
||||
// Every replica looks unreported before its first request and after a
|
||||
// router restart. Showing that as a very large duration would make an
|
||||
// untouched fleet read as an abandoned one.
|
||||
if got := idleColumn(-1); got != "-" {
|
||||
t.Errorf("idleColumn(-1) = %q, want a dash", got)
|
||||
}
|
||||
if got := idleColumn(90); got != "1m30s" {
|
||||
t.Errorf("idleColumn(90) = %q", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user