package cli import ( "fmt" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" ) // newStatusCmd prints the whole fleet in one screen. // // `ctl nodes` is the right tool for four nodes and the wrong one for two // hundred. The questions that matter at that size are aggregates — how full am // I, what is broken, what is holding cards nobody is using — and reading them // off a long table is a chore, which means it gets skipped, which means nobody // notices until something fails. func newStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { return &cobra.Command{ Use: "status", Short: "The whole fleet in one answer: capacity, health, and what is idle", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { var s adminapi.Summary if err := client().get(cmd.Context(), "/api/v1/summary", &s); err != nil { return err } if *asJSON { return emitJSON(s) } printSummary(cmd, s) return nil }, } } func printSummary(cmd *cobra.Command, s adminapi.Summary) { out := cmd.OutOrStdout() fmt.Fprintf(out, "NODES %d ready", s.Nodes.Ready) for _, x := range []struct { n int label string }{ {s.Nodes.Cordoned, "cordoned"}, {s.Nodes.Draining, "draining"}, {s.Nodes.Unreachable, "unreachable"}, } { if x.n > 0 { fmt.Fprintf(out, ", %d %s", x.n, x.label) } } fmt.Fprintf(out, " (of %d)\n", s.Nodes.Total) if len(s.GPUs) > 0 { fmt.Fprintln(out, "\nGPUS") w := tab() fmt.Fprintln(w, " CLASS\tIN USE\tFREE\tUNHEALTHY\tUSAGE") for _, g := range s.GPUs { fmt.Fprintf(w, " %s\t%d/%d\t%d\t%s\t%s\n", g.Class, g.Used, g.Total, g.Free(), unhealthyCell(g.Unhealthy), bar(g.Used, g.Total)) } _ = w.Flush() } if len(s.Fabrics) > 0 { fmt.Fprintln(out, "\nFABRICS") w := tab() // The free-block column is the one that decides whether a multi-node // replica can land: free cards on the wrong fabric are not capacity for // a model that spans nodes, because a replica may never cross one. fmt.Fprintln(w, " FABRIC\tNODES\tFULLY FREE\tWEIGHTS") for _, f := range s.Fabrics { fmt.Fprintf(w, " %s\t%d\t%d\t%s\n", f.Fabric, f.Nodes, f.LargestFreeBlock, weightsCell(f)) } _ = w.Flush() } fmt.Fprintf(out, "\nREPLICAS %d ready", s.Replicas.Ready) for _, x := range []struct { n int label string }{ {s.Replicas.Starting, "starting"}, {s.Replicas.Draining, "draining"}, {s.Replicas.Failed, "failed"}, } { if x.n > 0 { fmt.Fprintf(out, ", %d %s", x.n, x.label) } } fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n", s.Models.Satisfied, s.Models.Total) if len(s.Unplaceable) > 0 { fmt.Fprintln(out, "\nNOT PLACED") for _, m := range s.Unplaceable { fmt.Fprintf(out, " %s\n", m) } fmt.Fprintln(out, "\n Why, per model: llmgw ctl explain-placement ") } if len(s.IdleReplicas) > 0 { fmt.Fprintln(out, "\nIDLE") w := tab() fmt.Fprintln(w, " REPLICA\tMODEL\tGPUS\tUNUSED FOR") for _, r := range s.IdleReplicas { fmt.Fprintf(w, " %s\t%s\t%d\t%s\n", r.ID, r.Model, r.GPUs, (time.Duration(r.IdleSec) * time.Second).Round(time.Minute)) } _ = w.Flush() fmt.Fprintln(out, "\n These hold hardware nothing is asking for.") } } // weightsCell says what a fabric holds, and stays quiet when nobody asked. // // Fabrics are not connected to one another, so weights in one are not weights // in another: a model seeded into a single fabric leaves the rest of the // cluster unable to serve it without first moving tens of gigabytes over the // management network. This is the column that shows that before someone // discovers it during a scale-up. func weightsCell(f adminapi.FabricUse) string { if f.NodesWithWeights == 0 { // Nobody on this fabric was asked to look, which is the default and is // not the same as an empty fabric. return "-" } return fmt.Sprintf("%d model(s) on %d/%d nodes", f.Models, f.NodesWithWeights, f.Nodes) } // bar draws usage so a column of classes can be read at a glance rather than // compared digit by digit. func bar(used, total int) string { const width = 20 if total <= 0 { return "" } filled := used * width / total if filled > width { filled = width } b := make([]rune, width) for i := range b { if i < filled { b[i] = '#' } else { b[i] = '.' } } return fmt.Sprintf("%s %3d%%", string(b), used*100/total) } // unhealthyCell keeps a zero quiet. A column of noughts trains the eye to skip // the column, which is the opposite of what it is for. func unhealthyCell(n int) string { if n == 0 { return "-" } return fmt.Sprintf("%d", n) }