package cli import ( "fmt" "strings" "time" "github.com/spf13/cobra" "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" ) // newStatusCmd prints the whole fleet in one screen. // // `ctl nodes` is the right tool for four nodes and the wrong one for two // hundred. The questions that matter at that size are aggregates — how full am // I, what is broken, what is holding cards nobody is using — and reading them // off a long table is a chore, which means it gets skipped, which means nobody // notices until something fails. func newStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { return &cobra.Command{ Use: "status", Short: "The whole fleet in one answer: capacity, health, and what is idle", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, _ []string) error { var s adminapi.Summary if err := client().get(cmd.Context(), "/api/v1/summary", &s); err != nil { return err } if *asJSON { return emitJSON(s) } printSummary(cmd, s) return nil }, } } func printSummary(cmd *cobra.Command, s adminapi.Summary) { out := cmd.OutOrStdout() fmt.Fprintf(out, "NODES %d ready", s.Nodes.Ready) for _, x := range []struct { n int label string }{ {s.Nodes.Cordoned, "cordoned"}, {s.Nodes.Draining, "draining"}, {s.Nodes.Unreachable, "unreachable"}, } { if x.n > 0 { fmt.Fprintf(out, ", %d %s", x.n, x.label) } } fmt.Fprintf(out, " (of %d)\n", s.Nodes.Total) if len(s.GPUs) > 0 { fmt.Fprintln(out, "\nGPUS") w := tabTo(out) fmt.Fprintln(w, " CLASS\tIN USE\tFREE\tUNHEALTHY\tUSAGE") for _, g := range s.GPUs { fmt.Fprintf(w, " %s\t%d/%d\t%d\t%s\t%s\n", g.Class, g.Used, g.Total, g.Free(), unhealthyCell(g.Unhealthy), bar(g.Used, g.Total)) } _ = w.Flush() } if len(s.Fabrics) > 0 { fmt.Fprintln(out, "\nFABRICS") w := tabTo(out) // The free-block column is the one that decides whether a multi-node // replica can land: free cards on the wrong fabric are not capacity for // a model that spans nodes, because a replica may never cross one. fmt.Fprintln(w, " FABRIC\tNODES\tFULLY FREE\tWEIGHTS") for _, f := range s.Fabrics { fmt.Fprintf(w, " %s\t%d\t%d\t%s\n", f.Fabric, f.Nodes, f.LargestFreeBlock, weightsCell(f)) } _ = w.Flush() } fmt.Fprintf(out, "\nREPLICAS %d ready", s.Replicas.Ready) for _, x := range []struct { n int label string }{ {s.Replicas.Starting, "starting"}, {s.Replicas.Draining, "draining"}, {s.Replicas.Failed, "failed"}, } { if x.n > 0 { fmt.Fprintf(out, ", %d %s", x.n, x.label) } } fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n", s.Models.Satisfied, s.Models.Total) // The control planes, when there are several: who leads, whether the // others would be there to take over, and the address clients use. if cps := s.ControlPlanes; len(cps.Members) > 0 { var parts []string leader := false for _, cp := range cps.Members { p := cp.Name switch cp.State { case "leading": leader = true p += " leads" default: p += " " + cp.State } if cp.This { p += " (this one)" } parts = append(parts, p) } if !leader { parts = append([]string{"NO LEADER — nothing is placed"}, parts...) } line := strings.Join(parts, ", ") if cps.Floating != "" { if cps.FloatingUp { line += " · " + cps.Floating + " on the leader" } else { line += " · " + cps.Floating + " NOT ANSWERING" } } fmt.Fprintf(out, "CONTROL %s\n", line) } // The store, when it is a replicated cluster: one line, saying who leads // and whether every other member keeps up. switch { case s.StoreError != "": fmt.Fprintf(out, "STORE cannot read its cluster: %s\n", s.StoreError) case len(s.Store) > 0: var parts []string leader := false for _, m := range s.Store { switch { case m.Role == "leader": leader = true parts = append(parts, m.Name+" leads") case m.State == "streaming" && m.Lag >= 0: parts = append(parts, fmt.Sprintf("%s streaming (%s behind)", m.Name, bytesShort(m.Lag))) default: parts = append(parts, fmt.Sprintf("%s %s", m.Name, m.State)) } } if !leader { parts = append([]string{"NO LEADER — writes fail"}, parts...) } fmt.Fprintf(out, "STORE %s\n", strings.Join(parts, ", ")) } if s.Stopped { fmt.Fprintln(out, "SERVICE STOPPED: no model runs and requests are answered 503 (llmgw start)") } if s.GuardOff { fmt.Fprintln(out, "GUARD off: no request is checked, whatever the policies say (llmgw ctl guard on)") } if len(s.Judges) > 0 { fmt.Fprintln(out, "\nGUARDRAILS' JUDGE") for _, j := range s.Judges { fmt.Fprintf(out, " %s: %s\n", j.Model, j.Problem) } } if len(s.Parked) > 0 { fmt.Fprintln(out, "\nPARKED — failed every time they started; not retried until fixed") for _, p := range s.Parked { fmt.Fprintf(out, " %s (%d failures): %s\n", p.Model, p.Failures, p.Reason) } fmt.Fprintln(out, "\n Once fixed: llmgw ctl retry ") } if len(s.Unplaceable) > 0 { fmt.Fprintln(out, "\nNOT PLACED") for _, m := range s.Unplaceable { fmt.Fprintf(out, " %s\n", m) } fmt.Fprintln(out, "\n Why, per model: llmgw ctl explain-placement ") } if len(s.IdleReplicas) > 0 { fmt.Fprintln(out, "\nIDLE") w := tabTo(out) fmt.Fprintln(w, " REPLICA\tMODEL\tGPUS\tUNUSED FOR") for _, r := range s.IdleReplicas { fmt.Fprintf(w, " %s\t%s\t%d\t%s\n", r.ID, r.Model, r.GPUs, (time.Duration(r.IdleSec) * time.Second).Round(time.Minute)) } _ = w.Flush() fmt.Fprintln(out, "\n These hold hardware nothing is asking for.") } } // weightsCell says what a fabric holds, and stays quiet when nobody asked. // // Fabrics are not connected to one another, so weights in one are not weights // in another: a model seeded into a single fabric leaves the rest of the // cluster unable to serve it without first moving tens of gigabytes over the // management network. This is the column that shows that before someone // discovers it during a scale-up. func weightsCell(f adminapi.FabricUse) string { if f.NodesWithWeights == 0 { // Nobody on this fabric was asked to look, which is the default and is // not the same as an empty fabric. return "-" } return fmt.Sprintf("%d model(s) on %d/%d nodes", f.Models, f.NodesWithWeights, f.Nodes) } // bar draws usage so a column of classes can be read at a glance rather than // compared digit by digit. func bar(used, total int) string { const width = 20 if total <= 0 { return "" } filled := used * width / total if filled > width { filled = width } b := make([]rune, width) for i := range b { if i < filled { b[i] = '#' } else { b[i] = '.' } } return fmt.Sprintf("%s %3d%%", string(b), used*100/total) } // unhealthyCell keeps a zero quiet. A column of noughts trains the eye to skip // the column, which is the opposite of what it is for. func unhealthyCell(n int) string { if n == 0 { return "-" } return fmt.Sprintf("%d", n) } // bytesShort is a byte count as a person reads it: 0 B, 12 KB, 3.4 MB. func bytesShort(n int64) string { switch { case n < 1024: return fmt.Sprintf("%d B", n) case n < 1<<20: return fmt.Sprintf("%d KB", n>>10) case n < 1<<30: return fmt.Sprintf("%.1f MB", float64(n)/(1<<20)) } return fmt.Sprintf("%.1f GB", float64(n)/(1<<30)) }