llmgw/internal/cli/status.go

package cli

import (
	"fmt"
	"strings"
	"time"

	"github.com/spf13/cobra"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
)

// newStatusCmd prints the whole fleet in one screen.
//
// `ctl nodes` is the right tool for four nodes and the wrong one for two
// hundred. The questions that matter at that size are aggregates — how full am
// I, what is broken, what is holding cards nobody is using — and reading them
// off a long table is a chore, which means it gets skipped, which means nobody
// notices until something fails.
func newStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
	return &cobra.Command{
		Use:   "status",
		Short: "The whole fleet in one answer: capacity, health, and what is idle",
		Args:  cobra.NoArgs,
		RunE: func(cmd *cobra.Command, _ []string) error {
			var s adminapi.Summary
			if err := client().get(cmd.Context(), "/api/v1/summary", &s); err != nil {
				return err
			}
			if *asJSON {
				return emitJSON(s)
			}
			printSummary(cmd, s)
			return nil
		},
	}
}

func printSummary(cmd *cobra.Command, s adminapi.Summary) {
	out := cmd.OutOrStdout()

	fmt.Fprintf(out, "NODES     %d ready", s.Nodes.Ready)
	for _, x := range []struct {
		n     int
		label string
	}{
		{s.Nodes.Cordoned, "cordoned"},
		{s.Nodes.Draining, "draining"},
		{s.Nodes.Unreachable, "unreachable"},
	} {
		if x.n > 0 {
			fmt.Fprintf(out, ", %d %s", x.n, x.label)
		}
	}
	fmt.Fprintf(out, "  (of %d)\n", s.Nodes.Total)

	if len(s.GPUs) > 0 {
		fmt.Fprintln(out, "\nGPUS")
		w := tabTo(out)
		fmt.Fprintln(w, "  CLASS\tIN USE\tFREE\tUNHEALTHY\tUSAGE")
		for _, g := range s.GPUs {
			fmt.Fprintf(w, "  %s\t%d/%d\t%d\t%s\t%s\n",
				g.Class, g.Used, g.Total, g.Free(), unhealthyCell(g.Unhealthy), bar(g.Used, g.Total))
		}
		_ = w.Flush()
	}

	if len(s.Fabrics) > 0 {
		fmt.Fprintln(out, "\nFABRICS")
		w := tabTo(out)
		// The free-block column is the one that decides whether a multi-node
		// replica can land: free cards on the wrong fabric are not capacity for
		// a model that spans nodes, because a replica may never cross one.
		fmt.Fprintln(w, "  FABRIC\tNODES\tFULLY FREE\tWEIGHTS")
		for _, f := range s.Fabrics {
			fmt.Fprintf(w, "  %s\t%d\t%d\t%s\n",
				f.Fabric, f.Nodes, f.LargestFreeBlock, weightsCell(f))
		}
		_ = w.Flush()
	}

	fmt.Fprintf(out, "\nREPLICAS  %d ready", s.Replicas.Ready)
	for _, x := range []struct {
		n     int
		label string
	}{
		{s.Replicas.Starting, "starting"},
		{s.Replicas.Draining, "draining"},
		{s.Replicas.Failed, "failed"},
	} {
		if x.n > 0 {
			fmt.Fprintf(out, ", %d %s", x.n, x.label)
		}
	}
	fmt.Fprintf(out, "\nMODELS    %d of %d have the replicas they asked for\n",
		s.Models.Satisfied, s.Models.Total)

	// The control planes, when there are several: who leads, whether the
	// others would be there to take over, and the address clients use.
	if cps := s.ControlPlanes; len(cps.Members) > 0 {
		var parts []string
		leader := false
		for _, cp := range cps.Members {
			p := cp.Name
			switch cp.State {
			case "leading":
				leader = true
				p += " leads"
			default:
				p += " " + cp.State
			}
			if cp.This {
				p += " (this one)"
			}
			parts = append(parts, p)
		}
		if !leader {
			parts = append([]string{"NO LEADER — nothing is placed"}, parts...)
		}
		line := strings.Join(parts, ", ")
		if cps.Floating != "" {
			if cps.FloatingUp {
				line += " · " + cps.Floating + " on the leader"
			} else {
				line += " · " + cps.Floating + " NOT ANSWERING"
			}
		}
		fmt.Fprintf(out, "CONTROL   %s\n", line)
	}

	// The store, when it is a replicated cluster: one line, saying who leads
	// and whether every other member keeps up.
	switch {
	case s.StoreError != "":
		fmt.Fprintf(out, "STORE     cannot read its cluster: %s\n", s.StoreError)
	case len(s.Store) > 0:
		var parts []string
		leader := false
		for _, m := range s.Store {
			switch {
			case m.Role == "leader":
				leader = true
				parts = append(parts, m.Name+" leads")
			case m.State == "streaming" && m.Lag >= 0:
				parts = append(parts, fmt.Sprintf("%s streaming (%s behind)", m.Name, bytesShort(m.Lag)))
			default:
				parts = append(parts, fmt.Sprintf("%s %s", m.Name, m.State))
			}
		}
		if !leader {
			parts = append([]string{"NO LEADER — writes fail"}, parts...)
		}
		fmt.Fprintf(out, "STORE     %s\n", strings.Join(parts, ", "))
	}

	if s.Stopped {
		fmt.Fprintln(out, "SERVICE   STOPPED: no model runs and requests are answered 503 (llmgw start)")
	}
	if s.NoAlerts {
		fmt.Fprintln(out, "ALERTS    nowhere: a lost node or a judge down is told nobody "+
			"(llmgw ctl config set notify.webhooks, then llmgw ctl notify test)")
	}
	if s.GuardOff {
		fmt.Fprintln(out, "GUARD     off: no request is checked, whatever the policies say (llmgw ctl guard on)")
	}
	if len(s.Judges) > 0 {
		fmt.Fprintln(out, "\nGUARDRAILS' JUDGE")
		for _, j := range s.Judges {
			fmt.Fprintf(out, "  %s: %s\n", j.Model, j.Problem)
		}
	}

	if len(s.Parked) > 0 {
		fmt.Fprintln(out, "\nPARKED — failed every time they started; not retried until fixed")
		for _, p := range s.Parked {
			fmt.Fprintf(out, "  %s (%d failures): %s\n", p.Model, p.Failures, p.Reason)
		}
		fmt.Fprintln(out, "\n  Once fixed: llmgw ctl retry <model>")
	}

	if len(s.Unplaceable) > 0 {
		fmt.Fprintln(out, "\nNOT PLACED")
		for _, m := range s.Unplaceable {
			fmt.Fprintf(out, "  %s\n", m)
		}
		fmt.Fprintln(out, "\n  Why, per model: llmgw ctl explain-placement <model>")
	}

	if len(s.IdleReplicas) > 0 {
		fmt.Fprintln(out, "\nIDLE")
		w := tabTo(out)
		fmt.Fprintln(w, "  REPLICA\tMODEL\tGPUS\tUNUSED FOR")
		for _, r := range s.IdleReplicas {
			fmt.Fprintf(w, "  %s\t%s\t%d\t%s\n", r.ID, r.Model, r.GPUs,
				(time.Duration(r.IdleSec) * time.Second).Round(time.Minute))
		}
		_ = w.Flush()
		fmt.Fprintln(out, "\n  These hold hardware nothing is asking for.")
	}
}

// weightsCell says what a fabric holds, and stays quiet when nobody asked.
//
// Fabrics are not connected to one another, so weights in one are not weights
// in another: a model seeded into a single fabric leaves the rest of the
// cluster unable to serve it without first moving tens of gigabytes over the
// management network. This is the column that shows that before someone
// discovers it during a scale-up.
func weightsCell(f adminapi.FabricUse) string {
	if f.NodesWithWeights == 0 {
		// Nobody on this fabric was asked to look, which is the default and is
		// not the same as an empty fabric.
		return "-"
	}
	return fmt.Sprintf("%d model(s) on %d/%d nodes", f.Models, f.NodesWithWeights, f.Nodes)
}

// bar draws usage so a column of classes can be read at a glance rather than
// compared digit by digit.
func bar(used, total int) string {
	const width = 20
	if total <= 0 {
		return ""
	}
	filled := used * width / total
	if filled > width {
		filled = width
	}
	b := make([]rune, width)
	for i := range b {
		if i < filled {
			b[i] = '#'
		} else {
			b[i] = '.'
		}
	}
	return fmt.Sprintf("%s %3d%%", string(b), used*100/total)
}

// unhealthyCell keeps a zero quiet. A column of noughts trains the eye to skip
// the column, which is the opposite of what it is for.
func unhealthyCell(n int) string {
	if n == 0 {
		return "-"
	}
	return fmt.Sprintf("%d", n)
}

// bytesShort is a byte count as a person reads it: 0 B, 12 KB, 3.4 MB.
func bytesShort(n int64) string {
	switch {
	case n < 1024:
		return fmt.Sprintf("%d B", n)
	case n < 1<<20:
		return fmt.Sprintf("%d KB", n>>10)
	case n < 1<<30:
		return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
	}
	return fmt.Sprintf("%.1f GB", float64(n)/(1<<30))
}