Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
+28
-3
@@ -139,6 +139,8 @@ func newCtlCmd() *cobra.Command {
|
||||
newReplicasCmd(client, &asJSON),
|
||||
newExplainCmd(client, &asJSON),
|
||||
newFabricTestCmd(client, &asJSON),
|
||||
newFacilityCmd(client, &asJSON),
|
||||
newLabelCmd(client, &asJSON),
|
||||
newNodeStateCmd(client, "cordon", "cordoned",
|
||||
"Stop placing new work on a node; what runs there keeps running"),
|
||||
newNodeStateCmd(client, "uncordon", "ready",
|
||||
@@ -205,10 +207,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||
fmt.Fprintln(w, "NODE\tSTATE\tLOCATION\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||
for _, n := range nodes {
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
|
||||
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
|
||||
n.ID, stateColumn(n), locationColumn(n), n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
|
||||
diskColumn(n.DiskFreeGB, n.DiskTotalGB),
|
||||
// What the node can run, which is the first thing to look
|
||||
// at when a model will not place and the hardware looks
|
||||
@@ -239,6 +241,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Printf("\n%s: %s", n.ID, w)
|
||||
said = true
|
||||
}
|
||||
if n.Held != "" {
|
||||
fmt.Printf("\n%s: %s (kept out of placement)", n.ID, n.Held)
|
||||
said = true
|
||||
}
|
||||
}
|
||||
if said {
|
||||
fmt.Println()
|
||||
@@ -251,9 +257,28 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c.Flags().StringVar(&f.state, "state", "", "only nodes in this state (prefix match)")
|
||||
c.Flags().StringArrayVar(&f.labels, "label", nil,
|
||||
"only nodes carrying key=value; repeatable, and key alone matches any value")
|
||||
c.Flags().StringVar(&f.domain, "domain", "", "only nodes in this rack, row, room, site or PDU, as level:name (rack:r12-07)")
|
||||
return c
|
||||
}
|
||||
|
||||
// stateColumn is a node's state, with "held" when a held domain keeps it out
|
||||
// of placement while its own state says ready.
|
||||
func stateColumn(n adminapi.NodeView) string {
|
||||
if n.Held != "" && n.State == "ready" {
|
||||
return "ready, held"
|
||||
}
|
||||
return n.State
|
||||
}
|
||||
|
||||
// locationColumn is where a node is, in a few characters; a dash when it
|
||||
// carries no location label.
|
||||
func locationColumn(n adminapi.NodeView) string {
|
||||
if n.Location == nil || n.Location.Empty() {
|
||||
return "-"
|
||||
}
|
||||
return n.Location.Short()
|
||||
}
|
||||
|
||||
// modelStatus says what is happening to a model, in the words that decide what
|
||||
// to do about it.
|
||||
//
|
||||
|
||||
@@ -0,0 +1,212 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
)
|
||||
|
||||
// newFacilityCmd shows the failure domains and holds them for maintenance.
|
||||
func newFacilityCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
cmd := &cobra.Command{
|
||||
Use: "facility",
|
||||
Short: "Racks, PDUs and rooms: their power against their caps, and what is held",
|
||||
Long: `Where nodes are comes from their labels: llmgw.io/site, room, row, rack
|
||||
and pdu, set in agent.yaml or with 'llmgw ctl label'. Placement spreads a
|
||||
model's replicas across racks and PDUs, places nothing in a held domain, and
|
||||
refuses a replica that would take a domain past its power cap (facility.budgets
|
||||
in server.yaml).
|
||||
|
||||
USED is what counts against the cap: node by node, the larger of what the
|
||||
replicas placed there are estimated to draw and what the node's sensors read.`,
|
||||
Args: cobra.NoArgs,
|
||||
RunE: func(cmd *cobra.Command, _ []string) error {
|
||||
var v adminapi.FacilityView
|
||||
if err := client().get(cmd.Context(), "/api/v1/facility", &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(v)
|
||||
}
|
||||
printFacility(cmd.OutOrStdout(), v)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
cmd.AddCommand(newFacilityHoldCmd(client), newFacilityReleaseCmd(client))
|
||||
return cmd
|
||||
}
|
||||
|
||||
// printFacility is the facility view as a table, then what is held, then
|
||||
// the nodes nobody has said the place of.
|
||||
func printFacility(out io.Writer, v adminapi.FacilityView) {
|
||||
if len(v.Domains) == 0 {
|
||||
fmt.Fprintln(out, "No node says where it is. Label them with llmgw.io/rack (and site, room, row, pdu):\n"+
|
||||
" llmgw ctl label gpu01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b")
|
||||
return
|
||||
}
|
||||
w := tabTo(out)
|
||||
fmt.Fprintln(w, "DOMAIN\tNODES\tUSED\tLIMIT\tRESERVED\tMEASURED\tHELD")
|
||||
for _, d := range v.Domains {
|
||||
limit, used := "-", facility.KW(d.UsedWatts)
|
||||
if d.LimitWatts > 0 {
|
||||
limit = facility.KW(d.LimitWatts)
|
||||
used += fmt.Sprintf(" (%.0f%%)", 100*d.UsedWatts/d.LimitWatts)
|
||||
}
|
||||
held := "-"
|
||||
if d.Held != nil {
|
||||
held = "held"
|
||||
if d.Held.Reason != "" {
|
||||
held += ": " + d.Held.Reason
|
||||
}
|
||||
if d.Held.Drain {
|
||||
held += " (drained)"
|
||||
}
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%d\t%s\t%s\t%s\t%s\t%s\n", d.Domain, len(d.Nodes), used, limit,
|
||||
facility.KW(d.ReservedWatts), facility.KW(d.MeasuredWatts), held)
|
||||
}
|
||||
_ = w.Flush()
|
||||
for _, h := range v.Holds {
|
||||
fmt.Fprintf(out, "\n%s held by %s since %s", h.Domain, or(h.By, "someone"), h.At.Local().Format("2006-01-02 15:04"))
|
||||
}
|
||||
if len(v.Holds) > 0 {
|
||||
fmt.Fprintln(out)
|
||||
}
|
||||
if len(v.Unlocated) > 0 {
|
||||
fmt.Fprintf(out, "\nNo location: %s. Each is placed as a rack of its own.\n", strings.Join(v.Unlocated, ", "))
|
||||
}
|
||||
}
|
||||
|
||||
func newFacilityHoldCmd(client func() *ctlClient) *cobra.Command {
|
||||
var req adminapi.HoldRequest
|
||||
c := &cobra.Command{
|
||||
Use: "hold <level:name>",
|
||||
Short: "Place nothing new in a rack, row, PDU or room; with --drain, move its work off",
|
||||
Long: `For maintenance, a cooling fault, work on a PDU: like cordoning every node in
|
||||
the domain, including nodes labelled into it later, and kept across restarts
|
||||
when the control plane has a store. What runs there keeps running unless
|
||||
--drain is given; then each of its nodes is drained, one replica at a time
|
||||
with the replacement up elsewhere first.`,
|
||||
Example: ` llmgw ctl facility hold rack:r12-07 --reason "PDU A replacement"
|
||||
llmgw ctl facility hold pdu:r12-07-a --drain --reason "breaker work"`,
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req.Domain = args[0]
|
||||
if _, err := facility.ParseDomain(req.Domain); err != nil {
|
||||
return err
|
||||
}
|
||||
var v adminapi.HoldView
|
||||
if err := client().do(cmd.Context(), http.MethodPost, "/api/v1/facility/holds", req, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is held: nothing new is placed in it.\n", v.Domain)
|
||||
if len(v.Nodes) > 0 {
|
||||
fmt.Printf("Draining %s; their replicas move one at a time.\n", strings.Join(v.Nodes, ", "))
|
||||
}
|
||||
fmt.Printf("Release it with: llmgw ctl facility release %s\n", v.Domain)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
c.Flags().StringVar(&req.Reason, "reason", "", "why, as explain-placement and the node list will say")
|
||||
c.Flags().BoolVar(&req.Drain, "drain", false, "also drain the domain's nodes")
|
||||
return c
|
||||
}
|
||||
|
||||
func newFacilityReleaseCmd(client func() *ctlClient) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "release <level:name>",
|
||||
Short: "Allow placement in a held domain again",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
d, err := facility.ParseDomain(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var v adminapi.HoldView
|
||||
if err := client().delete(cmd.Context(), "/api/v1/facility/holds/"+url.PathEscape(string(d)), &v); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s is released.\n", v.Domain)
|
||||
if len(v.Nodes) > 0 {
|
||||
fmt.Printf("Ready again: %s.\n", strings.Join(v.Nodes, ", "))
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// newLabelCmd sets labels on a node through the control plane.
|
||||
func newLabelCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
return &cobra.Command{
|
||||
Use: "label <node> key=value... [key-...]",
|
||||
Short: "Set or remove a node's labels, over those in its agent.yaml",
|
||||
Long: `Labels set here are kept by the control plane and win over the agent's own;
|
||||
key- removes one, which puts the agent's value for the key back if it has
|
||||
one. The node need not have joined yet.
|
||||
|
||||
The llmgw.io/ labels say where a node is, for facility-aware placement:
|
||||
site, room, row, rack, and pdu (a comma separated list for a dual-fed rack).
|
||||
Any other key is free for a profile's node_selector.`,
|
||||
Example: ` llmgw ctl label gpu01 llmgw.io/rack=r12-07 llmgw.io/pdu=r12-07-a,r12-07-b
|
||||
llmgw ctl label gpu01 llmgw.io/row=12 llmgw.io/room=hall-b
|
||||
llmgw ctl label gpu01 llmgw.io/rack-`,
|
||||
Args: cobra.MinimumNArgs(2),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
req, err := parseLabelArgs(args[1:])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var v adminapi.NodeLabelsView
|
||||
if err := client().do(cmd.Context(), http.MethodPost,
|
||||
"/api/v1/nodes/"+url.PathEscape(args[0])+"/labels", req, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
if *asJSON {
|
||||
return emitJSON(v)
|
||||
}
|
||||
if v.Labels == nil {
|
||||
fmt.Printf("%s has not joined yet; it takes these labels when it does.\n", v.ID)
|
||||
}
|
||||
keys := slices.Sorted(maps.Keys(v.OperatorLabels))
|
||||
if len(keys) == 0 {
|
||||
fmt.Printf("%s carries no labels set here, only its agent's.\n", v.ID)
|
||||
}
|
||||
for _, k := range keys {
|
||||
fmt.Printf("%s: %s=%s\n", v.ID, k, v.OperatorLabels[k])
|
||||
}
|
||||
if v.Location != nil {
|
||||
fmt.Printf("%s is at %s.\n", v.ID, v.Location.Short())
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// parseLabelArgs reads key=value to set and key- to remove.
|
||||
func parseLabelArgs(args []string) (adminapi.NodeLabelsRequest, error) {
|
||||
req := adminapi.NodeLabelsRequest{Set: map[string]string{}}
|
||||
for _, a := range args {
|
||||
if k, v, ok := strings.Cut(a, "="); ok {
|
||||
if k == "" || v == "" {
|
||||
return req, fmt.Errorf("%q: give key=value, or key- to remove a label", a)
|
||||
}
|
||||
req.Set[k] = v
|
||||
continue
|
||||
}
|
||||
if k, ok := strings.CutSuffix(a, "-"); ok && k != "" {
|
||||
req.Remove = append(req.Remove, k)
|
||||
continue
|
||||
}
|
||||
return req, fmt.Errorf("%q: give key=value, or key- to remove a label", a)
|
||||
}
|
||||
return req, nil
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
)
|
||||
|
||||
func TestLabelArgumentsSetWithEqualsAndRemoveWithADash(t *testing.T) {
|
||||
req, err := parseLabelArgs([]string{"llmgw.io/rack=r1", "llmgw.io/pdu=p1,p2", "role-"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.Set["llmgw.io/rack"] != "r1" || req.Set["llmgw.io/pdu"] != "p1,p2" || !slices.Equal(req.Remove, []string{"role"}) {
|
||||
t.Errorf("got %+v", req)
|
||||
}
|
||||
for _, bad := range []string{"rack", "=r1", "rack=", "-"} {
|
||||
if _, err := parseLabelArgs([]string{bad}); err == nil {
|
||||
t.Errorf("%q was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesAreFilteredByDomain(t *testing.T) {
|
||||
nodes := fleet()
|
||||
nodes[0].Location = &facility.Location{Rack: "r1", PDUs: []string{"p1"}}
|
||||
nodes[1].Location = &facility.Location{Rack: "r2", PDUs: []string{"p1"}}
|
||||
got, err := nodeFilter{domain: "pdu:p1"}.apply(nodes)
|
||||
if err != nil || ids(got) != "n1,n2" {
|
||||
t.Errorf("--domain pdu:p1 = %s, %v", ids(got), err)
|
||||
}
|
||||
if got, _ := (nodeFilter{domain: "rack:r2"}).apply(nodes); ids(got) != "n2" {
|
||||
t.Errorf("--domain rack:r2 = %s", ids(got))
|
||||
}
|
||||
if _, err := (nodeFilter{domain: "r2"}).apply(nodes); err == nil {
|
||||
t.Error("a domain with no level was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLocationFormSetsWhatChangedAndRemovesWhatWasEmptied(t *testing.T) {
|
||||
n := adminapi.NodeView{
|
||||
Labels: map[string]string{"llmgw.io/rack": "r1", "llmgw.io/row": "12"},
|
||||
OperatorLabels: map[string]string{"llmgw.io/row": "12"},
|
||||
}
|
||||
// site, room, row, rack, pdu
|
||||
req := locationRequest(n, []string{"", "hall-b", "", "r1", "p1"})
|
||||
if len(req.Set) != 2 || req.Set["llmgw.io/room"] != "hall-b" || req.Set["llmgw.io/pdu"] != "p1" {
|
||||
t.Errorf("set %v: the unchanged rack is left alone", req.Set)
|
||||
}
|
||||
if !slices.Equal(req.Remove, []string{"llmgw.io/row"}) {
|
||||
t.Errorf("remove %v, want the row the operator had set", req.Remove)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFacilityTableShowsUseAgainstTheLimitAndWhatIsHeld(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
printFacility(&b, adminapi.FacilityView{
|
||||
Domains: []adminapi.DomainView{
|
||||
{Domain: "rack:r1", Nodes: []string{"a"}, LimitWatts: 10000, UsedWatts: 7500, ReservedWatts: 7000, MeasuredWatts: 7500,
|
||||
Held: &adminapi.HoldView{Domain: "rack:r1", Reason: "PDU swap"}},
|
||||
},
|
||||
Holds: []adminapi.HoldView{{Domain: "rack:r1", By: "ilker"}},
|
||||
Unlocated: []string{"b"},
|
||||
})
|
||||
out := b.String()
|
||||
for _, want := range []string{"7.5 kW (75%)", "10.0 kW", "held: PDU swap", "rack:r1 held by ilker", "No location: b"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
+19
-1
@@ -6,6 +6,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
)
|
||||
|
||||
// nodeFilter narrows a node listing.
|
||||
@@ -20,6 +21,8 @@ type nodeFilter struct {
|
||||
class string
|
||||
state string
|
||||
labels []string
|
||||
// domain is level:name, rack:r12-07.
|
||||
domain string
|
||||
}
|
||||
|
||||
// empty reports whether this filter would narrow anything.
|
||||
@@ -28,7 +31,7 @@ type nodeFilter struct {
|
||||
// different situations with different next steps, and "no nodes" for both
|
||||
// makes a mistyped fabric look like an outage.
|
||||
func (f nodeFilter) empty() bool {
|
||||
return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0
|
||||
return f.fabric == "" && f.class == "" && f.state == "" && len(f.labels) == 0 && f.domain == ""
|
||||
}
|
||||
|
||||
func (f nodeFilter) describe() string {
|
||||
@@ -45,6 +48,9 @@ func (f nodeFilter) describe() string {
|
||||
for _, l := range f.labels {
|
||||
parts = append(parts, "label "+l)
|
||||
}
|
||||
if f.domain != "" {
|
||||
parts = append(parts, "domain "+f.domain)
|
||||
}
|
||||
return strings.Join(parts, ", ")
|
||||
}
|
||||
|
||||
@@ -62,11 +68,23 @@ func (f nodeFilter) apply(in []adminapi.NodeView) ([]adminapi.NodeView, error) {
|
||||
want[k] = v
|
||||
}
|
||||
|
||||
var domain facility.Domain
|
||||
if f.domain != "" {
|
||||
d, err := facility.ParseDomain(f.domain)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("--domain: %w", err)
|
||||
}
|
||||
domain = d
|
||||
}
|
||||
|
||||
out := make([]adminapi.NodeView, 0, len(in))
|
||||
for _, n := range in {
|
||||
if f.fabric != "" && !slices.Contains(n.Fabrics, f.fabric) {
|
||||
continue
|
||||
}
|
||||
if domain != "" && (n.Location == nil || !slices.Contains(n.Location.Domains(), domain)) {
|
||||
continue
|
||||
}
|
||||
if f.class != "" && !slices.Contains(n.Classes, f.class) {
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -654,6 +654,10 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
||||
if u, ok := sink.(control.UsageSource); ok {
|
||||
ctl.SetUsageSource(u)
|
||||
}
|
||||
// Power caps and estimates; Validate has already refused a bad one.
|
||||
if pol, err := cfg.Facility.Policy(); err == nil {
|
||||
ctl.Facility().SetPolicy(pol)
|
||||
}
|
||||
// With a shared store, two control planes can see each other, so this one
|
||||
// competes rather than assuming it is alone. Without one there is nobody
|
||||
// to compete with and it places unconditionally, exactly as before.
|
||||
@@ -676,6 +680,8 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
||||
ctl.Reconciler().SetMeasurements(control.Measurements(st, log))
|
||||
// The engine builds chosen per node on the cluster page.
|
||||
ctl.SetEngineChoices(control.NewEngineChoices(ctx, st, log))
|
||||
// Domains held for maintenance and labels set on nodes.
|
||||
ctl.Facility().UseStore(ctx, st)
|
||||
log.Info("competing to be the control plane that places work",
|
||||
"owner", lead.Owner())
|
||||
}
|
||||
|
||||
@@ -17,6 +17,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/version"
|
||||
@@ -464,6 +465,7 @@ func (m consoleModel) nodeMenu(n adminapi.NodeView) *menuDialog {
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
items = append(items, m.locationItems(n)...)
|
||||
if n.GPUTotal > 0 {
|
||||
for _, engine := range []string{"vllm", "sglang"} {
|
||||
items = append(items, menuItem{engine + " build…", func(m *consoleModel) tea.Cmd {
|
||||
@@ -825,3 +827,72 @@ func (m consoleModel) updateItems(md adminapi.ModelView) []menuItem {
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// locationItems are the node menu's facility entries: where the node is,
|
||||
// and holding or releasing its rack.
|
||||
func (m consoleModel) locationItems(n adminapi.NodeView) []menuItem {
|
||||
path := "/api/v1/nodes/" + url.PathEscape(n.ID) + "/labels"
|
||||
var loc facility.Location
|
||||
if n.Location != nil {
|
||||
loc = *n.Location
|
||||
}
|
||||
items := []menuItem{{"Location…", func(m *consoleModel) tea.Cmd {
|
||||
m.openForm(&formDialog{title: "Where " + n.ID + " is",
|
||||
note: "Placement spreads a model's replicas across racks and PDUs, and keeps out of held ones. " +
|
||||
"Empty leaves a level unknown; a PDU list is comma separated, for a dual-fed rack.",
|
||||
fields: []formField{
|
||||
textF("site", loc.Site, "sto1"), textF("room", loc.Room, "hall-b"),
|
||||
textF("row", loc.Row, "12"), textF("rack", loc.Rack, "r12-07"),
|
||||
textF("pdu", strings.Join(loc.PDUs, ","), "r12-07-a,r12-07-b"),
|
||||
},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
req := locationRequest(n, v)
|
||||
if len(req.Set) == 0 && len(req.Remove) == 0 {
|
||||
return nil, fmt.Errorf("nothing changed")
|
||||
}
|
||||
return m.call("Location of "+n.ID, http.MethodPost, path, req,
|
||||
func(json.RawMessage) string { return n.ID + "'s location is set." }), nil
|
||||
}})
|
||||
return nil
|
||||
}}}
|
||||
switch {
|
||||
case n.HeldDomain != "":
|
||||
d := n.HeldDomain
|
||||
items = append(items, menuItem{"Release " + d, func(m *consoleModel) tea.Cmd {
|
||||
return m.call("Release "+d, http.MethodDelete, "/api/v1/facility/holds/"+url.PathEscape(d), nil,
|
||||
func(json.RawMessage) string { return d + " is released." })
|
||||
}})
|
||||
case loc.Rack != "":
|
||||
d := string(facility.Of(facility.Rack, loc.Rack))
|
||||
items = append(items, menuItem{"Hold rack " + loc.Rack + "…", func(m *consoleModel) tea.Cmd {
|
||||
m.openForm(&formDialog{title: "Hold rack " + loc.Rack,
|
||||
note: "Nothing new is placed in the rack. Drain also moves its replicas off, one at a time.",
|
||||
fields: []formField{textF("reason", "", "PDU A replacement"),
|
||||
choiceF("drain", []string{"no", "yes"}, "no")},
|
||||
submit: func(m *consoleModel, v []string) (tea.Cmd, error) {
|
||||
return m.call("Hold "+d, http.MethodPost, "/api/v1/facility/holds",
|
||||
adminapi.HoldRequest{Domain: d, Reason: v[0], Drain: v[1] == "yes"},
|
||||
func(json.RawMessage) string { return d + " is held." }), nil
|
||||
}})
|
||||
return nil
|
||||
}})
|
||||
}
|
||||
return items
|
||||
}
|
||||
|
||||
// locationRequest turns the location form's values, site to pdu, into the
|
||||
// labels to set and the ones to remove: a level emptied that was set through
|
||||
// the control plane is removed, which leaves the agent's own value if any.
|
||||
func locationRequest(n adminapi.NodeView, vals []string) adminapi.NodeLabelsRequest {
|
||||
req := adminapi.NodeLabelsRequest{Set: map[string]string{}}
|
||||
for i, lv := range facility.Levels {
|
||||
k, v := lv.Label(), strings.TrimSpace(vals[i])
|
||||
switch {
|
||||
case v != "" && n.Labels[k] != v:
|
||||
req.Set[k] = v
|
||||
case v == "" && n.OperatorLabels[k] != "":
|
||||
req.Remove = append(req.Remove, k)
|
||||
}
|
||||
}
|
||||
return req
|
||||
}
|
||||
@@ -1041,11 +1041,11 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
if n.Load != nil {
|
||||
cpu, mem = fmt.Sprintf("%.0f%%", n.Load.CPUPercent), fmt.Sprintf("%.0f%%", n.Load.MemPercent)
|
||||
}
|
||||
state := n.State
|
||||
state := stateColumn(n)
|
||||
if state != "ready" {
|
||||
state = sWarn.Render(state)
|
||||
}
|
||||
rows = append(rows, []string{n.ID, state, cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal),
|
||||
rows = append(rows, []string{n.ID, state, locationColumn(n), cpu, mem, fmt.Sprintf("%d/%d", n.GPUFree, n.GPUTotal),
|
||||
strconv.Itoa(n.Replicas), fmt.Sprintf("%d/%d GB", n.DiskFreeGB, n.DiskTotalGB), n.AgentVersion, n.LastSeenAgo})
|
||||
}
|
||||
t := "Nodes"
|
||||
@@ -1055,7 +1055,7 @@ func (m consoleModel) panel() (title, body, keys string) {
|
||||
if m.inPanel {
|
||||
keys = rowKeys
|
||||
}
|
||||
return t, "\n" + table([]string{"NODE", "STATE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"},
|
||||
return t, "\n" + table([]string{"NODE", "STATE", "WHERE", "CPU", "MEM", "GPU FREE", "REPLICAS", "DISK FREE", "AGENT", "SEEN"},
|
||||
rows, m.row, pw, m.inPanel), keys
|
||||
case cGPUs:
|
||||
if m.inPanel {
|
||||
@@ -1299,6 +1299,18 @@ func (m consoleModel) nodeDetail(n adminapi.NodeView) string {
|
||||
kv := func(k, v string) { fmt.Fprintf(&b, " %-14s %s\n", sDim.Render(k), v) }
|
||||
b.WriteString("\n")
|
||||
kv("state", n.State)
|
||||
if n.Location != nil {
|
||||
where := []string{}
|
||||
for _, d := range n.Location.Domains() {
|
||||
where = append(where, d.Human())
|
||||
}
|
||||
kv("location", strings.Join(where, " · "))
|
||||
} else {
|
||||
kv("location", sDim.Render("not labelled: Enter, then Location…"))
|
||||
}
|
||||
if n.Held != "" {
|
||||
kv("held", sWarn.Render(n.Held))
|
||||
}
|
||||
kv("address", n.MgmtAddr)
|
||||
kv("system", n.OS)
|
||||
kv("agent", n.AgentVersion+", seen "+n.LastSeenAgo+" ago")
|
||||
|
||||
Reference in new issue
Block a user