Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
+28
-3
@@ -139,6 +139,8 @@ func newCtlCmd() *cobra.Command {
|
||||
newReplicasCmd(client, &asJSON),
|
||||
newExplainCmd(client, &asJSON),
|
||||
newFabricTestCmd(client, &asJSON),
|
||||
newFacilityCmd(client, &asJSON),
|
||||
newLabelCmd(client, &asJSON),
|
||||
newNodeStateCmd(client, "cordon", "cordoned",
|
||||
"Stop placing new work on a node; what runs there keeps running"),
|
||||
newNodeStateCmd(client, "uncordon", "ready",
|
||||
@@ -205,10 +207,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
}
|
||||
|
||||
w := tab()
|
||||
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||
fmt.Fprintln(w, "NODE\tSTATE\tLOCATION\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
|
||||
for _, n := range nodes {
|
||||
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
|
||||
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
|
||||
n.ID, stateColumn(n), locationColumn(n), n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
|
||||
diskColumn(n.DiskFreeGB, n.DiskTotalGB),
|
||||
// What the node can run, which is the first thing to look
|
||||
// at when a model will not place and the hardware looks
|
||||
@@ -239,6 +241,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Printf("\n%s: %s", n.ID, w)
|
||||
said = true
|
||||
}
|
||||
if n.Held != "" {
|
||||
fmt.Printf("\n%s: %s (kept out of placement)", n.ID, n.Held)
|
||||
said = true
|
||||
}
|
||||
}
|
||||
if said {
|
||||
fmt.Println()
|
||||
@@ -251,9 +257,28 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c.Flags().StringVar(&f.state, "state", "", "only nodes in this state (prefix match)")
|
||||
c.Flags().StringArrayVar(&f.labels, "label", nil,
|
||||
"only nodes carrying key=value; repeatable, and key alone matches any value")
|
||||
c.Flags().StringVar(&f.domain, "domain", "", "only nodes in this rack, row, room, site or PDU, as level:name (rack:r12-07)")
|
||||
return c
|
||||
}
|
||||
|
||||
// stateColumn is a node's state, with "held" when a held domain keeps it out
|
||||
// of placement while its own state says ready.
|
||||
func stateColumn(n adminapi.NodeView) string {
|
||||
if n.Held != "" && n.State == "ready" {
|
||||
return "ready, held"
|
||||
}
|
||||
return n.State
|
||||
}
|
||||
|
||||
// locationColumn is where a node is, in a few characters; a dash when it
|
||||
// carries no location label.
|
||||
func locationColumn(n adminapi.NodeView) string {
|
||||
if n.Location == nil || n.Location.Empty() {
|
||||
return "-"
|
||||
}
|
||||
return n.Location.Short()
|
||||
}
|
||||
|
||||
// modelStatus says what is happening to a model, in the words that decide what
|
||||
// to do about it.
|
||||
//
|
||||
|
||||
Reference in new issue
Block a user