Facility-aware placement: racks, PDUs, holds and power caps

Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels,
from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's).
Placement spreads a model's replicas across racks and PDUs, keeps a
multi-node replica within one domain when its profile says keep_within,
places nothing in a domain held with `llmgw ctl facility hold` (optionally
draining it), and refuses a shard that would take a domain past its power
cap from facility.budgets, counting declared, derived and measured draw.
Explain-placement names the domain, and keeps the rejections of a placed
replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the
domains; the console's Nodes view shows and sets location and holds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:17:49 +02:00
1 parent d41104a452
commit 0d3eb82b9b
42 files changed
+3510 -63

No files matched your search

+28 -3
View File
@@ -139,6 +139,8 @@ func newCtlCmd() *cobra.Command {
newReplicasCmd(client, &asJSON),
newExplainCmd(client, &asJSON),
newFabricTestCmd(client, &asJSON),
newFacilityCmd(client, &asJSON),
newLabelCmd(client, &asJSON),
newNodeStateCmd(client, "cordon", "cordoned",
"Stop placing new work on a node; what runs there keeps running"),
newNodeStateCmd(client, "uncordon", "ready",
@@ -205,10 +207,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
}
w := tab()
fmt.Fprintln(w, "NODE\tSTATE\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
fmt.Fprintln(w, "NODE\tSTATE\tLOCATION\tGPUS\tMEM\tDISK\tENGINES\tCLASSES\tFABRICS\tREPLICAS\tLAST SEEN")
for _, n := range nodes {
fmt.Fprintf(w, "%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
n.ID, n.State, n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
fmt.Fprintf(w, "%s\t%s\t%s\t%d/%d free\t%s\t%s\t%s\t%s\t%s\t%d\t%s\n",
n.ID, stateColumn(n), locationColumn(n), n.GPUFree, n.GPUTotal, memColumn(n.MemoryGB),
diskColumn(n.DiskFreeGB, n.DiskTotalGB),
// What the node can run, which is the first thing to look
// at when a model will not place and the hardware looks
@@ -239,6 +241,10 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
fmt.Printf("\n%s: %s", n.ID, w)
said = true
}
if n.Held != "" {
fmt.Printf("\n%s: %s (kept out of placement)", n.ID, n.Held)
said = true
}
}
if said {
fmt.Println()
@@ -251,9 +257,28 @@ func newNodesCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
c.Flags().StringVar(&f.state, "state", "", "only nodes in this state (prefix match)")
c.Flags().StringArrayVar(&f.labels, "label", nil,
"only nodes carrying key=value; repeatable, and key alone matches any value")
c.Flags().StringVar(&f.domain, "domain", "", "only nodes in this rack, row, room, site or PDU, as level:name (rack:r12-07)")
return c
}
// stateColumn is a node's state, with "held" when a held domain keeps it out
// of placement while its own state says ready.
func stateColumn(n adminapi.NodeView) string {
if n.Held != "" && n.State == "ready" {
return "ready, held"
}
return n.State
}
// locationColumn is where a node is, in a few characters; a dash when it
// carries no location label.
func locationColumn(n adminapi.NodeView) string {
if n.Location == nil || n.Location.Empty() {
return "-"
}
return n.Location.Short()
}
// modelStatus says what is happening to a model, in the words that decide what
// to do about it.
//