Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -32,6 +32,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -153,6 +154,10 @@ func (a *API) Register(mux Mux) {
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/engines", a.setNodeEngine)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/labels", a.setNodeLabels)
|
||||
mux.HandleFunc("GET /api/v1/facility", a.facilityReport)
|
||||
mux.HandleFunc("POST /api/v1/facility/holds", a.holdDomain)
|
||||
mux.HandleFunc("DELETE /api/v1/facility/holds/{domain}", a.releaseDomain)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
@@ -254,6 +259,18 @@ type NodeView struct {
|
||||
// answer when a node looks idle but nothing will schedule onto it.
|
||||
Unhealthy []string `json:"unhealthy_gpus,omitempty"`
|
||||
|
||||
// Location is where the node is in the building, from its llmgw.io/site,
|
||||
// room, row, rack and pdu labels; absent when it carries none.
|
||||
Location *facility.Location `json:"location,omitempty"`
|
||||
// Held names the hold keeping the node out of placement, "rack r12-07
|
||||
// is held: PDU swap"; empty when none is.
|
||||
Held string `json:"held,omitempty"`
|
||||
// HeldDomain is the held domain, level:name, for releasing it.
|
||||
HeldDomain string `json:"held_domain,omitempty"`
|
||||
// OperatorLabels are the labels set through the control plane, which
|
||||
// Labels already includes over the agent's own.
|
||||
OperatorLabels map[string]string `json:"operator_labels,omitempty"`
|
||||
|
||||
LastSeenAgo string `json:"last_seen_ago"`
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
// Features are the requests the agent answers beyond the usual:
|
||||
@@ -566,6 +583,7 @@ func (a *API) Nodes() []NodeView {
|
||||
}
|
||||
|
||||
inv := a.ctl.Inventory()
|
||||
pol := inv.Facility()
|
||||
out := make([]NodeView, 0)
|
||||
for _, n := range inv.Nodes() {
|
||||
v := NodeView{
|
||||
@@ -576,6 +594,17 @@ func (a *API) Nodes() []NodeView {
|
||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
|
||||
Replicas: replicasOn[n.ID],
|
||||
}
|
||||
if loc := n.Location(); !loc.Empty() {
|
||||
v.Location = &loc
|
||||
}
|
||||
if h, held := pol.HeldIn(n.Domains()); held {
|
||||
v.HeldDomain = string(h.Domain)
|
||||
v.Held = h.Domain.Human() + " is held"
|
||||
if h.Reason != "" {
|
||||
v.Held += ": " + h.Reason
|
||||
}
|
||||
}
|
||||
v.OperatorLabels = inv.OperatorLabels(n.ID)
|
||||
if free, total := n.LocalDisk(); total > 0 {
|
||||
v.DiskFreeGB = int(free / (1 << 30))
|
||||
v.DiskTotalGB = int(total / (1 << 30))
|
||||
|
||||
Reference in new issue
Block a user