Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -32,6 +32,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
@@ -153,6 +154,10 @@ func (a *API) Register(mux Mux) {
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/engines", a.setNodeEngine)
|
||||
mux.HandleFunc("POST /api/v1/nodes/{id}/labels", a.setNodeLabels)
|
||||
mux.HandleFunc("GET /api/v1/facility", a.facilityReport)
|
||||
mux.HandleFunc("POST /api/v1/facility/holds", a.holdDomain)
|
||||
mux.HandleFunc("DELETE /api/v1/facility/holds/{domain}", a.releaseDomain)
|
||||
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
|
||||
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
|
||||
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
|
||||
@@ -254,6 +259,18 @@ type NodeView struct {
|
||||
// answer when a node looks idle but nothing will schedule onto it.
|
||||
Unhealthy []string `json:"unhealthy_gpus,omitempty"`
|
||||
|
||||
// Location is where the node is in the building, from its llmgw.io/site,
|
||||
// room, row, rack and pdu labels; absent when it carries none.
|
||||
Location *facility.Location `json:"location,omitempty"`
|
||||
// Held names the hold keeping the node out of placement, "rack r12-07
|
||||
// is held: PDU swap"; empty when none is.
|
||||
Held string `json:"held,omitempty"`
|
||||
// HeldDomain is the held domain, level:name, for releasing it.
|
||||
HeldDomain string `json:"held_domain,omitempty"`
|
||||
// OperatorLabels are the labels set through the control plane, which
|
||||
// Labels already includes over the agent's own.
|
||||
OperatorLabels map[string]string `json:"operator_labels,omitempty"`
|
||||
|
||||
LastSeenAgo string `json:"last_seen_ago"`
|
||||
AgentVersion string `json:"agent_version,omitempty"`
|
||||
// Features are the requests the agent answers beyond the usual:
|
||||
@@ -566,6 +583,7 @@ func (a *API) Nodes() []NodeView {
|
||||
}
|
||||
|
||||
inv := a.ctl.Inventory()
|
||||
pol := inv.Facility()
|
||||
out := make([]NodeView, 0)
|
||||
for _, n := range inv.Nodes() {
|
||||
v := NodeView{
|
||||
@@ -576,6 +594,17 @@ func (a *API) Nodes() []NodeView {
|
||||
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
|
||||
Replicas: replicasOn[n.ID],
|
||||
}
|
||||
if loc := n.Location(); !loc.Empty() {
|
||||
v.Location = &loc
|
||||
}
|
||||
if h, held := pol.HeldIn(n.Domains()); held {
|
||||
v.HeldDomain = string(h.Domain)
|
||||
v.Held = h.Domain.Human() + " is held"
|
||||
if h.Reason != "" {
|
||||
v.Held += ": " + h.Reason
|
||||
}
|
||||
}
|
||||
v.OperatorLabels = inv.OperatorLabels(n.ID)
|
||||
if free, total := n.LocalDisk(); total > 0 {
|
||||
v.DiskFreeGB = int(free / (1 << 30))
|
||||
v.DiskTotalGB = int(total / (1 << 30))
|
||||
|
||||
@@ -0,0 +1,203 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// FacilityView is the failure domains as placement sees them.
|
||||
type FacilityView struct {
|
||||
// Domains are every domain a node is in or a budget names, by level
|
||||
// (site, room, row, rack, pdu) and then name.
|
||||
Domains []DomainView `json:"domains"`
|
||||
// Holds are every domain held, including one no node is in yet.
|
||||
Holds []HoldView `json:"holds,omitempty"`
|
||||
// Margin is the share of every cap kept free.
|
||||
Margin float64 `json:"margin,omitempty"`
|
||||
// Unlocated are the nodes with no location label, which placement
|
||||
// treats as each in a rack of its own.
|
||||
Unlocated []string `json:"unlocated,omitempty"`
|
||||
}
|
||||
|
||||
// DomainView is one domain: its nodes, its power and whether it is held.
|
||||
type DomainView struct {
|
||||
Domain string `json:"domain"`
|
||||
Level string `json:"level"`
|
||||
Name string `json:"name"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
// CapWatts is the domain's cap, LimitWatts the cap less the margin;
|
||||
// both zero when it has none.
|
||||
CapWatts float64 `json:"cap_watts,omitempty"`
|
||||
LimitWatts float64 `json:"limit_watts,omitempty"`
|
||||
OtherWatts float64 `json:"other_watts,omitempty"`
|
||||
// ReservedWatts is what is placed in it is estimated to draw;
|
||||
// MeasuredWatts what its nodes' sensors read; UsedWatts what counts
|
||||
// against the cap, the larger of the two node by node.
|
||||
ReservedWatts float64 `json:"reserved_watts"`
|
||||
MeasuredWatts float64 `json:"measured_watts"`
|
||||
UsedWatts float64 `json:"used_watts"`
|
||||
// Held is set when the domain is held.
|
||||
Held *HoldView `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// HoldRequest holds a domain.
|
||||
type HoldRequest struct {
|
||||
// Domain is level:name: rack:r12-07, pdu:r12-07-a, row:12.
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
// Drain moves the work in the domain elsewhere too.
|
||||
Drain bool `json:"drain,omitempty"`
|
||||
}
|
||||
|
||||
// HoldView is a hold, and the nodes holding or releasing it changed.
|
||||
type HoldView struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Drain bool `json:"drain,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Nodes are those this call drained, or made ready again on release.
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
}
|
||||
|
||||
// NodeLabelsRequest sets and removes labels on a node.
|
||||
type NodeLabelsRequest struct {
|
||||
Set map[string]string `json:"set,omitempty"`
|
||||
// Remove are keys whose label set through the control plane goes; the
|
||||
// agent's own value for the key, if it has one, is in force again.
|
||||
Remove []string `json:"remove,omitempty"`
|
||||
}
|
||||
|
||||
// NodeLabelsView is a node's labels after a change.
|
||||
type NodeLabelsView struct {
|
||||
ID string `json:"id"`
|
||||
// OperatorLabels are those set through the control plane.
|
||||
OperatorLabels map[string]string `json:"operator_labels,omitempty"`
|
||||
// Labels are what placement matches: the agent's with the operator's
|
||||
// over them. Absent for a node that has not registered yet.
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
Location *facility.Location `json:"location,omitempty"`
|
||||
}
|
||||
|
||||
func holdView(h facility.Hold, nodes []string) HoldView {
|
||||
return HoldView{Domain: string(h.Domain), Reason: h.Reason, Drain: h.Drain, By: h.By, At: h.At, Nodes: nodes}
|
||||
}
|
||||
|
||||
// Facility builds the facility view; exported for the console's sake, as
|
||||
// Nodes is.
|
||||
func (a *API) Facility() FacilityView {
|
||||
inv := a.ctl.Inventory()
|
||||
pol := inv.Facility()
|
||||
use := inv.PowerUse(time.Now())
|
||||
v := FacilityView{Margin: pol.Margin}
|
||||
for _, n := range inv.Nodes() {
|
||||
if n.Location().Empty() {
|
||||
v.Unlocated = append(v.Unlocated, n.ID)
|
||||
}
|
||||
}
|
||||
holds := a.ctl.Facility().Holds()
|
||||
for _, h := range holds {
|
||||
v.Holds = append(v.Holds, holdView(h, nil))
|
||||
// Listed with the others even before a node is labelled into it.
|
||||
if _, ok := use[h.Domain]; !ok {
|
||||
use[h.Domain] = inventory.PowerUse{Domain: h.Domain}
|
||||
}
|
||||
}
|
||||
keys := slices.Collect(maps.Keys(use))
|
||||
slices.SortFunc(keys, func(a, b facility.Domain) int {
|
||||
if c := slices.Index(facility.Levels, a.Level()) - slices.Index(facility.Levels, b.Level()); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(a.Name(), b.Name())
|
||||
})
|
||||
for _, d := range keys {
|
||||
u := use[d]
|
||||
dv := DomainView{Domain: string(d), Level: string(d.Level()), Name: d.Name(), Nodes: u.Nodes,
|
||||
CapWatts: u.Budget.CapWatts, LimitWatts: u.Limit, OtherWatts: u.Budget.OtherWatts,
|
||||
ReservedWatts: u.Reserved, MeasuredWatts: u.Measured, UsedWatts: u.Used}
|
||||
if h, ok := pol.Holds[d]; ok {
|
||||
hv := holdView(h, nil)
|
||||
dv.Held = &hv
|
||||
}
|
||||
v.Domains = append(v.Domains, dv)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (a *API) facilityReport(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Facility())
|
||||
}
|
||||
|
||||
func (a *API) holdDomain(w http.ResponseWriter, r *http.Request) {
|
||||
var req HoldRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
d, err := facility.ParseDomain(req.Domain)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
h := facility.Hold{Domain: d, Reason: strings.TrimSpace(req.Reason), Drain: req.Drain, By: actorOf(r), At: time.Now()}
|
||||
drained, err := a.ctl.Facility().Hold(r.Context(), h)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
a.ctl.Reconciler().Notify()
|
||||
writeJSON(w, http.StatusOK, holdView(h, drained))
|
||||
}
|
||||
|
||||
func (a *API) releaseDomain(w http.ResponseWriter, r *http.Request) {
|
||||
d, err := facility.ParseDomain(r.PathValue("domain"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
h, readied, err := a.ctl.Facility().Release(r.Context(), d)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
a.ctl.Reconciler().Notify()
|
||||
writeJSON(w, http.StatusOK, holdView(h, readied))
|
||||
}
|
||||
|
||||
func (a *API) setNodeLabels(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
var req NodeLabelsRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(req.Set) == 0 && len(req.Remove) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "nothing to change: give set, remove or both")
|
||||
return
|
||||
}
|
||||
// A node not registered yet is allowed: labelling the machines of a
|
||||
// rack before their agents start is the order a site is built in. A
|
||||
// mistyped name is the risk, and the answer says it is not registered.
|
||||
ops, err := a.ctl.Facility().SetLabels(r.Context(), id, req.Set, req.Remove, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
a.ctl.Reconciler().Notify()
|
||||
v := NodeLabelsView{ID: id, OperatorLabels: ops}
|
||||
if n, ok := a.ctl.Inventory().Get(id); ok {
|
||||
v.Labels = n.Labels
|
||||
if loc := n.Location(); !loc.Empty() {
|
||||
v.Location = &loc
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
@@ -0,0 +1,68 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
)
|
||||
|
||||
func TestARackIsLabelledHeldShownAndReleasedThroughTheAPI(t *testing.T) {
|
||||
a, ctl := setup(t, testModels(), gpuNode("gpu01", 8), gpuNode("gpu02", 8))
|
||||
|
||||
var lv NodeLabelsView
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu01/labels",
|
||||
`{"set":{"llmgw.io/rack":"r12-07","llmgw.io/pdu":"p1,p2"}}`, &lv); code != http.StatusOK {
|
||||
t.Fatalf("labelling: status %d", code)
|
||||
}
|
||||
if lv.Location == nil || lv.Location.Rack != "r12-07" || len(lv.Location.PDUs) != 2 {
|
||||
t.Fatalf("location after labelling: %+v", lv.Location)
|
||||
}
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu01/labels", `{"set":{"llmgw.io/rak":"x"}}`, nil); code != http.StatusBadRequest {
|
||||
t.Errorf("a misspelt location label: status %d, want 400", code)
|
||||
}
|
||||
|
||||
var hv HoldView
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/facility/holds",
|
||||
`{"domain":"rack:r12-07","reason":"PDU A replacement","drain":true}`, &hv); code != http.StatusOK {
|
||||
t.Fatalf("holding: status %d", code)
|
||||
}
|
||||
if !slices.Equal(hv.Nodes, []string{"gpu01"}) {
|
||||
t.Errorf("drained %v, want gpu01", hv.Nodes)
|
||||
}
|
||||
|
||||
var fv FacilityView
|
||||
call(t, a, http.MethodGet, "/api/v1/facility", "", &fv)
|
||||
var rack *DomainView
|
||||
for i, d := range fv.Domains {
|
||||
if d.Domain == "rack:r12-07" {
|
||||
rack = &fv.Domains[i]
|
||||
}
|
||||
}
|
||||
if rack == nil || rack.Held == nil || rack.Held.Reason != "PDU A replacement" || !slices.Equal(rack.Nodes, []string{"gpu01"}) {
|
||||
t.Fatalf("the rack in the facility view: %+v", rack)
|
||||
}
|
||||
if !slices.Equal(fv.Unlocated, []string{"gpu02"}) {
|
||||
t.Errorf("unlocated %v, want gpu02", fv.Unlocated)
|
||||
}
|
||||
|
||||
var nodes []NodeView
|
||||
call(t, a, http.MethodGet, "/api/v1/nodes", "", &nodes)
|
||||
if nodes[0].Held != "rack r12-07 is held: PDU A replacement" || nodes[0].HeldDomain != "rack:r12-07" {
|
||||
t.Errorf("gpu01 in the node list: held %q (%q)", nodes[0].Held, nodes[0].HeldDomain)
|
||||
}
|
||||
|
||||
if code := call(t, a, http.MethodDelete, "/api/v1/facility/holds/rack:r12-07", "", &hv); code != http.StatusOK {
|
||||
t.Fatalf("releasing: status %d", code)
|
||||
}
|
||||
if _, held := ctl.Inventory().Facility().HeldIn([]facility.Domain{"rack:r12-07"}); held {
|
||||
t.Error("released, still held")
|
||||
}
|
||||
if code := call(t, a, http.MethodDelete, "/api/v1/facility/holds/rack:r12-07", "", nil); code != http.StatusNotFound {
|
||||
t.Errorf("releasing again: status %d, want 404", code)
|
||||
}
|
||||
if code := call(t, a, http.MethodPost, "/api/v1/facility/holds", `{"domain":"r12-07"}`, nil); code != http.StatusBadRequest {
|
||||
t.Errorf("a domain with no level: status %d, want 400", code)
|
||||
}
|
||||
}
|
||||
@@ -62,6 +62,7 @@ var PathParams = map[string]string{
|
||||
"runs/{id}": "The quality run's ID, as starting it answered.",
|
||||
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
|
||||
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
|
||||
"holds/{domain}": "The domain as level:name, rack:r12-07, as `llmgw ctl facility` lists it.",
|
||||
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
|
||||
}
|
||||
|
||||
@@ -84,6 +85,17 @@ var Routes = []Route{
|
||||
{Method: "POST", Path: "/api/v1/nodes/{id}/engines", Summary: "Choose an engine's build for a node",
|
||||
Description: "The node installs the build chosen. Build is auto, a build such as cu129, or none.",
|
||||
Request: NodeEngineRequest{}, Response: NodeEngineRequest{}},
|
||||
{Method: "POST", Path: "/api/v1/nodes/{id}/labels", Summary: "Set or remove labels on a node, over its agent's own",
|
||||
Description: "Kept in the store. llmgw.io/site, room, row, rack and pdu say where the node is, for facility-aware placement.",
|
||||
Request: NodeLabelsRequest{}, Response: NodeLabelsView{}},
|
||||
{Method: "GET", Path: "/api/v1/facility", Summary: "The failure domains, their power against their caps, and what is held",
|
||||
Response: FacilityView{}},
|
||||
{Method: "POST", Path: "/api/v1/facility/holds", Summary: "Hold a domain for maintenance: nothing new is placed in it",
|
||||
Description: "With drain, each of its nodes is drained too, one replica at a time with the replacement up first.",
|
||||
Request: HoldRequest{}, Response: HoldView{}},
|
||||
{Method: "DELETE", Path: "/api/v1/facility/holds/{domain}", Summary: "Release a held domain",
|
||||
Description: "Nodes a draining hold drained are made ready again.",
|
||||
Response: HoldView{}},
|
||||
{Method: "DELETE", Path: "/api/v1/nodes/{id}", Summary: "Forget a node that has gone",
|
||||
Description: "Only a node that is lost, or unreachable with nothing on it; its certificate is revoked.",
|
||||
Response: NodeRemoved{}},
|
||||
|
||||
Reference in new issue
Block a user