Facility-aware placement: racks, PDUs, holds and power caps

Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels,
from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's).
Placement spreads a model's replicas across racks and PDUs, keeps a
multi-node replica within one domain when its profile says keep_within,
places nothing in a domain held with `llmgw ctl facility hold` (optionally
draining it), and refuses a shard that would take a domain past its power
cap from facility.budgets, counting declared, derived and measured draw.
Explain-placement names the domain, and keeps the rejections of a placed
replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the
domains; the console's Nodes view shows and sets location and holds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:17:49 +02:00
1 parent d41104a452
commit 0d3eb82b9b
42 files changed
+3510 -63

No files matched your search

+29
View File
@@ -32,6 +32,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrics"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
@@ -153,6 +154,10 @@ func (a *API) Register(mux Mux) {
mux.HandleFunc("POST /api/v1/nodes/{id}/state", a.setNodeState)
mux.HandleFunc("POST /api/v1/nodes/{id}/upgrade", a.upgradeNode)
mux.HandleFunc("POST /api/v1/nodes/{id}/engines", a.setNodeEngine)
mux.HandleFunc("POST /api/v1/nodes/{id}/labels", a.setNodeLabels)
mux.HandleFunc("GET /api/v1/facility", a.facilityReport)
mux.HandleFunc("POST /api/v1/facility/holds", a.holdDomain)
mux.HandleFunc("DELETE /api/v1/facility/holds/{domain}", a.releaseDomain)
mux.HandleFunc("DELETE /api/v1/nodes/{id}", a.removeNode)
mux.HandleFunc("POST /api/v1/fabric-test", a.fabricTest)
mux.HandleFunc("GET /api/v1/nodes/{id}/logs", a.nodeLogs)
@@ -254,6 +259,18 @@ type NodeView struct {
// answer when a node looks idle but nothing will schedule onto it.
Unhealthy []string `json:"unhealthy_gpus,omitempty"`
// Location is where the node is in the building, from its llmgw.io/site,
// room, row, rack and pdu labels; absent when it carries none.
Location *facility.Location `json:"location,omitempty"`
// Held names the hold keeping the node out of placement, "rack r12-07
// is held: PDU swap"; empty when none is.
Held string `json:"held,omitempty"`
// HeldDomain is the held domain, level:name, for releasing it.
HeldDomain string `json:"held_domain,omitempty"`
// OperatorLabels are the labels set through the control plane, which
// Labels already includes over the agent's own.
OperatorLabels map[string]string `json:"operator_labels,omitempty"`
LastSeenAgo string `json:"last_seen_ago"`
AgentVersion string `json:"agent_version,omitempty"`
// Features are the requests the agent answers beyond the usual:
@@ -566,6 +583,7 @@ func (a *API) Nodes() []NodeView {
}
inv := a.ctl.Inventory()
pol := inv.Facility()
out := make([]NodeView, 0)
for _, n := range inv.Nodes() {
v := NodeView{
@@ -576,6 +594,17 @@ func (a *API) Nodes() []NodeView {
Fabrics: n.FabricNames(), AgentVersion: n.AgentVersion, Features: n.Features,
Replicas: replicasOn[n.ID],
}
if loc := n.Location(); !loc.Empty() {
v.Location = &loc
}
if h, held := pol.HeldIn(n.Domains()); held {
v.HeldDomain = string(h.Domain)
v.Held = h.Domain.Human() + " is held"
if h.Reason != "" {
v.Held += ": " + h.Reason
}
}
v.OperatorLabels = inv.OperatorLabels(n.ID)
if free, total := n.LocalDisk(); total > 0 {
v.DiskFreeGB = int(free / (1 << 30))
v.DiskTotalGB = int(total / (1 << 30))
+203
View File
@@ -0,0 +1,203 @@
package adminapi
import (
"encoding/json"
"io"
"maps"
"net/http"
"slices"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// FacilityView is the failure domains as placement sees them.
type FacilityView struct {
// Domains are every domain a node is in or a budget names, by level
// (site, room, row, rack, pdu) and then name.
Domains []DomainView `json:"domains"`
// Holds are every domain held, including one no node is in yet.
Holds []HoldView `json:"holds,omitempty"`
// Margin is the share of every cap kept free.
Margin float64 `json:"margin,omitempty"`
// Unlocated are the nodes with no location label, which placement
// treats as each in a rack of its own.
Unlocated []string `json:"unlocated,omitempty"`
}
// DomainView is one domain: its nodes, its power and whether it is held.
type DomainView struct {
Domain string `json:"domain"`
Level string `json:"level"`
Name string `json:"name"`
Nodes []string `json:"nodes,omitempty"`
// CapWatts is the domain's cap, LimitWatts the cap less the margin;
// both zero when it has none.
CapWatts float64 `json:"cap_watts,omitempty"`
LimitWatts float64 `json:"limit_watts,omitempty"`
OtherWatts float64 `json:"other_watts,omitempty"`
// ReservedWatts is what is placed in it is estimated to draw;
// MeasuredWatts what its nodes' sensors read; UsedWatts what counts
// against the cap, the larger of the two node by node.
ReservedWatts float64 `json:"reserved_watts"`
MeasuredWatts float64 `json:"measured_watts"`
UsedWatts float64 `json:"used_watts"`
// Held is set when the domain is held.
Held *HoldView `json:"held,omitempty"`
}
// HoldRequest holds a domain.
type HoldRequest struct {
// Domain is level:name: rack:r12-07, pdu:r12-07-a, row:12.
Domain string `json:"domain"`
Reason string `json:"reason,omitempty"`
// Drain moves the work in the domain elsewhere too.
Drain bool `json:"drain,omitempty"`
}
// HoldView is a hold, and the nodes holding or releasing it changed.
type HoldView struct {
Domain string `json:"domain"`
Reason string `json:"reason,omitempty"`
Drain bool `json:"drain,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
// Nodes are those this call drained, or made ready again on release.
Nodes []string `json:"nodes,omitempty"`
}
// NodeLabelsRequest sets and removes labels on a node.
type NodeLabelsRequest struct {
Set map[string]string `json:"set,omitempty"`
// Remove are keys whose label set through the control plane goes; the
// agent's own value for the key, if it has one, is in force again.
Remove []string `json:"remove,omitempty"`
}
// NodeLabelsView is a node's labels after a change.
type NodeLabelsView struct {
ID string `json:"id"`
// OperatorLabels are those set through the control plane.
OperatorLabels map[string]string `json:"operator_labels,omitempty"`
// Labels are what placement matches: the agent's with the operator's
// over them. Absent for a node that has not registered yet.
Labels map[string]string `json:"labels,omitempty"`
Location *facility.Location `json:"location,omitempty"`
}
func holdView(h facility.Hold, nodes []string) HoldView {
return HoldView{Domain: string(h.Domain), Reason: h.Reason, Drain: h.Drain, By: h.By, At: h.At, Nodes: nodes}
}
// Facility builds the facility view; exported for the console's sake, as
// Nodes is.
func (a *API) Facility() FacilityView {
inv := a.ctl.Inventory()
pol := inv.Facility()
use := inv.PowerUse(time.Now())
v := FacilityView{Margin: pol.Margin}
for _, n := range inv.Nodes() {
if n.Location().Empty() {
v.Unlocated = append(v.Unlocated, n.ID)
}
}
holds := a.ctl.Facility().Holds()
for _, h := range holds {
v.Holds = append(v.Holds, holdView(h, nil))
// Listed with the others even before a node is labelled into it.
if _, ok := use[h.Domain]; !ok {
use[h.Domain] = inventory.PowerUse{Domain: h.Domain}
}
}
keys := slices.Collect(maps.Keys(use))
slices.SortFunc(keys, func(a, b facility.Domain) int {
if c := slices.Index(facility.Levels, a.Level()) - slices.Index(facility.Levels, b.Level()); c != 0 {
return c
}
return strings.Compare(a.Name(), b.Name())
})
for _, d := range keys {
u := use[d]
dv := DomainView{Domain: string(d), Level: string(d.Level()), Name: d.Name(), Nodes: u.Nodes,
CapWatts: u.Budget.CapWatts, LimitWatts: u.Limit, OtherWatts: u.Budget.OtherWatts,
ReservedWatts: u.Reserved, MeasuredWatts: u.Measured, UsedWatts: u.Used}
if h, ok := pol.Holds[d]; ok {
hv := holdView(h, nil)
dv.Held = &hv
}
v.Domains = append(v.Domains, dv)
}
return v
}
func (a *API) facilityReport(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, http.StatusOK, a.Facility())
}
func (a *API) holdDomain(w http.ResponseWriter, r *http.Request) {
var req HoldRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
d, err := facility.ParseDomain(req.Domain)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
h := facility.Hold{Domain: d, Reason: strings.TrimSpace(req.Reason), Drain: req.Drain, By: actorOf(r), At: time.Now()}
drained, err := a.ctl.Facility().Hold(r.Context(), h)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
a.ctl.Reconciler().Notify()
writeJSON(w, http.StatusOK, holdView(h, drained))
}
func (a *API) releaseDomain(w http.ResponseWriter, r *http.Request) {
d, err := facility.ParseDomain(r.PathValue("domain"))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
h, readied, err := a.ctl.Facility().Release(r.Context(), d)
if err != nil {
writeError(w, http.StatusNotFound, err.Error())
return
}
a.ctl.Reconciler().Notify()
writeJSON(w, http.StatusOK, holdView(h, readied))
}
func (a *API) setNodeLabels(w http.ResponseWriter, r *http.Request) {
id := r.PathValue("id")
var req NodeLabelsRequest
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
return
}
if len(req.Set) == 0 && len(req.Remove) == 0 {
writeError(w, http.StatusBadRequest, "nothing to change: give set, remove or both")
return
}
// A node not registered yet is allowed: labelling the machines of a
// rack before their agents start is the order a site is built in. A
// mistyped name is the risk, and the answer says it is not registered.
ops, err := a.ctl.Facility().SetLabels(r.Context(), id, req.Set, req.Remove, actorOf(r))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
a.ctl.Reconciler().Notify()
v := NodeLabelsView{ID: id, OperatorLabels: ops}
if n, ok := a.ctl.Inventory().Get(id); ok {
v.Labels = n.Labels
if loc := n.Location(); !loc.Empty() {
v.Location = &loc
}
}
writeJSON(w, http.StatusOK, v)
}
@@ -0,0 +1,68 @@
package adminapi
import (
"net/http"
"slices"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
func TestARackIsLabelledHeldShownAndReleasedThroughTheAPI(t *testing.T) {
a, ctl := setup(t, testModels(), gpuNode("gpu01", 8), gpuNode("gpu02", 8))
var lv NodeLabelsView
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu01/labels",
`{"set":{"llmgw.io/rack":"r12-07","llmgw.io/pdu":"p1,p2"}}`, &lv); code != http.StatusOK {
t.Fatalf("labelling: status %d", code)
}
if lv.Location == nil || lv.Location.Rack != "r12-07" || len(lv.Location.PDUs) != 2 {
t.Fatalf("location after labelling: %+v", lv.Location)
}
if code := call(t, a, http.MethodPost, "/api/v1/nodes/gpu01/labels", `{"set":{"llmgw.io/rak":"x"}}`, nil); code != http.StatusBadRequest {
t.Errorf("a misspelt location label: status %d, want 400", code)
}
var hv HoldView
if code := call(t, a, http.MethodPost, "/api/v1/facility/holds",
`{"domain":"rack:r12-07","reason":"PDU A replacement","drain":true}`, &hv); code != http.StatusOK {
t.Fatalf("holding: status %d", code)
}
if !slices.Equal(hv.Nodes, []string{"gpu01"}) {
t.Errorf("drained %v, want gpu01", hv.Nodes)
}
var fv FacilityView
call(t, a, http.MethodGet, "/api/v1/facility", "", &fv)
var rack *DomainView
for i, d := range fv.Domains {
if d.Domain == "rack:r12-07" {
rack = &fv.Domains[i]
}
}
if rack == nil || rack.Held == nil || rack.Held.Reason != "PDU A replacement" || !slices.Equal(rack.Nodes, []string{"gpu01"}) {
t.Fatalf("the rack in the facility view: %+v", rack)
}
if !slices.Equal(fv.Unlocated, []string{"gpu02"}) {
t.Errorf("unlocated %v, want gpu02", fv.Unlocated)
}
var nodes []NodeView
call(t, a, http.MethodGet, "/api/v1/nodes", "", &nodes)
if nodes[0].Held != "rack r12-07 is held: PDU A replacement" || nodes[0].HeldDomain != "rack:r12-07" {
t.Errorf("gpu01 in the node list: held %q (%q)", nodes[0].Held, nodes[0].HeldDomain)
}
if code := call(t, a, http.MethodDelete, "/api/v1/facility/holds/rack:r12-07", "", &hv); code != http.StatusOK {
t.Fatalf("releasing: status %d", code)
}
if _, held := ctl.Inventory().Facility().HeldIn([]facility.Domain{"rack:r12-07"}); held {
t.Error("released, still held")
}
if code := call(t, a, http.MethodDelete, "/api/v1/facility/holds/rack:r12-07", "", nil); code != http.StatusNotFound {
t.Errorf("releasing again: status %d, want 404", code)
}
if code := call(t, a, http.MethodPost, "/api/v1/facility/holds", `{"domain":"r12-07"}`, nil); code != http.StatusBadRequest {
t.Errorf("a domain with no level: status %d, want 400", code)
}
}
+12
View File
@@ -62,6 +62,7 @@ var PathParams = map[string]string{
"runs/{id}": "The quality run's ID, as starting it answered.",
"requests/{id}": "The request's ID: the x-request-id a client was answered with.",
"fabrics/{id}": "The physical fabric's ID, as GET /api/v1/fabrics lists it.",
"holds/{domain}": "The domain as level:name, rack:r12-07, as `llmgw ctl facility` lists it.",
"quotas/{scope}": "The scope: org:NAME, team:ORG/TEAM, project:ORG/TEAM/PROJECT, or their IDs after the colon.",
}
@@ -84,6 +85,17 @@ var Routes = []Route{
{Method: "POST", Path: "/api/v1/nodes/{id}/engines", Summary: "Choose an engine's build for a node",
Description: "The node installs the build chosen. Build is auto, a build such as cu129, or none.",
Request: NodeEngineRequest{}, Response: NodeEngineRequest{}},
{Method: "POST", Path: "/api/v1/nodes/{id}/labels", Summary: "Set or remove labels on a node, over its agent's own",
Description: "Kept in the store. llmgw.io/site, room, row, rack and pdu say where the node is, for facility-aware placement.",
Request: NodeLabelsRequest{}, Response: NodeLabelsView{}},
{Method: "GET", Path: "/api/v1/facility", Summary: "The failure domains, their power against their caps, and what is held",
Response: FacilityView{}},
{Method: "POST", Path: "/api/v1/facility/holds", Summary: "Hold a domain for maintenance: nothing new is placed in it",
Description: "With drain, each of its nodes is drained too, one replica at a time with the replacement up first.",
Request: HoldRequest{}, Response: HoldView{}},
{Method: "DELETE", Path: "/api/v1/facility/holds/{domain}", Summary: "Release a held domain",
Description: "Nodes a draining hold drained are made ready again.",
Response: HoldView{}},
{Method: "DELETE", Path: "/api/v1/nodes/{id}", Summary: "Forget a node that has gone",
Description: "Only a node that is lost, or unreachable with nothing on it; its certificate is revoked.",
Response: NodeRemoved{}},
+26
View File
@@ -14,6 +14,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
)
@@ -226,6 +227,21 @@ type Profile struct {
// Agents have reported labels since the beginning; this is what reads them.
NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"`
// PowerWatts is what one replica of this profile draws, in watts, split
// evenly over its nodes, counted against the power cap of every rack,
// PDU or other domain its nodes are in (facility.budgets in server.yaml).
// Zero derives it: each card at the power limit it reports, each
// reserved core at facility.cpu_core_watts. Only matters where a domain
// has a cap.
PowerWatts float64 `json:"power_watts,omitempty" koanf:"power_watts"`
// KeepWithin keeps a multi-node replica's nodes inside one rack, row,
// room or site, read from their llmgw.io/rack (and so on) labels: for a
// fabric whose leaves do not reach across racks, or cables that do not.
// Nodes without the label are passed over for such a profile. Empty is
// no constraint.
KeepWithin string `json:"keep_within,omitempty" koanf:"keep_within"`
// Weight is the router's capacity weight. It should come from ctl bench
// rather than a guess (plan §6.2.1).
Weight float64 `json:"weight,omitempty" koanf:"weight"`
@@ -715,6 +731,14 @@ func (m Model) validate() error {
return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err)
}
if p.PowerWatts < 0 {
return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts)
}
if p.KeepWithin != "" {
if _, err := facility.ParseLevel(p.KeepWithin); err != nil {
return fmt.Errorf("model %q profile %q: keep_within: %w", m.Name, p.Name, err)
}
}
if p.GPUClass == "" && !cpuCapable(p.Engine) {
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
m.Name, p.Name, p.Engine)
@@ -844,6 +868,8 @@ func (m Model) PlacementProfiles() []placement.Profile {
MinMemoryGB: p.MinMemoryGB,
MinDiskGB: p.MinDiskGB,
CPUCores: p.CPUCores,
PowerWatts: p.PowerWatts,
KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
// The model's source, so placement can prefer nodes that already
// hold the weights rather than one that must first move them.
Source: m.Source,
+24 -18
View File
@@ -78,15 +78,17 @@ type Controller struct {
// engineChoices are the engine builds chosen per node; nil without a
// store to keep them in.
engineChoices *EngineChoices
cfg Config
log *slog.Logger
inv *inventory.Store
cat *catalog.Live
rec *reconciler.Reconciler
lead *Leadership
disp reconciler.Dispatcher
sink EndpointSink
usage UsageSource
// facility is the holds, budgets and operator labels placement reads.
facility *Facility
cfg Config
log *slog.Logger
inv *inventory.Store
cat *catalog.Live
rec *reconciler.Reconciler
lead *Leadership
disp reconciler.Dispatcher
sink EndpointSink
usage UsageSource
mu sync.Mutex
last string // fingerprint of the last published list
@@ -106,15 +108,16 @@ func New(cfg Config, cat *catalog.Live, disp reconciler.Dispatcher, sink Endpoin
}
inv := inventory.NewStore()
return &Controller{
cfg: cfg,
log: log,
inv: inv,
cat: cat,
rec: reconciler.New(cfg.Reconciler, cat, inv, disp, log),
disp: disp,
sink: sink,
stop: make(chan struct{}),
done: make(chan struct{}),
facility: NewFacility(inv, log),
cfg: cfg,
log: log,
inv: inv,
cat: cat,
rec: reconciler.New(cfg.Reconciler, cat, inv, disp, log),
disp: disp,
sink: sink,
stop: make(chan struct{}),
done: make(chan struct{}),
}
}
@@ -415,6 +418,9 @@ func (c *Controller) QuotaUse() []reconciler.QuotaUse { return c.rec.QuotaUse()
// SetEngineChoices gives the controller the engine builds chosen per node.
func (c *Controller) SetEngineChoices(e *EngineChoices) { c.engineChoices = e }
// Facility is where domains are held and nodes labelled; never nil.
func (c *Controller) Facility() *Facility { return c.facility }
// EngineChoices are the engine builds chosen per node, or nil.
func (c *Controller) EngineChoices() *EngineChoices { return c.engineChoices }
+231
View File
@@ -0,0 +1,231 @@
package control
import (
"context"
"fmt"
"log/slog"
"maps"
"slices"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// FacilityStore is where holds and operator labels are kept.
type FacilityStore interface {
FacilityHolds(ctx context.Context) ([]store.FacilityHold, error)
SetFacilityHold(ctx context.Context, h store.FacilityHold) error
DeleteFacilityHold(ctx context.Context, domain string) error
NodeLabels(ctx context.Context) (map[string]map[string]string, error)
SetNodeLabel(ctx context.Context, node, key, value, by string) error
}
// Facility is the operator's side of facility-aware placement: the budgets
// from server.yaml, the domains held for maintenance, and the labels set on
// nodes through the control plane. It keeps them in the store when there is
// one, and puts them in force in the inventory, where every placement —
// the reconciler's, headroom's and preemption's dry runs — reads them.
//
// Without a store it still works, for the life of the process: a held rack
// is worth honouring even on a control plane that will forget it.
type Facility struct {
inv *inventory.Store
log *slog.Logger
mu sync.Mutex
st FacilityStore
base facility.Policy
holds map[facility.Domain]facility.Hold
}
// NewFacility starts with nothing held and nothing capped.
func NewFacility(inv *inventory.Store, log *slog.Logger) *Facility {
if log == nil {
log = slog.Default()
}
return &Facility{inv: inv, log: log, holds: map[facility.Domain]facility.Hold{}}
}
// SetPolicy puts the budgets and estimates from configuration in force. Its
// holds are ignored: those are the operator's, kept here.
func (f *Facility) SetPolicy(p facility.Policy) {
f.mu.Lock()
defer f.mu.Unlock()
f.base = p.Clone()
f.applyLocked()
}
// Policy is what placement reads now.
func (f *Facility) Policy() facility.Policy { return f.inv.Facility() }
// UseStore keeps holds and labels in the store from now on, and puts in
// force what it already holds.
func (f *Facility) UseStore(ctx context.Context, st FacilityStore) {
f.mu.Lock()
defer f.mu.Unlock()
f.st = st
if holds, err := st.FacilityHolds(ctx); err != nil {
f.log.Error("could not read the domains held for maintenance; none is held until they are set again", "err", err)
} else {
for _, h := range holds {
d, err := facility.ParseDomain(h.Domain)
if err != nil {
continue
}
f.holds[d] = facility.Hold{Domain: d, Reason: h.Reason, Drain: h.Drain, By: h.By, At: h.At}
}
}
if labels, err := st.NodeLabels(ctx); err != nil {
f.log.Error("could not read the labels set on nodes; nodes carry only their agents' until they are set again", "err", err)
} else {
for node, l := range labels {
f.inv.SetOperatorLabels(node, l)
}
}
f.applyLocked()
}
func (f *Facility) applyLocked() {
p := f.base.Clone()
p.Holds = maps.Clone(f.holds)
f.inv.SetFacility(p)
}
// Holds are the domains held, oldest first.
func (f *Facility) Holds() []facility.Hold {
f.mu.Lock()
defer f.mu.Unlock()
out := slices.Collect(maps.Values(f.holds))
slices.SortFunc(out, func(a, b facility.Hold) int {
if c := a.At.Compare(b.At); c != 0 {
return c
}
return strings.Compare(string(a.Domain), string(b.Domain))
})
return out
}
// Hold takes a domain out of placement, and with drain moves the work in it
// elsewhere: each of its nodes is drained, one replica at a time with the
// replacement up first, as `ctl drain` does. It answers the nodes drained.
func (f *Facility) Hold(ctx context.Context, h facility.Hold) ([]string, error) {
if h.At.IsZero() {
h.At = time.Now()
}
f.mu.Lock()
defer f.mu.Unlock()
if f.st != nil {
if err := f.st.SetFacilityHold(ctx, store.FacilityHold{
Domain: string(h.Domain), Reason: h.Reason, Drain: h.Drain, At: h.At, By: h.By,
}); err != nil {
return nil, fmt.Errorf("keeping the hold: %w", err)
}
}
f.holds[h.Domain] = h
f.applyLocked()
var drained []string
if h.Drain {
for _, n := range f.inv.Nodes() {
if slices.Contains(n.Domains(), h.Domain) && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
f.inv.SetState(n.ID, inventory.Draining)
drained = append(drained, n.ID)
}
}
}
f.log.Info("domain held: nothing new is placed in it", "domain", h.Domain, "reason", h.Reason,
"drain", h.Drain, "by", h.By, "drained", drained)
return drained, nil
}
// Release puts a domain back into placement. Nodes a draining hold drained
// are made ready again, unless another hold still covers them; a node an
// operator drained by itself is not told apart from those, so it is readied
// too, which is what releasing its rack asks for.
func (f *Facility) Release(ctx context.Context, d facility.Domain) (facility.Hold, []string, error) {
f.mu.Lock()
defer f.mu.Unlock()
h, ok := f.holds[d]
if !ok {
return facility.Hold{}, nil, fmt.Errorf("%s is not held; `llmgw ctl facility` lists what is", d)
}
if f.st != nil {
if err := f.st.DeleteFacilityHold(ctx, string(d)); err != nil {
return facility.Hold{}, nil, fmt.Errorf("releasing the hold: %w", err)
}
}
delete(f.holds, d)
f.applyLocked()
var readied []string
if h.Drain {
pol := f.inv.Facility()
for _, n := range f.inv.Nodes() {
doms := n.Domains()
if !slices.Contains(doms, d) || n.State != inventory.Draining {
continue
}
if _, still := pol.HeldIn(doms); still {
continue
}
f.inv.SetState(n.ID, inventory.Ready)
readied = append(readied, n.ID)
}
}
f.log.Info("domain released", "domain", d, "readied", readied)
return h, readied, nil
}
// SetLabels sets and removes labels on a node through the control plane,
// over what its agent reports, and answers the operator's labels as they
// now are. A node not registered yet takes them when it does.
func (f *Facility) SetLabels(ctx context.Context, node string, set map[string]string, remove []string, by string) (map[string]string, error) {
for k, v := range set {
if err := checkLabel(k, v); err != nil {
return nil, err
}
}
f.mu.Lock()
defer f.mu.Unlock()
now := f.inv.OperatorLabels(node)
if now == nil {
now = map[string]string{}
}
if f.st != nil {
for _, k := range slices.Sorted(maps.Keys(set)) {
if err := f.st.SetNodeLabel(ctx, node, k, set[k], by); err != nil {
return nil, fmt.Errorf("keeping label %s: %w", k, err)
}
}
for _, k := range remove {
if err := f.st.SetNodeLabel(ctx, node, k, "", by); err != nil {
return nil, fmt.Errorf("removing label %s: %w", k, err)
}
}
}
maps.Copy(now, set)
for _, k := range remove {
delete(now, k)
}
f.inv.SetOperatorLabels(node, now)
f.log.Info("node labels set", "node", node, "set", set, "removed", remove, "by", by)
return now, nil
}
// checkLabel refuses a key no selector could match, and a misspelt location:
// llmgw.io/rak would be kept, shown, and never read by placement.
func checkLabel(k, v string) error {
switch {
case strings.TrimSpace(k) == "" || strings.ContainsAny(k, " =\t\n"):
return fmt.Errorf("label key %q is empty or has a space or = in it", k)
case strings.TrimSpace(v) == "":
return fmt.Errorf("label %s has no value; remove it with %s- instead", k, k)
case strings.HasPrefix(k, facility.LabelPrefix):
if _, err := facility.ParseLevel(strings.TrimPrefix(k, facility.LabelPrefix)); err != nil {
return fmt.Errorf("label %s: %s is reserved for where a node is: llmgw.io/site, room, row, rack or pdu", k, facility.LabelPrefix)
}
}
return nil
}
+268
View File
@@ -0,0 +1,268 @@
// Package facility is where nodes are in the building, and what the building
// allows: the racks, rows, rooms and power distribution units a node belongs
// to, which of them an operator has taken out of service, and how much power
// each may draw.
//
// Placement reads it to keep replicas of one model off a single rack or PDU,
// to keep a replica's shards inside one domain when its profile asks, to place
// nothing in a domain held for maintenance, and to refuse a replica that would
// take a domain past its power cap. Everything here is optional: a node with
// no location labels and a cluster with no facility configuration place
// exactly as they did before.
package facility
import (
"fmt"
"slices"
"strings"
"time"
)
// Level is a kind of failure domain, from the largest to the smallest.
type Level string
const (
Site Level = "site"
Room Level = "room"
Row Level = "row"
Rack Level = "rack"
// PDU is a power distribution unit. A node may be on several, one per
// feed of a dual-fed rack, so its label takes a comma separated list.
PDU Level = "pdu"
)
// Levels are every level, the building's tree first and power after it.
var Levels = []Level{Site, Room, Row, Rack, PDU}
// LabelPrefix starts the node labels that say where a node is.
const LabelPrefix = "llmgw.io/"
// Label is the node label a level is read from: llmgw.io/rack.
func (l Level) Label() string { return LabelPrefix + string(l) }
// ParseLevel reads a level's name, as a profile or a command gives it.
func ParseLevel(s string) (Level, error) {
l := Level(strings.ToLower(strings.TrimSpace(s)))
if slices.Contains(Levels, l) {
return l, nil
}
return "", fmt.Errorf("%q is not a facility level: use site, room, row, rack or pdu", s)
}
// Domain names one failure domain as level:name, rack:r12-07. Names are taken
// as the labels give them, so a rack is one domain across the whole cluster:
// two rooms that both have a rack r1 must label them apart.
type Domain string
// Of builds a domain from its level and name.
func Of(l Level, name string) Domain { return Domain(string(l) + ":" + name) }
// Level is the domain's level.
func (d Domain) Level() Level {
l, _, _ := strings.Cut(string(d), ":")
return Level(l)
}
// Name is the domain's name within its level.
func (d Domain) Name() string {
_, n, _ := strings.Cut(string(d), ":")
return n
}
// String is level:name; human text says "rack r12-07".
func (d Domain) String() string { return string(d) }
// Human is the domain as a sentence says it: rack r12-07.
func (d Domain) Human() string { return string(d.Level()) + " " + d.Name() }
// ParseDomain reads level:name, the way commands and configuration name a
// domain.
func ParseDomain(s string) (Domain, error) {
l, n, ok := strings.Cut(strings.TrimSpace(s), ":")
if !ok || strings.TrimSpace(n) == "" {
return "", fmt.Errorf("%q is not a domain: name one as level:name, like rack:r12-07", s)
}
lv, err := ParseLevel(l)
if err != nil {
return "", err
}
return Of(lv, strings.TrimSpace(n)), nil
}
// Location is where a node is, read from its labels. Empty fields are not
// known, which places as it always did.
type Location struct {
Site string `json:"site,omitempty"`
Room string `json:"room,omitempty"`
Row string `json:"row,omitempty"`
Rack string `json:"rack,omitempty"`
PDUs []string `json:"pdus,omitempty"`
}
// LocationOf reads a node's labels.
func LocationOf(labels map[string]string) Location {
get := func(l Level) string { return strings.TrimSpace(labels[l.Label()]) }
loc := Location{Site: get(Site), Room: get(Room), Row: get(Row), Rack: get(Rack)}
for _, p := range strings.Split(get(PDU), ",") {
if p = strings.TrimSpace(p); p != "" && !slices.Contains(loc.PDUs, p) {
loc.PDUs = append(loc.PDUs, p)
}
}
return loc
}
// Empty reports whether nothing is known about where the node is.
func (l Location) Empty() bool {
return l.Site == "" && l.Room == "" && l.Row == "" && l.Rack == "" && len(l.PDUs) == 0
}
// Domains are every domain the location is in, largest first.
func (l Location) Domains() []Domain {
var out []Domain
add := func(lv Level, name string) {
if name != "" {
out = append(out, Of(lv, name))
}
}
add(Site, l.Site)
add(Room, l.Room)
add(Row, l.Row)
add(Rack, l.Rack)
for _, p := range l.PDUs {
add(PDU, p)
}
return out
}
// At is the domain at one level, or empty. For PDUs it is the first.
func (l Location) At(lv Level) Domain {
for _, d := range l.Domains() {
if d.Level() == lv {
return d
}
}
return ""
}
// Short is the location in a few characters for a table: the rack, else the
// row, room or site, with the PDUs after a slash.
func (l Location) Short() string {
s := ""
switch {
case l.Rack != "":
s = l.Rack
case l.Row != "":
s = "row " + l.Row
case l.Room != "":
s = l.Room
case l.Site != "":
s = l.Site
}
if len(l.PDUs) > 0 {
if s != "" {
s += " "
}
s += "pdu " + strings.Join(l.PDUs, ",")
}
return s
}
// Hold takes a domain out of placement: maintenance, a cooling fault, work on
// a PDU. What runs there keeps running unless Drain is set.
type Hold struct {
Domain Domain `json:"domain"`
Reason string `json:"reason,omitempty"`
Drain bool `json:"drain,omitempty"`
By string `json:"by,omitempty"`
At time.Time `json:"at"`
}
// Budget is a domain's power cap.
type Budget struct {
// CapWatts is what the domain may draw. For a dual-fed rack it is what
// one feed can carry alone, since that is what the rack must survive.
CapWatts float64
// OtherWatts is drawn in the domain by things llmgw does not place on:
// switches, storage, nodes outside the cluster.
OtherWatts float64
}
// Policy is the facility's rules as placement reads them. The zero value
// holds nothing and caps nothing.
type Policy struct {
// Holds are the domains taken out of placement.
Holds map[Domain]Hold
// Budgets are the domains with a power cap.
Budgets map[Domain]Budget
// Margin is the share of every cap kept free, 0.1 for ten percent.
Margin float64
// GPUWatts is a card's draw when its own power limit is not known.
GPUWatts float64
// CPUCoreWatts is what a core reserved by a CPU replica draws.
CPUCoreWatts float64
// NodeWatts is what a node draws beyond the GPUs and the cores given to
// replicas: its idle draw, fans, memory and disks. Counted for every node
// in a capped domain, busy or not.
NodeWatts float64
// MeasuredFresh is how old a node's power reading may be and still be
// counted. Zero is two minutes.
MeasuredFresh time.Duration
}
// Clone is a copy whose maps are its own.
func (p Policy) Clone() Policy {
c := p
if p.Holds != nil {
c.Holds = make(map[Domain]Hold, len(p.Holds))
for k, v := range p.Holds {
c.Holds[k] = v
}
}
if p.Budgets != nil {
c.Budgets = make(map[Domain]Budget, len(p.Budgets))
for k, v := range p.Budgets {
c.Budgets[k] = v
}
}
return c
}
// HeldIn is the first hold among a node's domains, largest first.
func (p Policy) HeldIn(domains []Domain) (Hold, bool) {
for _, d := range domains {
if h, ok := p.Holds[d]; ok {
return h, true
}
}
return Hold{}, false
}
// Limit is the most a domain may draw, its cap less the margin; zero with
// no cap.
func (p Policy) Limit(d Domain) float64 {
b, ok := p.Budgets[d]
if !ok || b.CapWatts <= 0 {
return 0
}
m := p.Margin
if m < 0 || m >= 1 {
m = 0
}
return b.CapWatts * (1 - m)
}
// Fresh is how old a power reading may be.
func (p Policy) Fresh() time.Duration {
if p.MeasuredFresh > 0 {
return p.MeasuredFresh
}
return 2 * time.Minute
}
// KW renders watts as a sentence about a rack would.
func KW(w float64) string {
if w >= 1000 {
return fmt.Sprintf("%.1f kW", w/1000)
}
return fmt.Sprintf("%.0f W", w)
}
@@ -0,0 +1,67 @@
package facility
import (
"slices"
"testing"
)
func TestALocationIsReadFromTheWellKnownLabels(t *testing.T) {
loc := LocationOf(map[string]string{
"llmgw.io/site": "sto1", "llmgw.io/room": "hall-b", "llmgw.io/row": "12",
"llmgw.io/rack": "r12-07", "llmgw.io/pdu": "r12-07-a, r12-07-b,r12-07-a", "role": "gpu",
})
want := []Domain{"site:sto1", "room:hall-b", "row:12", "rack:r12-07", "pdu:r12-07-a", "pdu:r12-07-b"}
if got := loc.Domains(); !slices.Equal(got, want) {
t.Errorf("domains %v, want %v", got, want)
}
if loc.Short() != "r12-07 pdu r12-07-a,r12-07-b" {
t.Errorf("short %q", loc.Short())
}
if loc.At(Rack) != "rack:r12-07" || loc.At(PDU) != "pdu:r12-07-a" {
t.Errorf("at rack %q, at pdu %q", loc.At(Rack), loc.At(PDU))
}
}
func TestANodeWithoutLabelsIsNowhere(t *testing.T) {
loc := LocationOf(map[string]string{"role": "gpu"})
if !loc.Empty() || len(loc.Domains()) != 0 || loc.Short() != "" {
t.Errorf("an unlabelled node is somewhere: %+v", loc)
}
}
func TestADomainIsNamedAsLevelColonName(t *testing.T) {
d, err := ParseDomain(" Rack:r12-07 ")
if err != nil || d != "rack:r12-07" || d.Human() != "rack r12-07" {
t.Errorf("got %q, %v", d, err)
}
for _, bad := range []string{"r12-07", "rack:", "shelf:3"} {
if _, err := ParseDomain(bad); err == nil {
t.Errorf("%q was accepted as a domain", bad)
}
}
}
func TestTheLimitIsTheCapLessTheMargin(t *testing.T) {
p := Policy{Margin: 0.1, Budgets: map[Domain]Budget{"rack:r1": {CapWatts: 16000}}}
if got := p.Limit("rack:r1"); got != 14400 {
t.Errorf("limit %v, want 14400", got)
}
if p.Limit("rack:r2") != 0 {
t.Error("a domain with no budget has a limit")
}
}
func TestTheFirstHoldAmongANodesDomainsIsFound(t *testing.T) {
p := Policy{Holds: map[Domain]Hold{"pdu:p2": {Domain: "pdu:p2", Reason: "breaker"}}}
if h, ok := p.HeldIn([]Domain{"rack:r1", "pdu:p1", "pdu:p2"}); !ok || h.Reason != "breaker" {
t.Errorf("got %+v, %v", h, ok)
}
if _, ok := p.HeldIn([]Domain{"rack:r1"}); ok {
t.Error("an unheld rack is held")
}
c := p.Clone()
delete(c.Holds, "pdu:p2")
if _, ok := p.Holds["pdu:p2"]; !ok {
t.Error("changing a clone changed the original")
}
}
+132
View File
@@ -0,0 +1,132 @@
package control
import (
"context"
"log/slog"
"maps"
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// memFacilityStore is the store's two tables in memory.
type memFacilityStore struct {
holds map[string]store.FacilityHold
labels map[string]map[string]string
}
func newMemFacilityStore() *memFacilityStore {
return &memFacilityStore{holds: map[string]store.FacilityHold{}, labels: map[string]map[string]string{}}
}
func (m *memFacilityStore) FacilityHolds(context.Context) ([]store.FacilityHold, error) {
return slices.Collect(maps.Values(m.holds)), nil
}
func (m *memFacilityStore) SetFacilityHold(_ context.Context, h store.FacilityHold) error {
m.holds[h.Domain] = h
return nil
}
func (m *memFacilityStore) DeleteFacilityHold(_ context.Context, d string) error {
delete(m.holds, d)
return nil
}
func (m *memFacilityStore) NodeLabels(context.Context) (map[string]map[string]string, error) {
return m.labels, nil
}
func (m *memFacilityStore) SetNodeLabel(_ context.Context, node, k, v, _ string) error {
if m.labels[node] == nil {
m.labels[node] = map[string]string{}
}
if v == "" {
delete(m.labels[node], k)
} else {
m.labels[node][k] = v
}
return nil
}
func rackNode(id, rack string) inventory.Node {
return inventory.Node{ID: id, State: inventory.Ready, Labels: map[string]string{"llmgw.io/rack": rack}}
}
func TestADrainingHoldDrainsTheRackAndReleasingItReadiesItAgain(t *testing.T) {
inv := inventory.NewStore()
inv.Upsert(rackNode("a1", "r1"))
inv.Upsert(rackNode("a2", "r1"))
inv.Upsert(rackNode("b1", "r2"))
f := NewFacility(inv, slog.New(slog.DiscardHandler))
ctx := t.Context()
drained, err := f.Hold(ctx, facility.Hold{Domain: "rack:r1", Reason: "PDU swap", Drain: true})
if err != nil || !slices.Equal(drained, []string{"a1", "a2"}) {
t.Fatalf("drained %v, %v", drained, err)
}
if n, _ := inv.Get("b1"); n.State != inventory.Ready {
t.Errorf("b1, in another rack, is %s", n.State)
}
if _, held := inv.Facility().HeldIn([]facility.Domain{"rack:r1"}); !held {
t.Error("the hold is not in force for placement")
}
_, readied, err := f.Release(ctx, "rack:r1")
if err != nil || !slices.Equal(readied, []string{"a1", "a2"}) {
t.Fatalf("readied %v, %v", readied, err)
}
if _, _, err := f.Release(ctx, "rack:r1"); err == nil {
t.Error("releasing a rack not held is not an error")
}
}
func TestAHoldWithoutDrainLeavesTheNodesAsTheyAre(t *testing.T) {
inv := inventory.NewStore()
inv.Upsert(rackNode("a1", "r1"))
f := NewFacility(inv, nil)
if drained, _ := f.Hold(t.Context(), facility.Hold{Domain: "rack:r1"}); len(drained) != 0 {
t.Errorf("drained %v without being asked", drained)
}
if n, _ := inv.Get("a1"); n.State != inventory.Ready {
t.Errorf("a1 is %s", n.State)
}
}
func TestHoldsAndLabelsAreKeptAndReadBackByTheNextControlPlane(t *testing.T) {
st := newMemFacilityStore()
inv := inventory.NewStore()
f := NewFacility(inv, nil)
f.UseStore(t.Context(), st)
if _, err := f.Hold(t.Context(), facility.Hold{Domain: "pdu:p1", Reason: "breaker", By: "ilker"}); err != nil {
t.Fatal(err)
}
if _, err := f.SetLabels(t.Context(), "gpu01", map[string]string{"llmgw.io/rack": "r9"}, nil, "ilker"); err != nil {
t.Fatal(err)
}
// A restart: a new inventory and facility over the same store.
inv2 := inventory.NewStore()
f2 := NewFacility(inv2, nil)
f2.SetPolicy(facility.Policy{Margin: 0.1})
f2.UseStore(t.Context(), st)
inv2.Upsert(inventory.Node{ID: "gpu01", State: inventory.Ready})
if n, _ := inv2.Get("gpu01"); n.Labels["llmgw.io/rack"] != "r9" {
t.Errorf("the label set before the restart is gone: %v", n.Labels)
}
pol := inv2.Facility()
if h, ok := pol.Holds["pdu:p1"]; !ok || h.Reason != "breaker" || pol.Margin != 0.1 {
t.Errorf("after the restart: holds %v, margin %v", pol.Holds, pol.Margin)
}
}
func TestAMisspeltLocationLabelIsRefused(t *testing.T) {
f := NewFacility(inventory.NewStore(), nil)
_, err := f.SetLabels(t.Context(), "n1", map[string]string{"llmgw.io/rak": "r1"}, nil, "")
if err == nil || !strings.Contains(err.Error(), "llmgw.io/site, room, row, rack or pdu") {
t.Errorf("got %v", err)
}
if _, err := f.SetLabels(t.Context(), "n1", map[string]string{"team": "nlp"}, nil, ""); err != nil {
t.Errorf("a free label was refused: %v", err)
}
}
+176
View File
@@ -0,0 +1,176 @@
package inventory
import (
"maps"
"slices"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
// Location is where the node is, from its labels.
func (n Node) Location() facility.Location { return facility.LocationOf(n.Labels) }
// Domains are the failure domains the node is in, largest first.
func (n Node) Domains() []facility.Domain { return n.Location().Domains() }
// mergeLabels lays the operator's labels over the agent's. A copy, so
// neither is written through.
func mergeLabels(agent, operator map[string]string) map[string]string {
if len(operator) == 0 {
return maps.Clone(agent)
}
out := make(map[string]string, len(agent)+len(operator))
maps.Copy(out, agent)
maps.Copy(out, operator)
return out
}
// SetFacility puts the facility's holds and power budgets in force for
// every placement from now on.
func (s *Store) SetFacility(p facility.Policy) {
s.mu.Lock()
defer s.mu.Unlock()
s.facility = p.Clone()
}
// Facility is the policy in force; a copy, so a caller may change it.
func (s *Store) Facility() facility.Policy {
s.mu.RLock()
defer s.mu.RUnlock()
return s.facility.Clone()
}
// SetOperatorLabels replaces the labels set through the control plane for a
// node, which may not have registered yet: they are laid over its agent's
// when it does.
func (s *Store) SetOperatorLabels(id string, labels map[string]string) {
s.mu.Lock()
defer s.mu.Unlock()
if s.operatorLabels == nil {
s.operatorLabels = map[string]map[string]string{}
}
if len(labels) == 0 {
delete(s.operatorLabels, id)
} else {
s.operatorLabels[id] = maps.Clone(labels)
}
if n, ok := s.nodes[id]; ok {
n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[id])
}
}
// OperatorLabels are the labels set through the control plane for a node.
func (s *Store) OperatorLabels(id string) map[string]string {
s.mu.RLock()
defer s.mu.RUnlock()
return maps.Clone(s.operatorLabels[id])
}
// PowerUse is one domain's draw as placement counts it.
type PowerUse struct {
Domain facility.Domain
// Nodes are the domain's nodes, sorted.
Nodes []string
// Reserved is what the replicas placed in the domain are estimated to
// draw, with each node's own draw and the domain's other equipment.
Reserved float64
// Measured is what the nodes' sensors read, with the same additions,
// for the nodes that read anything; the others count what is reserved.
Measured float64
// Used is what placement counts against the cap: per node, the larger
// of reserved and measured, so an estimate that was too low is
// corrected by the room it stops offering rather than by a breaker.
Used float64
// Limit is the cap less the margin; zero is no cap.
Limit float64
Budget facility.Budget
}
// PowerUse is every domain any node is in, or that has a budget, by domain.
func (s *Store) PowerUse(now time.Time) map[facility.Domain]PowerUse {
s.mu.RLock()
defer s.mu.RUnlock()
pol := s.facility
out := map[facility.Domain]PowerUse{}
for d, b := range pol.Budgets {
out[d] = PowerUse{Domain: d, Budget: b, Limit: pol.Limit(d),
Reserved: b.OtherWatts, Measured: b.OtherWatts, Used: b.OtherWatts}
}
ids := slices.Sorted(maps.Keys(s.nodes))
for _, id := range ids {
n := s.nodes[id]
reserved := 0.0
for _, a := range s.allocs[id] {
reserved += AllocationWatts(*n, a, pol)
}
measured, known := MeasuredWatts(*n, now, pol.Fresh())
used := reserved
if known && measured > used {
used = measured
}
if !known {
measured = reserved
}
for _, d := range n.Domains() {
u, ok := out[d]
if !ok {
u = PowerUse{Domain: d}
}
u.Nodes = append(u.Nodes, id)
u.Reserved += pol.NodeWatts + reserved
u.Measured += pol.NodeWatts + measured
u.Used += pol.NodeWatts + used
out[d] = u
}
}
return out
}
// AllocationWatts is what one shard is estimated to draw: what its profile
// declared, or else each of its cards at the power limit the card reports
// (the policy's figure for a card that reports none) and each of its cores
// at the policy's figure.
func AllocationWatts(n Node, a Allocation, pol facility.Policy) float64 {
if a.PowerW > 0 {
return a.PowerW
}
w := float64(a.CPUCores) * pol.CPUCoreWatts
for _, i := range a.GPUIndices {
w += GPUWatts(n, i, pol)
}
return w
}
// GPUWatts is one card's estimated draw: the power limit it reports, else
// the policy's figure for a card.
func GPUWatts(n Node, index int, pol facility.Policy) float64 {
if n.Load != nil {
for _, g := range n.Load.GPUs {
if g.Index == index && g.PowerLimitW > 0 {
return g.PowerLimitW
}
}
}
return pol.GPUWatts
}
// MeasuredWatts is what the node's sensors read at its last heartbeat: its
// cards and its CPU packages. Known is false when the reading is older than
// fresh, or the node has no power sensor at all, which a VM never has.
func MeasuredWatts(n Node, now time.Time, fresh time.Duration) (w float64, known bool) {
if n.Load == nil || now.Sub(n.Load.At) > fresh {
return 0, false
}
for _, g := range n.Load.GPUs {
if g.PowerW > 0 {
w += g.PowerW
known = true
}
}
if n.Load.CPUPowerW > 0 {
w += n.Load.CPUPowerW
known = true
}
return w, known
}
@@ -0,0 +1,79 @@
package inventory
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
func TestOperatorLabelsWinOverTheAgentsAndSurviveItsHeartbeats(t *testing.T) {
s := NewStore()
// Set before the node registers: labelling a rack's machines comes first.
s.SetOperatorLabels("n1", map[string]string{"llmgw.io/rack": "r2"})
s.Upsert(Node{ID: "n1", Labels: map[string]string{"llmgw.io/rack": "r1", "role": "gpu"}})
n, _ := s.Get("n1")
if n.Labels["llmgw.io/rack"] != "r2" || n.Labels["role"] != "gpu" {
t.Fatalf("labels %v: the operator's rack over the agent's, the agent's role kept", n.Labels)
}
// A heartbeat hands back the node it read.
s.Upsert(n)
s.Upsert(n)
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "r2" || n.AgentLabels["llmgw.io/rack"] != "r1" {
t.Fatalf("after heartbeats: labels %v, agent's %v", n.Labels, n.AgentLabels)
}
s.SetOperatorLabels("n1", nil)
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "r1" {
t.Errorf("the operator's label removed, rack is %q; want the agent's r1 back", n.Labels["llmgw.io/rack"])
}
}
func TestAnOperatorLabelOnANodeWithNoAgentLabelsIsNotTakenForTheAgents(t *testing.T) {
s := NewStore()
s.Upsert(Node{ID: "n1"})
s.SetOperatorLabels("n1", map[string]string{"llmgw.io/rack": "r2"})
n, _ := s.Get("n1")
s.Upsert(n)
s.SetOperatorLabels("n1", nil)
if n, _ := s.Get("n1"); n.Labels["llmgw.io/rack"] != "" {
t.Errorf("removed, but the node still carries rack %q", n.Labels["llmgw.io/rack"])
}
}
func TestADomainsPowerIsTheLargerOfReservedAndMeasuredNodeByNode(t *testing.T) {
s := NewStore()
gpus := []discovery.GPU{{Index: 0}, {Index: 1}}
rack := map[string]string{"llmgw.io/rack": "r1"}
now := time.Now()
// a: two cards placed at a 700 W limit, drawing 300 each: 1400 reserved.
s.Upsert(Node{ID: "a", Labels: rack, GPUs: gpus})
s.Upsert(Node{ID: "b", Labels: rack, GPUs: gpus})
setLoad(s, "a", &Load{At: now, GPUs: []GPUSensor{{Index: 0, PowerLimitW: 700, PowerW: 300}, {Index: 1, PowerLimitW: 700, PowerW: 300}}})
// b: nothing placed, but drawing 500 W.
setLoad(s, "b", &Load{At: now, GPUs: []GPUSensor{{Index: 0, PowerW: 250}, {Index: 1, PowerW: 250}}})
s.Allocate("a", Allocation{ShardID: "x", GPUIndices: []int{0, 1}})
s.SetFacility(facility.Policy{NodeWatts: 100, Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000, OtherWatts: 50}}})
u := s.PowerUse(now)["rack:r1"]
if u.Reserved != 50+100+1400+100+0 {
t.Errorf("reserved %v", u.Reserved)
}
if u.Measured != 50+100+600+100+500 {
t.Errorf("measured %v", u.Measured)
}
if u.Used != 50+100+1400+100+500 {
t.Errorf("used %v: a's reservation and b's reading", u.Used)
}
if len(u.Nodes) != 2 || u.Limit != 5000 {
t.Errorf("nodes %v, limit %v", u.Nodes, u.Limit)
}
}
func setLoad(s *Store, id string, l *Load) {
n, _ := s.Get(id)
n.Load = l
s.Upsert(n)
}
+33 -4
View File
@@ -11,6 +11,7 @@ import (
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)
// NodeState is where a node stands.
@@ -71,10 +72,17 @@ type FabricAttachment struct {
// Node is one machine in the cluster.
type Node struct {
ID string `json:"id"`
Hostname string `json:"hostname"`
MgmtAddr string `json:"mgmt_addr"`
Labels map[string]string `json:"labels,omitempty"`
ID string `json:"id"`
Hostname string `json:"hostname"`
MgmtAddr string `json:"mgmt_addr"`
// Labels are what placement matches: the agent's own, from agent.yaml,
// with any an operator set through the control plane over them.
Labels map[string]string `json:"labels,omitempty"`
// AgentLabels are the labels the agent reported, kept apart so an
// operator's label removed brings the agent's value back. Nil on a node
// built anywhere but a registration; Upsert then takes Labels as the
// agent's.
AgentLabels map[string]string `json:"agent_labels,omitempty"`
// MemoryTotalMB is the machine's RAM. Zero means the agent did not report
// it, which is treated as no constraint — the same as every node before
@@ -461,6 +469,10 @@ type Allocation struct {
// CPUCores is the cores this shard was given to compute on, for a shard
// with no GPU; zero for one that shares whatever is free.
CPUCores int
// PowerW is the draw this shard's profile declared; zero derives it from
// the node's cards and the cores, which is what most profiles leave to
// placement.
PowerW float64
}
// Store holds the cluster inventory. It is safe for concurrent use.
@@ -473,6 +485,11 @@ type Store struct {
smChanges []SMChange
// scans are the latest ibnetdiscover of each physical fabric, by key.
scans map[string]FabricScan
// facility is the holds and power budgets placement reads.
facility facility.Policy
// operatorLabels are labels set through the control plane, by node,
// laid over what each agent reports.
operatorLabels map[string]map[string]string
}
// NewStore returns an empty inventory.
@@ -494,6 +511,7 @@ func (s *Store) Clone() *Store {
for id, list := range s.allocs {
c.allocs[id] = slices.Clone(list)
}
c.facility = s.facility.Clone()
return c
}
@@ -564,6 +582,17 @@ func (s *Store) Upsert(n Node) {
n.State = prev.State
}
}
// A registration carries the agent's labels in Labels; a heartbeat hands
// back the node it read, whose Labels already have the operator's in
// them, and its AgentLabels set. Never nil after this, so the second case
// is always told from the first.
if n.AgentLabels == nil {
n.AgentLabels = maps.Clone(n.Labels)
if n.AgentLabels == nil {
n.AgentLabels = map[string]string{}
}
}
n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[n.ID])
n.LastSeen = time.Now()
s.nodes[n.ID] = &n
}
+193
View File
@@ -0,0 +1,193 @@
package placement
import (
"fmt"
"slices"
"sort"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// declaredShardWatts is the profile's declared draw for one node's shard.
func (p Profile) declaredShardWatts() float64 {
if p.PowerWatts <= 0 {
return 0
}
return p.PowerWatts / float64(max(p.Nodes, 1))
}
// shardWatts is what one shard of this profile is estimated to draw on a
// candidate: the declared figure, or its cards at the highest power limit
// among the node's free ones of the class (the cards are not chosen yet, and
// guessing low is the mistake that trips a breaker), or its cores.
func (s *Scheduler) shardWatts(p Profile, c candidate) float64 {
if w := p.declaredShardWatts(); w > 0 {
return w
}
pol := s.inv.Facility()
if p.CPUOnly() {
return float64(p.CPUCores) * pol.CPUCoreWatts
}
per := 0.0
for _, g := range c.free {
per = max(per, inventory.GPUWatts(c.node, g.Index, pol))
}
if per == 0 {
per = pol.GPUWatts
}
return per * float64(p.GPUsPerNode)
}
// budget is the power budgets as one placement decision spends them: the
// domains' use when it began, and what the shards it has chosen so far add.
type budget struct {
pol facility.Policy
use map[facility.Domain]inventory.PowerUse
added map[facility.Domain]float64
}
// newBudget reads the domains' use, once per decision. Without budgets there
// is nothing to read and every check passes.
func (s *Scheduler) newBudget() *budget {
pol := s.inv.Facility()
b := &budget{pol: pol, added: map[facility.Domain]float64{}}
if len(pol.Budgets) > 0 {
b.use = s.inv.PowerUse(s.clock())
}
return b
}
// check says why a shard drawing need watts on a node would take one of the
// node's domains past its limit; empty when it fits them all. A shard whose
// draw is not known (no declared figure, no card limit, no policy figure)
// needs nothing: missing data never blocks placement.
func (b *budget) check(n inventory.Node, need float64) string {
if b.use == nil {
return ""
}
for _, d := range n.Domains() {
limit := b.pol.Limit(d)
if limit <= 0 {
continue
}
used := b.use[d].Used + b.added[d]
if used+need > limit {
capW := b.pol.Budgets[d].CapWatts
of := facility.KW(limit) + " cap"
if limit < capW {
of = fmt.Sprintf("%s limit (%s cap less %.0f%%)", facility.KW(limit), facility.KW(capW), 100-100*limit/capW)
}
return fmt.Sprintf("%s: %s of a %s in use, the shard needs ~%s",
d.Human(), facility.KW(used), of, facility.KW(need))
}
}
return ""
}
// add counts a chosen shard against the node's domains.
func (b *budget) add(n inventory.Node, need float64) {
for _, d := range n.Domains() {
b.added[d] += need
}
}
// clock is the scheduler's time, for the freshness of power readings.
func (s *Scheduler) clock() time.Time {
if s.now != nil {
return s.now()
}
return time.Now()
}
// heldReason is a hold as a rejection says it.
func heldReason(h facility.Hold) string {
why := h.Domain.Human() + " is held"
if h.Reason != "" {
why += ": " + h.Reason
}
return why
}
// spreadLevels are the domains replicas of one model are kept apart in:
// the rack, which a top-of-rack switch or a door left open takes out whole,
// and the PDU, which a breaker does.
var spreadLevels = []facility.Level{facility.Rack, facility.PDU}
// spreadAcrossDomains moves to the front the candidates sharing the fewest
// racks and PDUs with the nodes the model already runs on, keeping the order
// given among equals.
//
// A preference and never a filter, as spreading across nodes is: a rack
// holding a model's only free cards still takes its second replica, which
// survives an engine crash if not a breaker. Nodes with no location share
// nothing and lose nothing, so a cluster without labels places as before.
func (s *Scheduler) spreadAcrossDomains(candidates []candidate, avoid []string) {
if len(avoid) == 0 {
return
}
taken := map[facility.Domain]bool{}
for _, id := range avoid {
if n, ok := s.inv.Get(id); ok {
for _, d := range n.Domains() {
if slices.Contains(spreadLevels, d.Level()) {
taken[d] = true
}
}
}
}
if len(taken) == 0 {
return
}
shared := func(c candidate) int {
k := 0
for _, d := range c.node.Domains() {
if taken[d] {
k++
}
}
return k
}
sort.SliceStable(candidates, func(i, j int) bool { return shared(candidates[i]) < shared(candidates[j]) })
}
// withinGroups divides a multi-node replica's candidates by the domain its
// profile keeps it within, largest first, and rejects the nodes that do not
// say which they are in. Without the constraint it is all of them, as one.
func withinGroups(p Profile, pool string, cands []candidate, rejections *[]Rejection) [][]candidate {
if p.KeepWithin == "" {
return [][]candidate{cands}
}
by := map[facility.Domain][]candidate{}
for _, c := range cands {
d := c.node.Location().At(p.KeepWithin)
if d == "" {
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, Node: c.node.ID,
Reason: fmt.Sprintf("has no %s label, and the profile keeps its nodes within one %s",
p.KeepWithin.Label(), p.KeepWithin)})
continue
}
by[d] = append(by[d], c)
}
keys := make([]facility.Domain, 0, len(by))
for d := range by {
keys = append(keys, d)
}
sort.Slice(keys, func(i, j int) bool {
if len(by[keys[i]]) != len(by[keys[j]]) {
return len(by[keys[i]]) > len(by[keys[j]])
}
return keys[i] < keys[j]
})
out := make([][]candidate, 0, len(keys))
for _, d := range keys {
if len(by[d]) < p.Nodes {
*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool,
Reason: fmt.Sprintf("needs %d nodes within one %s, %s has %d", p.Nodes, p.KeepWithin, d.Human(), len(by[d]))})
continue
}
out = append(out, by[d])
}
return out
}
+325
View File
@@ -0,0 +1,325 @@
package placement
import (
"fmt"
"math/rand/v2"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
// at puts a node in a rack, and on PDUs when given.
func at(n inventory.Node, rack string, pdus ...string) inventory.Node {
if n.Labels == nil {
n.Labels = map[string]string{}
}
n.Labels[facility.Rack.Label()] = rack
if len(pdus) > 0 {
n.Labels[facility.PDU.Label()] = strings.Join(pdus, ",")
}
return n
}
// withPowerLimit has a node's cards report a power limit, and draw, as a
// heartbeat would.
func withPowerLimit(n inventory.Node, limit, draw float64) inventory.Node {
l := &inventory.Load{At: time.Now()}
for _, g := range n.GPUs {
l.GPUs = append(l.GPUs, inventory.GPUSensor{Index: g.Index, PowerLimitW: limit, PowerW: draw})
}
n.Load = l
return n
}
func onePerNode(name string, gpus int) Profile {
return Profile{Name: name, GPUClass: "h100-80g", GPUsPerNode: gpus, Nodes: 1}
}
func TestNothingIsPlacedInAHeldRackAndTheRejectionSaysWhy(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8), "r1"), at(node("b1", "h100-80g", 8), "r2"))
st.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{
"rack:r2": {Domain: "rack:r2", Reason: "PDU A replacement"},
}})
st.Allocate("a1", inventory.Allocation{ShardID: "x", GPUIndices: []int{0, 1, 2, 3, 4, 5, 6, 7}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
if res.Placement != nil {
t.Fatalf("placed on %s, in a held rack", res.Placement.Nodes[0].NodeID)
}
if !strings.Contains(res.Explain(), "b1: rack r2 is held: PDU A replacement") {
t.Errorf("the rejection does not name the hold:\n%s", res.Explain())
}
}
func TestAHeldPDUHoldsEveryRackOnIt(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8), "r1", "p1", "p2"), at(node("b1", "h100-80g", 8), "r2", "p3"))
st.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{"pdu:p2": {Domain: "pdu:p2"}}})
res := New(st).Place([]Profile{onePerNode("tp8", 8)}, nil)
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "b1" {
t.Fatalf("want b1, the node on no held PDU:\n%s", res.Explain())
}
}
func TestASecondReplicaGoesToAnotherRackBeforeAFullerNodeInTheSameOne(t *testing.T) {
// Bin packing alone would choose a2, the partly used node beside the
// first replica; a rack apart is worth an emptier node.
st := store(
at(node("a1", "h100-80g", 8), "r1"),
at(node("a2", "h100-80g", 8), "r1"),
at(node("b1", "h100-80g", 8), "r2"),
)
st.Allocate("a2", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
if res.Placement == nil {
t.Fatalf("not placed:\n%s", res.Explain())
}
if got := res.Placement.Nodes[0].NodeID; got != "b1" {
t.Errorf("placed on %s, in the rack of the first replica; want b1 in another", got)
}
}
func TestReplicasAreSpreadAcrossPDUsAsWellAsRacks(t *testing.T) {
st := store(
at(node("a1", "h100-80g", 8), "r1", "p1"),
at(node("b1", "h100-80g", 8), "r2", "p1"),
at(node("c1", "h100-80g", 8), "r3", "p2"),
)
st.Allocate("b1", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "c1" {
t.Fatalf("want c1, on another rack and another PDU:\n%s", res.Explain())
}
}
func TestSpreadingAcrossRacksNeverLeavesAReplicaUnplaced(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8), "r1"), at(node("a2", "h100-80g", 8), "r1"))
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, []string{"a1"})
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "a2" {
t.Fatalf("the rack's other node should still be used:\n%s", res.Explain())
}
}
func TestNodesWithoutALocationPlaceAsTheyAlwaysDid(t *testing.T) {
// Bin packing picks the partly used node; nothing about racks changes it.
st := store(node("n1", "h100-80g", 8), node("n2", "h100-80g", 8))
st.Allocate("n2", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1}})
st.SetFacility(facility.Policy{GPUWatts: 700, Margin: 0.1})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "n2" {
t.Fatalf("want n2, the bin-packed choice:\n%s", res.Explain())
}
}
func TestAReplicaThatWouldTakeARackPastItsCapGoesToAnotherRack(t *testing.T) {
// Rack r1: 8 cards at 700 W, four of them placed: 2.8 kW used of a
// 4.5 kW limit (5 kW less 10%). Four more need 2.8 kW, which does not fit.
st := store(
withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 0),
withPowerLimit(at(node("b1", "h100-80g", 8), "r2"), 700, 0),
)
st.Allocate("a1", inventory.Allocation{ShardID: "other", GPUIndices: []int{0, 1, 2, 3}})
st.SetFacility(facility.Policy{Margin: 0.1, Budgets: map[facility.Domain]facility.Budget{
"rack:r1": {CapWatts: 5000},
}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
if res.Placement == nil || res.Placement.Nodes[0].NodeID != "b1" {
t.Fatalf("want b1, in the rack with room:\n%s", res.Explain())
}
want := "a1: rack r1: 2.8 kW of a 4.5 kW limit (5.0 kW cap less 10%) in use, the shard needs ~2.8 kW"
if !strings.Contains(res.Explain(), want) {
t.Errorf("the rejection should give the figures, %q:\n%s", want, res.Explain())
}
}
func TestAProfilesDeclaredDrawIsWhatIsCounted(t *testing.T) {
st := store(withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 0))
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 2000}}})
derived := onePerNode("tp4", 4) // 4 × 700 W: too much for 2 kW
if res := New(st).Place([]Profile{derived}, nil); res.Placement != nil {
t.Fatalf("2.8 kW was placed under a 2 kW cap")
}
declared := derived
declared.PowerWatts = 1500
res := New(st).Place([]Profile{declared}, nil)
if res.Placement == nil {
t.Fatalf("a profile declaring 1.5 kW was refused under a 2 kW cap:\n%s", res.Explain())
}
if res.Placement.Nodes[0].PowerW != 1500 {
t.Errorf("the shard carries %v W to record, want the declared 1500", res.Placement.Nodes[0].PowerW)
}
}
func TestWhatTheSensorsReadCountsWhenItIsMoreThanWasReserved(t *testing.T) {
// Nothing placed, but the cards draw 400 W each: 3.2 kW measured, and
// another 2.8 kW would pass a 5 kW cap only on the reservations.
st := store(withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 400))
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000}}})
res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil)
if res.Placement != nil {
t.Fatalf("placed against reservations alone, with 3.2 kW measured")
}
if !strings.Contains(res.Explain(), "3.2 kW of a 5.0 kW cap in use") {
t.Errorf("the measured draw should be what is in use:\n%s", res.Explain())
}
}
func TestAStaleReadingIsNotCounted(t *testing.T) {
n := withPowerLimit(at(node("a1", "h100-80g", 8), "r1"), 700, 400)
n.Load.At = time.Now().Add(-10 * time.Minute)
st := store(n)
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 5000}}})
if res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil); res.Placement == nil {
t.Fatalf("a ten-minute-old reading kept a replica out:\n%s", res.Explain())
}
}
func TestADrawNobodyKnowsNeverBlocksPlacement(t *testing.T) {
// A cap, but no card reports a limit and no figure is configured: the
// shard's draw is unknown, which is not a reason to refuse it.
st := store(at(node("a1", "h100-80g", 8), "r1"))
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 100}}})
if res := New(st).Place([]Profile{onePerNode("tp8", 8)}, nil); res.Placement == nil {
t.Fatalf("an unknown draw was refused:\n%s", res.Explain())
}
}
func TestTheOtherEquipmentInARackCountsAgainstItsCap(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8), "r1"))
st.SetFacility(facility.Policy{GPUWatts: 500, Budgets: map[facility.Domain]facility.Budget{
"rack:r1": {CapWatts: 5000, OtherWatts: 3500},
}})
if res := New(st).Place([]Profile{onePerNode("tp4", 4)}, nil); res.Placement != nil {
t.Fatal("2 kW was placed where switches already take 3.5 of 5")
}
}
func TestAMultiNodeReplicasShardsAreCountedTogetherAgainstTheirRack(t *testing.T) {
// Each node alone fits r1's budget; two together do not, so the second
// shard goes to the rack with room — on the same fabric, as always.
st := store(
withPowerLimit(at(node("a1", "h100-80g", 8, "ib"), "r1"), 500, 0),
withPowerLimit(at(node("a2", "h100-80g", 8, "ib"), "r1"), 500, 0),
withPowerLimit(at(node("b1", "h100-80g", 8, "ib"), "r2"), 500, 0),
)
st.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}})
res := New(st).Place([]Profile{{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2}}, nil)
if res.Placement == nil {
t.Fatalf("not placed:\n%s", res.Explain())
}
racks := map[string]int{}
for _, a := range res.Placement.Nodes {
n, _ := st.Get(a.NodeID)
racks[n.Location().Rack]++
}
if racks["r1"] != 1 || racks["r2"] != 1 {
t.Errorf("shards by rack %v: two in r1 draw 8 kW against a 6 kW cap", racks)
}
if !strings.Contains(res.Explain(), "with this replica's other shards") {
t.Errorf("the node passed over should say why:\n%s", res.Explain())
}
}
func TestAProfileKeptWithinARackNeverSpansTwo(t *testing.T) {
// The emptiest-first order would take a1 and b1; the profile wants one
// rack, and only r2 has two nodes.
st := store(
at(node("a1", "h100-80g", 8, "ib"), "r1"),
at(node("b1", "h100-80g", 8, "ib"), "r2"),
at(node("b2", "h100-80g", 8, "ib"), "r2"),
node("x1", "h100-80g", 8, "ib"),
)
p := Profile{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, KeepWithin: facility.Rack}
res := New(st).Place([]Profile{p}, nil)
if res.Placement == nil {
t.Fatalf("not placed:\n%s", res.Explain())
}
for _, a := range res.Placement.Nodes {
if a.NodeID != "b1" && a.NodeID != "b2" {
t.Errorf("a shard on %s, outside rack r2", a.NodeID)
}
}
explain := res.Explain()
if !strings.Contains(explain, "x1: has no llmgw.io/rack label, and the profile keeps its nodes within one rack") {
t.Errorf("the unlabelled node should be named:\n%s", explain)
}
if !strings.Contains(explain, "needs 2 nodes within one rack, rack r1 has 1") {
t.Errorf("the rack too small should be named:\n%s", explain)
}
}
func TestAProfileKeptWithinARackIsNotPlacedAcrossRacks(t *testing.T) {
st := store(at(node("a1", "h100-80g", 8, "ib"), "r1"), at(node("b1", "h100-80g", 8, "ib"), "r2"))
p := Profile{Name: "2x8", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, KeepWithin: facility.Rack}
if res := New(st).Place([]Profile{p}, nil); res.Placement != nil {
t.Fatal("a replica kept within one rack was placed across two")
}
}
// TestNoCapIsEverExceeded fills random clusters until nothing more fits, and
// checks every capped domain stayed within its limit — and that a cluster
// whose draw nobody knows is filled as if it had no caps at all.
func TestNoCapIsEverExceeded(t *testing.T) {
rng := rand.New(rand.NewPCG(3, 4))
for iter := range 200 {
st := inventory.NewStore()
known := rng.IntN(4) != 0
cards := 0
for i := range 2 + rng.IntN(6) {
n := at(node(fmt.Sprintf("n%d", i), "h100-80g", 1+rng.IntN(8), "ib"),
fmt.Sprintf("r%d", rng.IntN(3)), fmt.Sprintf("p%d", rng.IntN(2)))
if known {
n = withPowerLimit(n, float64(300+rng.IntN(500)), 0)
}
cards += len(n.GPUs)
st.Upsert(n)
}
pol := facility.Policy{Margin: float64(rng.IntN(20)) / 100, Budgets: map[facility.Domain]facility.Budget{}}
for _, d := range []facility.Domain{"rack:r0", "rack:r1", "rack:r2", "pdu:p0", "pdu:p1"} {
if rng.IntN(2) == 0 {
pol.Budgets[d] = facility.Budget{CapWatts: float64(1000 + rng.IntN(8000))}
}
}
st.SetFacility(pol)
for k := range 60 {
p := Profile{Name: "gen", GPUClass: "h100-80g", GPUsPerNode: 1, Nodes: 1 + rng.IntN(2)}
res := New(st).Place([]Profile{p}, nil)
if res.Placement == nil {
continue
}
for _, a := range res.Placement.Nodes {
st.Allocate(a.NodeID, inventory.Allocation{ShardID: fmt.Sprintf("s%d-%d", k, a.Rank), GPUIndices: a.GPUIndices})
}
}
for d, u := range st.PowerUse(time.Now()) {
if u.Limit > 0 && u.Used > u.Limit+1e-6 {
t.Fatalf("iteration %d: %s uses %.0f W of a %.0f W limit", iter, d, u.Used, u.Limit)
}
}
if !known {
free := 0
for _, n := range st.Nodes() {
free += len(st.FreeGPUs(n.ID))
}
if free != 0 {
t.Fatalf("iteration %d: %d of %d cards left free with no draw known", iter, free, cards)
}
}
}
}
+85 -14
View File
@@ -16,8 +16,10 @@ import (
"slices"
"sort"
"strings"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
@@ -45,6 +47,13 @@ type Profile struct {
// Source is the model's weights, as the engine would be given it. Used to
// prefer nodes that already hold them; empty asks for no preference.
Source string
// PowerWatts is what a replica of this profile draws, split evenly over
// its nodes. Zero derives it from the cards' power limits and the cores.
PowerWatts float64
// KeepWithin keeps a multi-node replica's nodes inside one domain of
// this level, rack or row, where the fabric or the cabling needs them
// close. Empty is no constraint.
KeepWithin facility.Level
}
// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a
@@ -81,6 +90,9 @@ type NodeAssignment struct {
GPUIndices []int
// HCAs are the adapters of the chosen fabric only.
HCAs []string
// PowerW is the draw the profile declared for this shard, to be recorded
// with its allocation; zero when the profile declared none.
PowerW float64
}
// Placement is a decision: which profile, which fabric, which nodes.
@@ -139,6 +151,9 @@ func (r Result) Explain() string {
// Scheduler places replicas onto an inventory.
type Scheduler struct {
inv *inventory.Store
// now is the clock power readings are judged fresh by; nil is the wall
// clock.
now func() time.Time
}
// New builds a scheduler over an inventory.
@@ -148,6 +163,11 @@ func New(inv *inventory.Store) *Scheduler { return &Scheduler{inv: inv} }
//
// avoid lists nodes already hosting this model, so replicas of one model spread
// across failure domains instead of stacking on one machine.
//
// The rejections of the profile that was placed are kept with the placement,
// not only those of the profiles before it: a replica placed in the second
// rack because the first was held or at its power cap is placed somewhere
// nobody expected, and explain-placement is where the operator asks why.
func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
var res Result
@@ -168,6 +188,7 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
if p.Nodes == 1 {
if pl, rej := s.placeSingleNode(p, avoid); pl != nil {
res.Placement = pl
res.Rejections = append(res.Rejections, rej...)
return res
} else {
res.Rejections = append(res.Rejections, rej...)
@@ -176,6 +197,7 @@ func (s *Scheduler) Place(profiles []Profile, avoid []string) Result {
}
if pl, rej := s.placeMultiNode(p, avoid); pl != nil {
res.Placement = pl
res.Rejections = append(res.Rejections, rej...)
return res
} else {
res.Rejections = append(res.Rejections, rej...)
@@ -212,17 +234,21 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
len(s.inv.Allocations(candidates[j].node.ID))
})
preferNodesHoldingWeights(p.Source, candidates)
s.spreadAcrossDomains(candidates, avoid)
return &Placement{
Profile: p,
Pool: "none/cpu",
Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0}},
Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0, PowerW: p.declaredShardWatts()}},
}, rejections
}
// Last, so it takes precedence: moving tens of gigabytes costs more than an
// imperfectly packed node, and with isolated fabrics it may cost a
// cross-fabric copy over the management network.
// After packing, so it takes precedence: moving tens of gigabytes costs
// more than an imperfectly packed node, and with isolated fabrics it may
// cost a cross-fabric copy over the management network.
preferNodesHoldingWeights(p.Source, candidates)
// And racks over weights: a second replica in the rack of the first is
// one breaker away from none, which no copy saved is worth.
s.spreadAcrossDomains(candidates, avoid)
best := candidates[0]
gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode, nil)
@@ -237,7 +263,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re
return &Placement{
Profile: p,
Pool: "none/" + p.classLabel(),
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}},
Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus, PowerW: p.declaredShardWatts()}},
}, rejections
}
@@ -273,8 +299,10 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej
if group.key != "" && group.split {
pool = fabric + "@" + shortKey(group.key) + "/" + p.classLabel()
}
if pl := s.placeOnFabric(p, fabric, pool, group.nodes, &rejections); pl != nil {
return pl, rejections
for _, within := range withinGroups(p, pool, group.nodes, &rejections) {
if pl := s.placeOnFabric(p, fabric, pool, within, &rejections); pl != nil {
return pl, rejections
}
}
}
}
@@ -325,8 +353,10 @@ func (s *Scheduler) placeOnNVLink(p Profile, avoid []string, rejections *[]Rejec
})
for _, d := range keys {
pool := NVLinkFabric + shortDomain(d) + "/" + p.classLabel()
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, by[d], rejections); pl != nil {
return pl
for _, within := range withinGroups(p, pool, by[d], rejections) {
if pl := s.placeOnFabric(p, NVLinkFabric+d, pool, within, rejections); pl != nil {
return pl
}
}
}
return nil
@@ -416,8 +446,22 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c
// together, a degraded cable last.
candidates = byLeaf(candidates, hints(s.inv))
// Each node passed the power budgets on its own; together, two shards in
// one rack may not. Nodes whose shard would take a domain past its cap
// with the shards already chosen are passed over for the next.
budget := s.newBudget()
assignments := make([]NodeAssignment, 0, p.Nodes)
for rank, c := range candidates[:p.Nodes] {
for _, c := range candidates {
if len(assignments) == p.Nodes {
break
}
if why := budget.check(c.node, s.shardWatts(p, c)); why != "" {
*rejections = append(*rejections, Rejection{
Profile: p.Name, Pool: pool, Node: c.node.ID,
Reason: why + ", with this replica's other shards",
})
continue
}
gpus := s.pickGPUs(c.node, c.free, p.GPUsPerNode, c.node.HCAsFor(fabric))
if gpus == nil {
*rejections = append(*rejections, Rejection{
@@ -426,14 +470,23 @@ func (s *Scheduler) placeOnFabric(p Profile, fabric, pool string, candidates []c
})
return nil
}
budget.add(c.node, s.shardWatts(p, c))
assignments = append(assignments, NodeAssignment{
NodeID: c.node.ID,
Rank: rank,
Rank: len(assignments),
GPUIndices: gpus,
// Only this fabric's adapters, never the node's others.
HCAs: c.node.HCAsFor(fabric),
HCAs: c.node.HCAsFor(fabric),
PowerW: p.declaredShardWatts(),
})
}
if len(assignments) < p.Nodes {
*rejections = append(*rejections, Rejection{
Profile: p.Name, Pool: pool,
Reason: fmt.Sprintf("needs %d nodes, only %d fit within the power budgets together", p.Nodes, len(assignments)),
})
return nil
}
return &Placement{Profile: p, Fabric: fabric, Pool: pool, Nodes: assignments}
}
@@ -508,6 +561,8 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
}
pool += "/" + p.classLabel()
pol := s.inv.Facility()
budget := s.newBudget()
var out []candidate
for _, n := range s.inv.Nodes() {
reject := func(reason string) {
@@ -518,6 +573,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
reject("node is " + n.State.String())
continue
}
// A domain held for maintenance takes nothing new, as a cordon of
// every node in it would, and says which hold and why.
if h, held := pol.HeldIn(n.Domains()); held {
reject(heldReason(h))
continue
}
if slices.Contains(avoid, n.ID) {
reject("already hosts this model; spreading across failure domains")
continue
@@ -608,7 +669,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
if p.CPUOnly() {
// Every ready node can run it; there is no card to be free.
out = append(out, candidate{node: n})
c := candidate{node: n}
if why := budget.check(n, s.shardWatts(p, c)); why != "" {
reject(why)
continue
}
out = append(out, c)
continue
}
@@ -624,7 +690,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje
reject(fmt.Sprintf("has %d free %s GPUs, needs %d", len(free), p.GPUClass, p.GPUsPerNode))
continue
}
out = append(out, candidate{node: n, free: free})
c := candidate{node: n, free: free}
if why := budget.check(n, s.shardWatts(p, c)); why != "" {
reject(why)
continue
}
out = append(out, c)
}
return out
}
+1
View File
@@ -145,6 +145,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
MemoryGB: r.profileMemory(rep.Model, rep.Profile),
CPUCores: r.adoptedCores(rep.Model, rep.Profile),
PowerW: r.profileWatts(rep.Model, rep.Profile),
})
// The endpoint comes from the leader, which may be in any of the
// groups — the rank 0 node is not necessarily the first to register.
@@ -0,0 +1,88 @@
package reconciler
import (
"context"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)
func inRack(n inventory.Node, rack string) inventory.Node {
n.Labels = map[string]string{"llmgw.io/rack": rack}
return n
}
func runPasses(h *harness, n int) {
for range n {
h.r.Reconcile(context.Background())
h.markReady()
h.advance(time.Second)
}
}
func racksOf(h *harness) map[string]int {
out := map[string]int{}
for _, rep := range h.r.Replicas() {
for _, s := range rep.Shards {
n, _ := h.inv.Get(s.NodeID)
out[n.Location().Rack]++
}
}
return out
}
func TestAModelsReplicasLandInDifferentRacks(t *testing.T) {
// Two nodes in r1 and one in r2, each with room for one replica of
// four cards twice over: without racks the second replica's node is
// a coin toss between r1's other node and r2's.
m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4})
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"), inRack(gpuNode("2", 8), "r1"), inRack(gpuNode("3", 8), "r2"))
runPasses(h, 6)
if got := racksOf(h); got["r1"] != 1 || got["r2"] != 1 {
t.Errorf("replicas by rack %v, want one in each", got)
}
}
func TestNothingIsPlacedInAHeldRackAndExplainSaysSo(t *testing.T) {
m := testModel(1, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4})
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"))
h.inv.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{
"rack:r1": {Domain: "rack:r1", Reason: "cooling fault"},
}})
runPasses(h, 3)
if n := len(h.r.Replicas()); n != 0 {
t.Fatalf("%d replicas placed in a held rack", n)
}
res, _ := h.r.Explain("llama")
if !strings.Contains(res.Explain(), "rack r1 is held: cooling fault") {
t.Errorf("explain-placement does not say the rack is held:\n%s", res.Explain())
}
h.inv.SetFacility(facility.Policy{})
runPasses(h, 3)
if n := len(h.r.Replicas()); n != 1 {
t.Errorf("released, %d replicas; want 1", n)
}
}
func TestADeclaredDrawIsRecordedSoTheNextReplicaSeesTheRackFull(t *testing.T) {
// 6 kW cap; each replica declares 4 kW. The first fits, the second would
// take the rack to 8 and is refused, though the node has the cards.
m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4, PowerWatts: 4000})
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"))
h.inv.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}})
runPasses(h, 6)
if n := len(h.r.Replicas()); n != 1 {
t.Fatalf("%d replicas under a cap with room for one", n)
}
if u := h.inv.PowerUse(time.Now())["rack:r1"]; u.Reserved != 4000 {
t.Errorf("rack r1 has %v W reserved, want the declared 4000", u.Reserved)
}
}
+1 -1
View File
@@ -129,7 +129,7 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
var nodes []string
for _, a := range res.Placement.Nodes {
inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores,
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, PowerW: a.PowerW,
})
nodes = append(nodes, a.NodeID)
}
+12 -1
View File
@@ -1034,7 +1034,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
})
r.inv.Allocate(a.NodeID, inventory.Allocation{
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof),
MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof), PowerW: a.PowerW,
})
}
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
@@ -1361,6 +1361,17 @@ func (r *Reconciler) profileMemory(model, profile string) int {
return p.MinMemoryGB
}
// profileWatts is the draw a profile declared for each of its shards, so an
// adopted replica counts against its racks' power budgets as a placed one
// does; zero when it declared none.
func (r *Reconciler) profileWatts(model, profile string) float64 {
p, _ := r.profileOf(model, profile)
if p.PowerWatts <= 0 {
return 0
}
return p.PowerWatts / float64(max(p.Nodes, 1))
}
// profileOf is a replica's profile as the catalog holds it now.
func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) {
m, ok := r.cat.Load().Resolve(model)