Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -0,0 +1,203 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"io"
|
||||
"maps"
|
||||
"net/http"
|
||||
"slices"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
// FacilityView is the failure domains as placement sees them.
|
||||
type FacilityView struct {
|
||||
// Domains are every domain a node is in or a budget names, by level
|
||||
// (site, room, row, rack, pdu) and then name.
|
||||
Domains []DomainView `json:"domains"`
|
||||
// Holds are every domain held, including one no node is in yet.
|
||||
Holds []HoldView `json:"holds,omitempty"`
|
||||
// Margin is the share of every cap kept free.
|
||||
Margin float64 `json:"margin,omitempty"`
|
||||
// Unlocated are the nodes with no location label, which placement
|
||||
// treats as each in a rack of its own.
|
||||
Unlocated []string `json:"unlocated,omitempty"`
|
||||
}
|
||||
|
||||
// DomainView is one domain: its nodes, its power and whether it is held.
|
||||
type DomainView struct {
|
||||
Domain string `json:"domain"`
|
||||
Level string `json:"level"`
|
||||
Name string `json:"name"`
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
// CapWatts is the domain's cap, LimitWatts the cap less the margin;
|
||||
// both zero when it has none.
|
||||
CapWatts float64 `json:"cap_watts,omitempty"`
|
||||
LimitWatts float64 `json:"limit_watts,omitempty"`
|
||||
OtherWatts float64 `json:"other_watts,omitempty"`
|
||||
// ReservedWatts is what is placed in it is estimated to draw;
|
||||
// MeasuredWatts what its nodes' sensors read; UsedWatts what counts
|
||||
// against the cap, the larger of the two node by node.
|
||||
ReservedWatts float64 `json:"reserved_watts"`
|
||||
MeasuredWatts float64 `json:"measured_watts"`
|
||||
UsedWatts float64 `json:"used_watts"`
|
||||
// Held is set when the domain is held.
|
||||
Held *HoldView `json:"held,omitempty"`
|
||||
}
|
||||
|
||||
// HoldRequest holds a domain.
|
||||
type HoldRequest struct {
|
||||
// Domain is level:name: rack:r12-07, pdu:r12-07-a, row:12.
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
// Drain moves the work in the domain elsewhere too.
|
||||
Drain bool `json:"drain,omitempty"`
|
||||
}
|
||||
|
||||
// HoldView is a hold, and the nodes holding or releasing it changed.
|
||||
type HoldView struct {
|
||||
Domain string `json:"domain"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Drain bool `json:"drain,omitempty"`
|
||||
By string `json:"by,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
// Nodes are those this call drained, or made ready again on release.
|
||||
Nodes []string `json:"nodes,omitempty"`
|
||||
}
|
||||
|
||||
// NodeLabelsRequest sets and removes labels on a node.
|
||||
type NodeLabelsRequest struct {
|
||||
Set map[string]string `json:"set,omitempty"`
|
||||
// Remove are keys whose label set through the control plane goes; the
|
||||
// agent's own value for the key, if it has one, is in force again.
|
||||
Remove []string `json:"remove,omitempty"`
|
||||
}
|
||||
|
||||
// NodeLabelsView is a node's labels after a change.
|
||||
type NodeLabelsView struct {
|
||||
ID string `json:"id"`
|
||||
// OperatorLabels are those set through the control plane.
|
||||
OperatorLabels map[string]string `json:"operator_labels,omitempty"`
|
||||
// Labels are what placement matches: the agent's with the operator's
|
||||
// over them. Absent for a node that has not registered yet.
|
||||
Labels map[string]string `json:"labels,omitempty"`
|
||||
Location *facility.Location `json:"location,omitempty"`
|
||||
}
|
||||
|
||||
func holdView(h facility.Hold, nodes []string) HoldView {
|
||||
return HoldView{Domain: string(h.Domain), Reason: h.Reason, Drain: h.Drain, By: h.By, At: h.At, Nodes: nodes}
|
||||
}
|
||||
|
||||
// Facility builds the facility view; exported for the console's sake, as
|
||||
// Nodes is.
|
||||
func (a *API) Facility() FacilityView {
|
||||
inv := a.ctl.Inventory()
|
||||
pol := inv.Facility()
|
||||
use := inv.PowerUse(time.Now())
|
||||
v := FacilityView{Margin: pol.Margin}
|
||||
for _, n := range inv.Nodes() {
|
||||
if n.Location().Empty() {
|
||||
v.Unlocated = append(v.Unlocated, n.ID)
|
||||
}
|
||||
}
|
||||
holds := a.ctl.Facility().Holds()
|
||||
for _, h := range holds {
|
||||
v.Holds = append(v.Holds, holdView(h, nil))
|
||||
// Listed with the others even before a node is labelled into it.
|
||||
if _, ok := use[h.Domain]; !ok {
|
||||
use[h.Domain] = inventory.PowerUse{Domain: h.Domain}
|
||||
}
|
||||
}
|
||||
keys := slices.Collect(maps.Keys(use))
|
||||
slices.SortFunc(keys, func(a, b facility.Domain) int {
|
||||
if c := slices.Index(facility.Levels, a.Level()) - slices.Index(facility.Levels, b.Level()); c != 0 {
|
||||
return c
|
||||
}
|
||||
return strings.Compare(a.Name(), b.Name())
|
||||
})
|
||||
for _, d := range keys {
|
||||
u := use[d]
|
||||
dv := DomainView{Domain: string(d), Level: string(d.Level()), Name: d.Name(), Nodes: u.Nodes,
|
||||
CapWatts: u.Budget.CapWatts, LimitWatts: u.Limit, OtherWatts: u.Budget.OtherWatts,
|
||||
ReservedWatts: u.Reserved, MeasuredWatts: u.Measured, UsedWatts: u.Used}
|
||||
if h, ok := pol.Holds[d]; ok {
|
||||
hv := holdView(h, nil)
|
||||
dv.Held = &hv
|
||||
}
|
||||
v.Domains = append(v.Domains, dv)
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func (a *API) facilityReport(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, http.StatusOK, a.Facility())
|
||||
}
|
||||
|
||||
func (a *API) holdDomain(w http.ResponseWriter, r *http.Request) {
|
||||
var req HoldRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
d, err := facility.ParseDomain(req.Domain)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
h := facility.Hold{Domain: d, Reason: strings.TrimSpace(req.Reason), Drain: req.Drain, By: actorOf(r), At: time.Now()}
|
||||
drained, err := a.ctl.Facility().Hold(r.Context(), h)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
a.ctl.Reconciler().Notify()
|
||||
writeJSON(w, http.StatusOK, holdView(h, drained))
|
||||
}
|
||||
|
||||
func (a *API) releaseDomain(w http.ResponseWriter, r *http.Request) {
|
||||
d, err := facility.ParseDomain(r.PathValue("domain"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
h, readied, err := a.ctl.Facility().Release(r.Context(), d)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, err.Error())
|
||||
return
|
||||
}
|
||||
a.ctl.Reconciler().Notify()
|
||||
writeJSON(w, http.StatusOK, holdView(h, readied))
|
||||
}
|
||||
|
||||
func (a *API) setNodeLabels(w http.ResponseWriter, r *http.Request) {
|
||||
id := r.PathValue("id")
|
||||
var req NodeLabelsRequest
|
||||
if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
|
||||
writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
if len(req.Set) == 0 && len(req.Remove) == 0 {
|
||||
writeError(w, http.StatusBadRequest, "nothing to change: give set, remove or both")
|
||||
return
|
||||
}
|
||||
// A node not registered yet is allowed: labelling the machines of a
|
||||
// rack before their agents start is the order a site is built in. A
|
||||
// mistyped name is the risk, and the answer says it is not registered.
|
||||
ops, err := a.ctl.Facility().SetLabels(r.Context(), id, req.Set, req.Remove, actorOf(r))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
a.ctl.Reconciler().Notify()
|
||||
v := NodeLabelsView{ID: id, OperatorLabels: ops}
|
||||
if n, ok := a.ctl.Inventory().Get(id); ok {
|
||||
v.Labels = n.Labels
|
||||
if loc := n.Location(); !loc.Empty() {
|
||||
v.Location = &loc
|
||||
}
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
Reference in new issue
Block a user