Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -0,0 +1,75 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
)
|
||||
|
||||
func TestLabelArgumentsSetWithEqualsAndRemoveWithADash(t *testing.T) {
|
||||
req, err := parseLabelArgs([]string{"llmgw.io/rack=r1", "llmgw.io/pdu=p1,p2", "role-"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.Set["llmgw.io/rack"] != "r1" || req.Set["llmgw.io/pdu"] != "p1,p2" || !slices.Equal(req.Remove, []string{"role"}) {
|
||||
t.Errorf("got %+v", req)
|
||||
}
|
||||
for _, bad := range []string{"rack", "=r1", "rack=", "-"} {
|
||||
if _, err := parseLabelArgs([]string{bad}); err == nil {
|
||||
t.Errorf("%q was accepted", bad)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNodesAreFilteredByDomain(t *testing.T) {
|
||||
nodes := fleet()
|
||||
nodes[0].Location = &facility.Location{Rack: "r1", PDUs: []string{"p1"}}
|
||||
nodes[1].Location = &facility.Location{Rack: "r2", PDUs: []string{"p1"}}
|
||||
got, err := nodeFilter{domain: "pdu:p1"}.apply(nodes)
|
||||
if err != nil || ids(got) != "n1,n2" {
|
||||
t.Errorf("--domain pdu:p1 = %s, %v", ids(got), err)
|
||||
}
|
||||
if got, _ := (nodeFilter{domain: "rack:r2"}).apply(nodes); ids(got) != "n2" {
|
||||
t.Errorf("--domain rack:r2 = %s", ids(got))
|
||||
}
|
||||
if _, err := (nodeFilter{domain: "r2"}).apply(nodes); err == nil {
|
||||
t.Error("a domain with no level was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheLocationFormSetsWhatChangedAndRemovesWhatWasEmptied(t *testing.T) {
|
||||
n := adminapi.NodeView{
|
||||
Labels: map[string]string{"llmgw.io/rack": "r1", "llmgw.io/row": "12"},
|
||||
OperatorLabels: map[string]string{"llmgw.io/row": "12"},
|
||||
}
|
||||
// site, room, row, rack, pdu
|
||||
req := locationRequest(n, []string{"", "hall-b", "", "r1", "p1"})
|
||||
if len(req.Set) != 2 || req.Set["llmgw.io/room"] != "hall-b" || req.Set["llmgw.io/pdu"] != "p1" {
|
||||
t.Errorf("set %v: the unchanged rack is left alone", req.Set)
|
||||
}
|
||||
if !slices.Equal(req.Remove, []string{"llmgw.io/row"}) {
|
||||
t.Errorf("remove %v, want the row the operator had set", req.Remove)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFacilityTableShowsUseAgainstTheLimitAndWhatIsHeld(t *testing.T) {
|
||||
var b bytes.Buffer
|
||||
printFacility(&b, adminapi.FacilityView{
|
||||
Domains: []adminapi.DomainView{
|
||||
{Domain: "rack:r1", Nodes: []string{"a"}, LimitWatts: 10000, UsedWatts: 7500, ReservedWatts: 7000, MeasuredWatts: 7500,
|
||||
Held: &adminapi.HoldView{Domain: "rack:r1", Reason: "PDU swap"}},
|
||||
},
|
||||
Holds: []adminapi.HoldView{{Domain: "rack:r1", By: "ilker"}},
|
||||
Unlocated: []string{"b"},
|
||||
})
|
||||
out := b.String()
|
||||
for _, want := range []string{"7.5 kW (75%)", "10.0 kW", "held: PDU swap", "rack:r1 held by ilker", "No location: b"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("missing %q in:\n%s", want, out)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user