Ilker ManapandClaude Opus 5.5
0d3eb82b9b
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
89 lines
2.8 KiB
Go
89 lines
2.8 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"context"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
)
|
|
|
|
func inRack(n inventory.Node, rack string) inventory.Node {
|
|
n.Labels = map[string]string{"llmgw.io/rack": rack}
|
|
return n
|
|
}
|
|
|
|
func runPasses(h *harness, n int) {
|
|
for range n {
|
|
h.r.Reconcile(context.Background())
|
|
h.markReady()
|
|
h.advance(time.Second)
|
|
}
|
|
}
|
|
|
|
func racksOf(h *harness) map[string]int {
|
|
out := map[string]int{}
|
|
for _, rep := range h.r.Replicas() {
|
|
for _, s := range rep.Shards {
|
|
n, _ := h.inv.Get(s.NodeID)
|
|
out[n.Location().Rack]++
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func TestAModelsReplicasLandInDifferentRacks(t *testing.T) {
|
|
// Two nodes in r1 and one in r2, each with room for one replica of
|
|
// four cards twice over: without racks the second replica's node is
|
|
// a coin toss between r1's other node and r2's.
|
|
m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4})
|
|
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"), inRack(gpuNode("2", 8), "r1"), inRack(gpuNode("3", 8), "r2"))
|
|
runPasses(h, 6)
|
|
|
|
if got := racksOf(h); got["r1"] != 1 || got["r2"] != 1 {
|
|
t.Errorf("replicas by rack %v, want one in each", got)
|
|
}
|
|
}
|
|
|
|
func TestNothingIsPlacedInAHeldRackAndExplainSaysSo(t *testing.T) {
|
|
m := testModel(1, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4})
|
|
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"))
|
|
h.inv.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{
|
|
"rack:r1": {Domain: "rack:r1", Reason: "cooling fault"},
|
|
}})
|
|
runPasses(h, 3)
|
|
|
|
if n := len(h.r.Replicas()); n != 0 {
|
|
t.Fatalf("%d replicas placed in a held rack", n)
|
|
}
|
|
res, _ := h.r.Explain("llama")
|
|
if !strings.Contains(res.Explain(), "rack r1 is held: cooling fault") {
|
|
t.Errorf("explain-placement does not say the rack is held:\n%s", res.Explain())
|
|
}
|
|
|
|
h.inv.SetFacility(facility.Policy{})
|
|
runPasses(h, 3)
|
|
if n := len(h.r.Replicas()); n != 1 {
|
|
t.Errorf("released, %d replicas; want 1", n)
|
|
}
|
|
}
|
|
|
|
func TestADeclaredDrawIsRecordedSoTheNextReplicaSeesTheRackFull(t *testing.T) {
|
|
// 6 kW cap; each replica declares 4 kW. The first fits, the second would
|
|
// take the rack to 8 and is refused, though the node has the cards.
|
|
m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4, PowerWatts: 4000})
|
|
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"))
|
|
h.inv.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}})
|
|
runPasses(h, 6)
|
|
|
|
if n := len(h.r.Replicas()); n != 1 {
|
|
t.Fatalf("%d replicas under a cap with room for one", n)
|
|
}
|
|
if u := h.inv.PowerUse(time.Now())["rack:r1"]; u.Reserved != 4000 {
|
|
t.Errorf("rack r1 has %v W reserved, want the declared 4000", u.Reserved)
|
|
}
|
|
}
|