Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -145,6 +145,7 @@ func (r *Reconciler) mergeAdopted(rep *Replica, group []AdoptedShard) {
|
||||
ShardID: s.ShardID, ReplicaID: rep.ID, GPUIndices: s.GPUIndices,
|
||||
MemoryGB: r.profileMemory(rep.Model, rep.Profile),
|
||||
CPUCores: r.adoptedCores(rep.Model, rep.Profile),
|
||||
PowerW: r.profileWatts(rep.Model, rep.Profile),
|
||||
})
|
||||
// The endpoint comes from the leader, which may be in any of the
|
||||
// groups — the rank 0 node is not necessarily the first to register.
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
)
|
||||
|
||||
func inRack(n inventory.Node, rack string) inventory.Node {
|
||||
n.Labels = map[string]string{"llmgw.io/rack": rack}
|
||||
return n
|
||||
}
|
||||
|
||||
func runPasses(h *harness, n int) {
|
||||
for range n {
|
||||
h.r.Reconcile(context.Background())
|
||||
h.markReady()
|
||||
h.advance(time.Second)
|
||||
}
|
||||
}
|
||||
|
||||
func racksOf(h *harness) map[string]int {
|
||||
out := map[string]int{}
|
||||
for _, rep := range h.r.Replicas() {
|
||||
for _, s := range rep.Shards {
|
||||
n, _ := h.inv.Get(s.NodeID)
|
||||
out[n.Location().Rack]++
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestAModelsReplicasLandInDifferentRacks(t *testing.T) {
|
||||
// Two nodes in r1 and one in r2, each with room for one replica of
|
||||
// four cards twice over: without racks the second replica's node is
|
||||
// a coin toss between r1's other node and r2's.
|
||||
m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4})
|
||||
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"), inRack(gpuNode("2", 8), "r1"), inRack(gpuNode("3", 8), "r2"))
|
||||
runPasses(h, 6)
|
||||
|
||||
if got := racksOf(h); got["r1"] != 1 || got["r2"] != 1 {
|
||||
t.Errorf("replicas by rack %v, want one in each", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNothingIsPlacedInAHeldRackAndExplainSaysSo(t *testing.T) {
|
||||
m := testModel(1, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4})
|
||||
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"))
|
||||
h.inv.SetFacility(facility.Policy{Holds: map[facility.Domain]facility.Hold{
|
||||
"rack:r1": {Domain: "rack:r1", Reason: "cooling fault"},
|
||||
}})
|
||||
runPasses(h, 3)
|
||||
|
||||
if n := len(h.r.Replicas()); n != 0 {
|
||||
t.Fatalf("%d replicas placed in a held rack", n)
|
||||
}
|
||||
res, _ := h.r.Explain("llama")
|
||||
if !strings.Contains(res.Explain(), "rack r1 is held: cooling fault") {
|
||||
t.Errorf("explain-placement does not say the rack is held:\n%s", res.Explain())
|
||||
}
|
||||
|
||||
h.inv.SetFacility(facility.Policy{})
|
||||
runPasses(h, 3)
|
||||
if n := len(h.r.Replicas()); n != 1 {
|
||||
t.Errorf("released, %d replicas; want 1", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADeclaredDrawIsRecordedSoTheNextReplicaSeesTheRackFull(t *testing.T) {
|
||||
// 6 kW cap; each replica declares 4 kW. The first fits, the second would
|
||||
// take the rack to 8 and is refused, though the node has the cards.
|
||||
m := testModel(2, catalog.Profile{Name: "tp4", GPUClass: "h100-80g", TP: 4, PowerWatts: 4000})
|
||||
h := newHarness(t, m, inRack(gpuNode("1", 8), "r1"))
|
||||
h.inv.SetFacility(facility.Policy{Budgets: map[facility.Domain]facility.Budget{"rack:r1": {CapWatts: 6000}}})
|
||||
runPasses(h, 6)
|
||||
|
||||
if n := len(h.r.Replicas()); n != 1 {
|
||||
t.Fatalf("%d replicas under a cap with room for one", n)
|
||||
}
|
||||
if u := h.inv.PowerUse(time.Now())["rack:r1"]; u.Reserved != 4000 {
|
||||
t.Errorf("rack r1 has %v W reserved, want the declared 4000", u.Reserved)
|
||||
}
|
||||
}
|
||||
@@ -129,7 +129,7 @@ func (r *Reconciler) Headroom(model string, limit int) (Headroom, bool) {
|
||||
var nodes []string
|
||||
for _, a := range res.Placement.Nodes {
|
||||
inv.Allocate(a.NodeID, inventory.Allocation{
|
||||
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores,
|
||||
ShardID: "headroom", ReplicaID: "headroom", GPUIndices: a.GPUIndices, MemoryGB: mem, CPUCores: cores, PowerW: a.PowerW,
|
||||
})
|
||||
nodes = append(nodes, a.NodeID)
|
||||
}
|
||||
|
||||
@@ -1034,7 +1034,7 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
})
|
||||
r.inv.Allocate(a.NodeID, inventory.Allocation{
|
||||
ShardID: shardID, ReplicaID: id, GPUIndices: a.GPUIndices,
|
||||
MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof),
|
||||
MemoryGB: prof.MinMemoryGB, CPUCores: cpuCores(prof), PowerW: a.PowerW,
|
||||
})
|
||||
}
|
||||
rep.Endpoint = fmt.Sprintf("http://%s:%d", addrOf(leaderNode), basePort)
|
||||
@@ -1361,6 +1361,17 @@ func (r *Reconciler) profileMemory(model, profile string) int {
|
||||
return p.MinMemoryGB
|
||||
}
|
||||
|
||||
// profileWatts is the draw a profile declared for each of its shards, so an
|
||||
// adopted replica counts against its racks' power budgets as a placed one
|
||||
// does; zero when it declared none.
|
||||
func (r *Reconciler) profileWatts(model, profile string) float64 {
|
||||
p, _ := r.profileOf(model, profile)
|
||||
if p.PowerWatts <= 0 {
|
||||
return 0
|
||||
}
|
||||
return p.PowerWatts / float64(max(p.Nodes, 1))
|
||||
}
|
||||
|
||||
// profileOf is a replica's profile as the catalog holds it now.
|
||||
func (r *Reconciler) profileOf(model, profile string) (catalog.Profile, bool) {
|
||||
m, ok := r.cat.Load().Resolve(model)
|
||||
|
||||
Reference in new issue
Block a user