Facility-aware placement: racks, PDUs, holds and power caps
Nodes say where they are with llmgw.io/site, room, row, rack and pdu labels, from agent.yaml or `llmgw ctl label` (kept in the store, over the agent's). Placement spreads a model's replicas across racks and PDUs, keeps a multi-node replica within one domain when its profile says keep_within, places nothing in a domain held with `llmgw ctl facility hold` (optionally draining it), and refuses a shard that would take a domain past its power cap from facility.budgets, counting declared, derived and measured draw. Explain-placement names the domain, and keeps the rejections of a placed replica; `ctl nodes` gains LOCATION and --domain; `ctl facility` lists the domains; the console's Nodes view shows and sets location and holds. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
0d3eb82b9b
42 files changed
+3510
-63
No files matched your search
@@ -14,6 +14,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
)
|
||||
|
||||
@@ -226,6 +227,21 @@ type Profile struct {
|
||||
// Agents have reported labels since the beginning; this is what reads them.
|
||||
NodeSelector map[string]string `json:"node_selector,omitempty" koanf:"node_selector"`
|
||||
|
||||
// PowerWatts is what one replica of this profile draws, in watts, split
|
||||
// evenly over its nodes, counted against the power cap of every rack,
|
||||
// PDU or other domain its nodes are in (facility.budgets in server.yaml).
|
||||
// Zero derives it: each card at the power limit it reports, each
|
||||
// reserved core at facility.cpu_core_watts. Only matters where a domain
|
||||
// has a cap.
|
||||
PowerWatts float64 `json:"power_watts,omitempty" koanf:"power_watts"`
|
||||
|
||||
// KeepWithin keeps a multi-node replica's nodes inside one rack, row,
|
||||
// room or site, read from their llmgw.io/rack (and so on) labels: for a
|
||||
// fabric whose leaves do not reach across racks, or cables that do not.
|
||||
// Nodes without the label are passed over for such a profile. Empty is
|
||||
// no constraint.
|
||||
KeepWithin string `json:"keep_within,omitempty" koanf:"keep_within"`
|
||||
|
||||
// Weight is the router's capacity weight. It should come from ctl bench
|
||||
// rather than a guess (plan §6.2.1).
|
||||
Weight float64 `json:"weight,omitempty" koanf:"weight"`
|
||||
@@ -715,6 +731,14 @@ func (m Model) validate() error {
|
||||
return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err)
|
||||
}
|
||||
|
||||
if p.PowerWatts < 0 {
|
||||
return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts)
|
||||
}
|
||||
if p.KeepWithin != "" {
|
||||
if _, err := facility.ParseLevel(p.KeepWithin); err != nil {
|
||||
return fmt.Errorf("model %q profile %q: keep_within: %w", m.Name, p.Name, err)
|
||||
}
|
||||
}
|
||||
if p.GPUClass == "" && !cpuCapable(p.Engine) {
|
||||
return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q",
|
||||
m.Name, p.Name, p.Engine)
|
||||
@@ -844,6 +868,8 @@ func (m Model) PlacementProfiles() []placement.Profile {
|
||||
MinMemoryGB: p.MinMemoryGB,
|
||||
MinDiskGB: p.MinDiskGB,
|
||||
CPUCores: p.CPUCores,
|
||||
PowerWatts: p.PowerWatts,
|
||||
KeepWithin: facility.Level(strings.ToLower(strings.TrimSpace(p.KeepWithin))),
|
||||
// The model's source, so placement can prefer nodes that already
|
||||
// hold the weights rather than one that must first move them.
|
||||
Source: m.Source,
|
||||
|
||||
Reference in new issue
Block a user