llmgw/internal/control/placement/facility.go

package placement

import (
	"fmt"
	"slices"
	"sort"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)

// declaredShardWatts is the profile's declared draw for one node's shard.
func (p Profile) declaredShardWatts() float64 {
	if p.PowerWatts <= 0 {
		return 0
	}
	return p.PowerWatts / float64(max(p.Nodes, 1))
}

// shardWatts is what one shard of this profile is estimated to draw on a
// candidate: the declared figure, or its cards at the highest power limit
// among the node's free ones of the class (the cards are not chosen yet, and
// guessing low is the mistake that trips a breaker), or its cores.
func (s *Scheduler) shardWatts(p Profile, c candidate) float64 {
	if w := p.declaredShardWatts(); w > 0 {
		return w
	}
	pol := s.inv.Facility()
	if p.CPUOnly() {
		return float64(p.CPUCores) * pol.CPUCoreWatts
	}
	per := 0.0
	for _, g := range c.free {
		per = max(per, inventory.GPUWatts(c.node, g.Index, pol))
	}
	if per == 0 {
		per = pol.GPUWatts
	}
	return per * float64(p.GPUsPerNode)
}

// budget is the power budgets as one placement decision spends them: the
// domains' use when it began, and what the shards it has chosen so far add.
type budget struct {
	pol   facility.Policy
	use   map[facility.Domain]inventory.PowerUse
	added map[facility.Domain]float64
}

// newBudget reads the domains' use, once per decision. Without budgets there
// is nothing to read and every check passes.
func (s *Scheduler) newBudget() *budget {
	pol := s.inv.Facility()
	b := &budget{pol: pol, added: map[facility.Domain]float64{}}
	if len(pol.Budgets) > 0 {
		b.use = s.inv.PowerUse(s.clock())
	}
	return b
}

// check says why a shard drawing need watts on a node would take one of the
// node's domains past its limit; empty when it fits them all. A shard whose
// draw is not known (no declared figure, no card limit, no policy figure)
// needs nothing: missing data never blocks placement.
func (b *budget) check(n inventory.Node, need float64) string {
	if b.use == nil {
		return ""
	}
	for _, d := range n.Domains() {
		limit := b.pol.Limit(d)
		if limit <= 0 {
			continue
		}
		used := b.use[d].Used + b.added[d]
		if used+need > limit {
			capW := b.pol.Budgets[d].CapWatts
			of := facility.KW(limit) + " cap"
			if limit < capW {
				of = fmt.Sprintf("%s limit (%s cap less %.0f%%)", facility.KW(limit), facility.KW(capW), 100-100*limit/capW)
			}
			return fmt.Sprintf("%s: %s of a %s in use, the shard needs ~%s",
				d.Human(), facility.KW(used), of, facility.KW(need))
		}
	}
	return ""
}

// add counts a chosen shard against the node's domains.
func (b *budget) add(n inventory.Node, need float64) {
	for _, d := range n.Domains() {
		b.added[d] += need
	}
}

// clock is the scheduler's time, for the freshness of power readings.
func (s *Scheduler) clock() time.Time {
	if s.now != nil {
		return s.now()
	}
	return time.Now()
}

// heldReason is a hold as a rejection says it.
func heldReason(h facility.Hold) string {
	why := h.Domain.Human() + " is held"
	if h.Reason != "" {
		why += ": " + h.Reason
	}
	return why
}

// spreadLevels are the domains replicas of one model are kept apart in:
// the rack, which a top-of-rack switch or a door left open takes out whole,
// and the PDU, which a breaker does.
var spreadLevels = []facility.Level{facility.Rack, facility.PDU}

// spreadAcrossDomains moves to the front the candidates sharing the fewest
// racks and PDUs with the nodes the model already runs on, keeping the order
// given among equals.
//
// A preference and never a filter, as spreading across nodes is: a rack
// holding a model's only free cards still takes its second replica, which
// survives an engine crash if not a breaker. Nodes with no location share
// nothing and lose nothing, so a cluster without labels places as before.
func (s *Scheduler) spreadAcrossDomains(candidates []candidate, avoid []string) {
	if len(avoid) == 0 {
		return
	}
	taken := map[facility.Domain]bool{}
	for _, id := range avoid {
		if n, ok := s.inv.Get(id); ok {
			for _, d := range n.Domains() {
				if slices.Contains(spreadLevels, d.Level()) {
					taken[d] = true
				}
			}
		}
	}
	if len(taken) == 0 {
		return
	}
	shared := func(c candidate) int {
		k := 0
		for _, d := range c.node.Domains() {
			if taken[d] {
				k++
			}
		}
		return k
	}
	sort.SliceStable(candidates, func(i, j int) bool { return shared(candidates[i]) < shared(candidates[j]) })
}

// withinGroups divides a multi-node replica's candidates by the domain its
// profile keeps it within, largest first, and rejects the nodes that do not
// say which they are in. Without the constraint it is all of them, as one.
func withinGroups(p Profile, pool string, cands []candidate, rejections *[]Rejection) [][]candidate {
	if p.KeepWithin == "" {
		return [][]candidate{cands}
	}
	by := map[facility.Domain][]candidate{}
	for _, c := range cands {
		d := c.node.Location().At(p.KeepWithin)
		if d == "" {
			*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool, Node: c.node.ID,
				Reason: fmt.Sprintf("has no %s label, and the profile keeps its nodes within one %s",
					p.KeepWithin.Label(), p.KeepWithin)})
			continue
		}
		by[d] = append(by[d], c)
	}
	keys := make([]facility.Domain, 0, len(by))
	for d := range by {
		keys = append(keys, d)
	}
	sort.Slice(keys, func(i, j int) bool {
		if len(by[keys[i]]) != len(by[keys[j]]) {
			return len(by[keys[i]]) > len(by[keys[j]])
		}
		return keys[i] < keys[j]
	})
	out := make([][]candidate, 0, len(keys))
	for _, d := range keys {
		if len(by[d]) < p.Nodes {
			*rejections = append(*rejections, Rejection{Profile: p.Name, Pool: pool,
				Reason: fmt.Sprintf("needs %d nodes within one %s, %s has %d", p.Nodes, p.KeepWithin, d.Human(), len(by[d]))})
			continue
		}
		out = append(out, by[d])
	}
	return out
}