llmgw/internal/control/inventory/facility.go

package inventory

import (
	"maps"
	"slices"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
)

// Location is where the node is, from its labels.
func (n Node) Location() facility.Location { return facility.LocationOf(n.Labels) }

// Domains are the failure domains the node is in, largest first.
func (n Node) Domains() []facility.Domain { return n.Location().Domains() }

// mergeLabels lays the operator's labels over the agent's. A copy, so
// neither is written through.
func mergeLabels(agent, operator map[string]string) map[string]string {
	if len(operator) == 0 {
		return maps.Clone(agent)
	}
	out := make(map[string]string, len(agent)+len(operator))
	maps.Copy(out, agent)
	maps.Copy(out, operator)
	return out
}

// SetFacility puts the facility's holds and power budgets in force for
// every placement from now on.
func (s *Store) SetFacility(p facility.Policy) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.facility = p.Clone()
}

// Facility is the policy in force; a copy, so a caller may change it.
func (s *Store) Facility() facility.Policy {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return s.facility.Clone()
}

// SetOperatorLabels replaces the labels set through the control plane for a
// node, which may not have registered yet: they are laid over its agent's
// when it does.
func (s *Store) SetOperatorLabels(id string, labels map[string]string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.operatorLabels == nil {
		s.operatorLabels = map[string]map[string]string{}
	}
	if len(labels) == 0 {
		delete(s.operatorLabels, id)
	} else {
		s.operatorLabels[id] = maps.Clone(labels)
	}
	if n, ok := s.nodes[id]; ok {
		n.Labels = mergeLabels(n.AgentLabels, s.operatorLabels[id])
	}
}

// OperatorLabels are the labels set through the control plane for a node.
func (s *Store) OperatorLabels(id string) map[string]string {
	s.mu.RLock()
	defer s.mu.RUnlock()
	return maps.Clone(s.operatorLabels[id])
}

// PowerUse is one domain's draw as placement counts it.
type PowerUse struct {
	Domain facility.Domain
	// Nodes are the domain's nodes, sorted.
	Nodes []string
	// Reserved is what the replicas placed in the domain are estimated to
	// draw, with each node's own draw and the domain's other equipment.
	Reserved float64
	// Measured is what the nodes' sensors read, with the same additions,
	// for the nodes that read anything; the others count what is reserved.
	Measured float64
	// Used is what placement counts against the cap: per node, the larger
	// of reserved and measured, so an estimate that was too low is
	// corrected by the room it stops offering rather than by a breaker.
	Used float64
	// Limit is the cap less the margin; zero is no cap.
	Limit  float64
	Budget facility.Budget
}

// PowerUse is every domain any node is in, or that has a budget, by domain.
func (s *Store) PowerUse(now time.Time) map[facility.Domain]PowerUse {
	s.mu.RLock()
	defer s.mu.RUnlock()
	pol := s.facility
	out := map[facility.Domain]PowerUse{}
	for d, b := range pol.Budgets {
		out[d] = PowerUse{Domain: d, Budget: b, Limit: pol.Limit(d),
			Reserved: b.OtherWatts, Measured: b.OtherWatts, Used: b.OtherWatts}
	}
	ids := slices.Sorted(maps.Keys(s.nodes))
	for _, id := range ids {
		n := s.nodes[id]
		reserved := 0.0
		for _, a := range s.allocs[id] {
			reserved += AllocationWatts(*n, a, pol)
		}
		measured, known := MeasuredWatts(*n, now, pol.Fresh())
		used := reserved
		if known && measured > used {
			used = measured
		}
		if !known {
			measured = reserved
		}
		for _, d := range n.Domains() {
			u, ok := out[d]
			if !ok {
				u = PowerUse{Domain: d}
			}
			u.Nodes = append(u.Nodes, id)
			u.Reserved += pol.NodeWatts + reserved
			u.Measured += pol.NodeWatts + measured
			u.Used += pol.NodeWatts + used
			out[d] = u
		}
	}
	return out
}

// AllocationWatts is what one shard is estimated to draw: what its profile
// declared, or else each of its cards at the power limit the card reports
// (the policy's figure for a card that reports none) and each of its cores
// at the policy's figure.
func AllocationWatts(n Node, a Allocation, pol facility.Policy) float64 {
	if a.PowerW > 0 {
		return a.PowerW
	}
	w := float64(a.CPUCores) * pol.CPUCoreWatts
	for _, i := range a.GPUIndices {
		w += GPUWatts(n, i, pol)
	}
	return w
}

// GPUWatts is one card's estimated draw: the power limit it reports, else
// the policy's figure for a card.
func GPUWatts(n Node, index int, pol facility.Policy) float64 {
	if n.Load != nil {
		for _, g := range n.Load.GPUs {
			if g.Index == index && g.PowerLimitW > 0 {
				return g.PowerLimitW
			}
		}
	}
	return pol.GPUWatts
}

// MeasuredWatts is what the node's sensors read at its last heartbeat: its
// cards and its CPU packages. Known is false when the reading is older than
// fresh, or the node has no power sensor at all, which a VM never has.
func MeasuredWatts(n Node, now time.Time, fresh time.Duration) (w float64, known bool) {
	if n.Load == nil || now.Sub(n.Load.At) > fresh {
		return 0, false
	}
	for _, g := range n.Load.GPUs {
		if g.PowerW > 0 {
			w += g.PowerW
			known = true
		}
	}
	if n.Load.CPUPowerW > 0 {
		w += n.Load.CPUPowerW
		known = true
	}
	return w, known
}