llmgw/internal/control/adminapi/facility.go

package adminapi

import (
	"encoding/json"
	"io"
	"maps"
	"net/http"
	"slices"
	"strings"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/facility"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
)

// FacilityView is the failure domains as placement sees them.
type FacilityView struct {
	// Domains are every domain a node is in or a budget names, by level
	// (site, room, row, rack, pdu) and then name.
	Domains []DomainView `json:"domains"`
	// Holds are every domain held, including one no node is in yet.
	Holds []HoldView `json:"holds,omitempty"`
	// Margin is the share of every cap kept free.
	Margin float64 `json:"margin,omitempty"`
	// Unlocated are the nodes with no location label, which placement
	// treats as each in a rack of its own.
	Unlocated []string `json:"unlocated,omitempty"`
}

// DomainView is one domain: its nodes, its power and whether it is held.
type DomainView struct {
	Domain string   `json:"domain"`
	Level  string   `json:"level"`
	Name   string   `json:"name"`
	Nodes  []string `json:"nodes,omitempty"`
	// CapWatts is the domain's cap, LimitWatts the cap less the margin;
	// both zero when it has none.
	CapWatts   float64 `json:"cap_watts,omitempty"`
	LimitWatts float64 `json:"limit_watts,omitempty"`
	OtherWatts float64 `json:"other_watts,omitempty"`
	// ReservedWatts is what is placed in it is estimated to draw;
	// MeasuredWatts what its nodes' sensors read; UsedWatts what counts
	// against the cap, the larger of the two node by node.
	ReservedWatts float64 `json:"reserved_watts"`
	MeasuredWatts float64 `json:"measured_watts"`
	UsedWatts     float64 `json:"used_watts"`
	// Held is set when the domain is held.
	Held *HoldView `json:"held,omitempty"`
}

// HoldRequest holds a domain.
type HoldRequest struct {
	// Domain is level:name: rack:r12-07, pdu:r12-07-a, row:12.
	Domain string `json:"domain"`
	Reason string `json:"reason,omitempty"`
	// Drain moves the work in the domain elsewhere too.
	Drain bool `json:"drain,omitempty"`
}

// HoldView is a hold, and the nodes holding or releasing it changed.
type HoldView struct {
	Domain string    `json:"domain"`
	Reason string    `json:"reason,omitempty"`
	Drain  bool      `json:"drain,omitempty"`
	By     string    `json:"by,omitempty"`
	At     time.Time `json:"at"`
	// Nodes are those this call drained, or made ready again on release.
	Nodes []string `json:"nodes,omitempty"`
}

// NodeLabelsRequest sets and removes labels on a node.
type NodeLabelsRequest struct {
	Set map[string]string `json:"set,omitempty"`
	// Remove are keys whose label set through the control plane goes; the
	// agent's own value for the key, if it has one, is in force again.
	Remove []string `json:"remove,omitempty"`
}

// NodeLabelsView is a node's labels after a change.
type NodeLabelsView struct {
	ID string `json:"id"`
	// OperatorLabels are those set through the control plane.
	OperatorLabels map[string]string `json:"operator_labels,omitempty"`
	// Labels are what placement matches: the agent's with the operator's
	// over them. Absent for a node that has not registered yet.
	Labels   map[string]string  `json:"labels,omitempty"`
	Location *facility.Location `json:"location,omitempty"`
}

func holdView(h facility.Hold, nodes []string) HoldView {
	return HoldView{Domain: string(h.Domain), Reason: h.Reason, Drain: h.Drain, By: h.By, At: h.At, Nodes: nodes}
}

// Facility builds the facility view; exported for the console's sake, as
// Nodes is.
func (a *API) Facility() FacilityView {
	inv := a.ctl.Inventory()
	pol := inv.Facility()
	use := inv.PowerUse(time.Now())
	v := FacilityView{Margin: pol.Margin}
	for _, n := range inv.Nodes() {
		if n.Location().Empty() {
			v.Unlocated = append(v.Unlocated, n.ID)
		}
	}
	holds := a.ctl.Facility().Holds()
	for _, h := range holds {
		v.Holds = append(v.Holds, holdView(h, nil))
		// Listed with the others even before a node is labelled into it.
		if _, ok := use[h.Domain]; !ok {
			use[h.Domain] = inventory.PowerUse{Domain: h.Domain}
		}
	}
	keys := slices.Collect(maps.Keys(use))
	slices.SortFunc(keys, func(a, b facility.Domain) int {
		if c := slices.Index(facility.Levels, a.Level()) - slices.Index(facility.Levels, b.Level()); c != 0 {
			return c
		}
		return strings.Compare(a.Name(), b.Name())
	})
	for _, d := range keys {
		u := use[d]
		dv := DomainView{Domain: string(d), Level: string(d.Level()), Name: d.Name(), Nodes: u.Nodes,
			CapWatts: u.Budget.CapWatts, LimitWatts: u.Limit, OtherWatts: u.Budget.OtherWatts,
			ReservedWatts: u.Reserved, MeasuredWatts: u.Measured, UsedWatts: u.Used}
		if h, ok := pol.Holds[d]; ok {
			hv := holdView(h, nil)
			dv.Held = &hv
		}
		v.Domains = append(v.Domains, dv)
	}
	return v
}

func (a *API) facilityReport(w http.ResponseWriter, _ *http.Request) {
	writeJSON(w, http.StatusOK, a.Facility())
}

func (a *API) holdDomain(w http.ResponseWriter, r *http.Request) {
	var req HoldRequest
	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<12)).Decode(&req); err != nil {
		writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
		return
	}
	d, err := facility.ParseDomain(req.Domain)
	if err != nil {
		writeError(w, http.StatusBadRequest, err.Error())
		return
	}
	h := facility.Hold{Domain: d, Reason: strings.TrimSpace(req.Reason), Drain: req.Drain, By: actorOf(r), At: time.Now()}
	drained, err := a.ctl.Facility().Hold(r.Context(), h)
	if err != nil {
		writeError(w, http.StatusInternalServerError, err.Error())
		return
	}
	a.ctl.Reconciler().Notify()
	writeJSON(w, http.StatusOK, holdView(h, drained))
}

func (a *API) releaseDomain(w http.ResponseWriter, r *http.Request) {
	d, err := facility.ParseDomain(r.PathValue("domain"))
	if err != nil {
		writeError(w, http.StatusBadRequest, err.Error())
		return
	}
	h, readied, err := a.ctl.Facility().Release(r.Context(), d)
	if err != nil {
		writeError(w, http.StatusNotFound, err.Error())
		return
	}
	a.ctl.Reconciler().Notify()
	writeJSON(w, http.StatusOK, holdView(h, readied))
}

func (a *API) setNodeLabels(w http.ResponseWriter, r *http.Request) {
	id := r.PathValue("id")
	var req NodeLabelsRequest
	if err := json.NewDecoder(io.LimitReader(r.Body, 1<<14)).Decode(&req); err != nil {
		writeError(w, http.StatusBadRequest, "the request is not valid JSON: "+err.Error())
		return
	}
	if len(req.Set) == 0 && len(req.Remove) == 0 {
		writeError(w, http.StatusBadRequest, "nothing to change: give set, remove or both")
		return
	}
	// A node not registered yet is allowed: labelling the machines of a
	// rack before their agents start is the order a site is built in. A
	// mistyped name is the risk, and the answer says it is not registered.
	ops, err := a.ctl.Facility().SetLabels(r.Context(), id, req.Set, req.Remove, actorOf(r))
	if err != nil {
		writeError(w, http.StatusBadRequest, err.Error())
		return
	}
	a.ctl.Reconciler().Notify()
	v := NodeLabelsView{ID: id, OperatorLabels: ops}
	if n, ok := a.ctl.Inventory().Get(id); ok {
		v.Labels = n.Labels
		if loc := n.Location(); !loc.Empty() {
			v.Location = &loc
		}
	}
	writeJSON(w, http.StatusOK, v)
}