rollcall/cmd/rollcall/stateless.go

package main

// Stateless (Warewulf-style) provisioning: profiles, node naming and IP pools,
// per-node overlays rendered from templates, dnsmasq host export and the
// per-MAC PXE entry that boots a profile's root image from RAM.

import (
	"archive/tar"
	"bytes"
	"context"
	"crypto/ed25519"
	"crypto/sha256"
	"encoding/binary"
	"encoding/hex"
	"encoding/json"
	"encoding/pem"
	"errors"
	"fmt"
	"io/fs"
	"log"
	"net"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"text/template"
	"time"

	"golang.org/x/crypto/ssh"
)

// ---------------------------------------------------------------- profiles

type Profile struct {
	Name       string         `json:"name"`
	Image      string         `json:"image"`       // directory under images_dir: vmlinuz, initrd.img, filesystem.squashfs
	Append     string         `json:"append"`      // extra kernel parameters
	Overlays   []string       `json:"overlays"`    // directories under overlays_dir
	NameFormat string         `json:"name_format"` // e.g. node%03d
	Vars       map[string]any `json:"vars"`
	Created    time.Time      `json:"created"`
}

func (s *Store) Profiles(ctx context.Context) ([]Profile, error) {
	rows, err := s.pool.Query(ctx, `SELECT name, image, coalesce(append,''), overlays, coalesce(name_format,''), vars, created FROM profiles ORDER BY name`)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	out := []Profile{}
	for rows.Next() {
		var p Profile
		var ov, vars []byte
		if err := rows.Scan(&p.Name, &p.Image, &p.Append, &ov, &p.NameFormat, &vars, &p.Created); err != nil {
			return nil, err
		}
		_ = json.Unmarshal(ov, &p.Overlays)
		_ = json.Unmarshal(vars, &p.Vars)
		out = append(out, p)
	}
	return out, rows.Err()
}

func (s *Store) Profile(ctx context.Context, name string) (*Profile, error) {
	ps, err := s.Profiles(ctx)
	if err != nil {
		return nil, err
	}
	for i := range ps {
		if ps[i].Name == name {
			return &ps[i], nil
		}
	}
	return nil, errNotFound
}

func (s *Store) PutProfile(ctx context.Context, p Profile) error {
	if p.Overlays == nil {
		p.Overlays = []string{}
	}
	if p.Vars == nil {
		p.Vars = map[string]any{}
	}
	if p.NameFormat == "" {
		p.NameFormat = "node%03d"
	}
	ov, _ := json.Marshal(p.Overlays)
	vars, _ := json.Marshal(p.Vars)
	_, err := s.pool.Exec(ctx, `INSERT INTO profiles(name, image, append, overlays, name_format, vars) VALUES ($1,$2,$3,$4,$5,$6)
		ON CONFLICT (name) DO UPDATE SET image=$2, append=$3, overlays=$4, name_format=$5, vars=$6`,
		p.Name, p.Image, p.Append, ov, p.NameFormat, vars)
	return err
}

func (s *Store) DeleteProfile(ctx context.Context, name string) error {
	var n int
	_ = s.pool.QueryRow(ctx, `SELECT count(*) FROM machines WHERE profile=$1`, name).Scan(&n)
	if n > 0 {
		return fmt.Errorf("profile %s is assigned to %d machine(s)", name, n)
	}
	tag, err := s.pool.Exec(ctx, `DELETE FROM profiles WHERE name=$1`, name)
	if err != nil {
		return err
	}
	if tag.RowsAffected() == 0 {
		return errNotFound
	}
	return nil
}

// ---------------------------------------------------------------- allocation

type nodeAssignment struct {
	Profile   string
	Name      string
	MgmtIP    string
	IBIP      string
	TokenHash string
	Token     []byte // sealed
}

func (s *Store) Assign(ctx context.Context, id string, n nodeAssignment) error {
	if !validDomain(n.Name) {
		return fmt.Errorf("node name %q is not a host name (letters, digits, hyphens)", n.Name)
	}
	tag, err := s.pool.Exec(ctx, `UPDATE machines SET profile=$2, node_name=$3, mgmt_ip=$4, ib_ip=$5,
		node_token_hash=$6, node_token=$7, state='stateless', booted_at=NULL WHERE id=$1`,
		id, n.Profile, n.Name, nilIfEmpty(n.MgmtIP), nilIfEmpty(n.IBIP), n.TokenHash, n.Token)
	if err != nil {
		return err
	}
	if tag.RowsAffected() == 0 {
		return errNotFound
	}
	return nil
}

func (s *Store) Unassign(ctx context.Context, id string) error {
	_, err := s.pool.Exec(ctx, `UPDATE machines SET profile=NULL, node_name=NULL, mgmt_ip=NULL, ib_ip=NULL,
		node_token_hash=NULL, node_token=NULL, state='ready', booted_at=NULL WHERE id=$1`, id)
	return err
}

func (s *Store) ByNodeToken(ctx context.Context, hash string) (*Machine, error) {
	return scanMachine(s.pool.QueryRow(ctx, `SELECT `+machineCols+` FROM machines WHERE node_token_hash=$1`, hash), true)
}

func (s *Store) MarkBooted(ctx context.Context, id string) error {
	_, err := s.pool.Exec(ctx, `UPDATE machines SET state='booted', booted_at=coalesce(booted_at, now()), last_seen=now() WHERE id=$1`, id)
	return err
}

func (s *Store) SetHostKey(ctx context.Context, id string, sealed []byte) error {
	_, err := s.pool.Exec(ctx, `UPDATE machines SET host_key=$2 WHERE id=$1`, id, sealed)
	return err
}

func nilIfEmpty(s string) *string {
	if s == "" {
		return nil
	}
	return &s
}

// parsePool turns "10.0.0.50-10.0.0.99" into the ordered list of addresses.
func parsePool(spec string) ([]string, error) {
	lo, hi, ok := strings.Cut(strings.TrimSpace(spec), "-")
	if !ok {
		return nil, fmt.Errorf("pool %q: want A.B.C.D-A.B.C.D", spec)
	}
	a, b := net.ParseIP(strings.TrimSpace(lo)).To4(), net.ParseIP(strings.TrimSpace(hi)).To4()
	if a == nil || b == nil {
		return nil, fmt.Errorf("pool %q: bad address", spec)
	}
	ai, bi := binary.BigEndian.Uint32(a), binary.BigEndian.Uint32(b)
	if bi < ai || bi-ai > 65536 {
		return nil, fmt.Errorf("pool %q: bad range", spec)
	}
	out := make([]string, 0, bi-ai+1)
	for i := ai; i <= bi; i++ {
		ip := make(net.IP, 4)
		binary.BigEndian.PutUint32(ip, i)
		out = append(out, ip.String())
	}
	return out, nil
}

// allocate picks a node name and addresses: from IPAM when the machine's boot
// NIC sits in a modeled subnet, else from the legacy node_pools.
func (a *App) allocate(ctx context.Context, m *Machine, nameFormat, wantName string) (name, mgmt, ib string, err error) {
	if name, _, _, err = a.allocatePools(ctx, nameFormat, wantName, false); err != nil {
		return
	}
	if b := bootNIC(machineNICs(m), m.MACs); b != nil {
		for _, ip := range b.IPv4 {
			if sn := subnetFor(a.subnetsSnapshot(), ip); sn != nil {
				have, _ := a.store.IPs(ctx, sn.CIDR, m.ID)
				for _, x := range have {
					return name, x.IP, "", nil
				}
				mgmt, err = a.allocIP(ctx, *sn, m.ID, b.Name, name)
				return
			}
		}
	}
	_, mgmt, ib, err = a.allocatePools(ctx, nameFormat, name, true)
	return
}

func (a *App) allocatePools(ctx context.Context, nameFormat, wantName string, pools bool) (name, mgmt, ib string, err error) {
	ms, err := a.store.List(ctx)
	if err != nil {
		return
	}
	usedName, usedIP := map[string]bool{}, map[string]bool{}
	for _, m := range ms {
		if m.NodeName != nil {
			usedName[*m.NodeName] = true
		}
		if m.MgmtIP != nil {
			usedIP[*m.MgmtIP] = true
		}
		if m.IBIP != nil {
			usedIP[*m.IBIP] = true
		}
	}
	name = wantName
	if name == "" {
		f := nameFormat
		if f == "" {
			f = "node%03d"
		}
		for i := 1; i < 100000; i++ {
			if c := fmt.Sprintf(f, i); !usedName[c] {
				name = c
				break
			}
		}
	} else if usedName[name] && !pools {
		err = fmt.Errorf("node name %s already in use", name)
		return
	}
	if !pools {
		return
	}
	pick := func(pool string) (string, error) {
		spec := a.cfg.NodePools[pool]
		if spec == "" {
			return "", nil
		}
		ips, err := parsePool(spec)
		if err != nil {
			return "", err
		}
		for _, ip := range ips {
			if !usedIP[ip] {
				usedIP[ip] = true
				return ip, nil
			}
		}
		return "", fmt.Errorf("pool %s exhausted", pool)
	}
	if mgmt, err = pick("mgmt"); err != nil {
		return
	}
	ib, err = pick("ib")
	return
}

// ---------------------------------------------------------------- template context

type NodeCtx struct {
	ID, Name, Profile, State string
	BootMAC                  string
	MACs                     []string
	MgmtIP, IBIP             string
	Sockets, CoresPerSocket  int
	ThreadsPerCore, CPUs     int
	MemMB                    int
	GPUs                     int
	IBGUIDs                  []string
	HostKey, HostKeyPub      string // only filled for the node being rendered
	Tags                     []string
	Manufacturer, Product    string
	Disks                    int
	NVMe                     bool
	PCIVendors               []string // vendor ids seen on the PCI bus (e.g. 10de = NVIDIA, 15b3 = Mellanox)
	OS                       string
	Building, Room, Rack     string
	Unit                     int
	AssetTag                 string
}

type overlayCtx struct {
	Node     NodeCtx
	Nodes    []NodeCtx // all machines with a profile, sorted by name
	Profile  Profile
	Overlays []string       // profile overlays plus those of the node's tags
	CA       string         // the installation's CA certificate (PEM)
	Vars     map[string]any // cluster_vars overlaid with profile vars
	Server   string
	Now      string
}

func atoi(v any) int {
	n, _ := strconv.Atoi(strings.TrimSpace(str(v)))
	return n
}

func nodeCtx(m *Machine) NodeCtx {
	var inv obj
	_ = json.Unmarshal(m.Inventory, &inv)
	n := NodeCtx{ID: m.ID, Name: deref(m.NodeName), Profile: deref(m.Profile), State: m.State, MACs: m.MACs,
		MgmtIP: deref(m.MgmtIP), IBIP: deref(m.IBIP)}
	if len(m.MACs) > 0 {
		n.BootMAC = m.MACs[0]
	}
	n.Sockets = atoi(dig(inv, "cpu", "Socket(s)"))
	n.CoresPerSocket = atoi(dig(inv, "cpu", "Core(s) per socket"))
	n.ThreadsPerCore = atoi(dig(inv, "cpu", "Thread(s) per core"))
	n.CPUs = atoi(dig(inv, "cpu", "CPU(s)"))
	if n.Sockets == 0 {
		n.Sockets = 1
	}
	if n.CoresPerSocket == 0 {
		n.CoresPerSocket = max(1, n.CPUs/max(1, n.ThreadsPerCore))
	}
	if n.ThreadsPerCore == 0 {
		n.ThreadsPerCore = 1
	}
	if n.CPUs == 0 {
		n.CPUs = n.Sockets * n.CoresPerSocket * n.ThreadsPerCore
	}
	n.MemMB = int(num(dig(inv, "memory", "total_bytes")) / (1024 * 1024))
	// count real accelerators only (NVIDIA, AMD, or PCI class 0302 "3D controller"); emulated VGA is not a GPU
	for _, g := range toList(inv["gpus"]) {
		vid, cls := str(dig(g, "vendor_id")), str(dig(g, "class_id"))
		if vid == "10de" || vid == "1002" || cls == "0302" || cls == "1200" {
			n.GPUs++
		}
	}
	for _, h := range toList(dig(inv, "infiniband", "hcas")) {
		if g := str(dig(h, "node_guid")); g != "" {
			n.IBGUIDs = append(n.IBGUIDs, g)
		}
	}
	n.Tags = effectiveTags(m)
	n.Manufacturer, n.Product, n.OS = deref(m.Manufacturer), deref(m.Product), deref(m.OS)
	n.Building, n.Room, n.Rack, n.Unit, n.AssetTag = m.Location.Building, m.Location.Room, m.Location.Rack, m.Location.Unit, m.Location.AssetTag
	n.Disks = int(num(dig(inv, "storage", "disk_count")))
	for _, d := range toList(dig(inv, "storage", "disks")) {
		if strings.HasPrefix(str(dig(d, "name")), "nvme") || str(dig(d, "tran")) == "nvme" {
			n.NVMe = true
		}
	}
	seen := map[string]bool{}
	for _, p := range toList(inv["pci"]) {
		if v := str(dig(p, "vendor_id")); v != "" && !seen[v] {
			seen[v] = true
			n.PCIVendors = append(n.PCIVendors, v)
		}
	}
	return n
}

var tmplFuncs = template.FuncMap{
	"join":    strings.Join,
	"upper":   strings.ToUpper,
	"lower":   strings.ToLower,
	"replace": strings.ReplaceAll,
	"default": func(d, v any) any {
		if v == nil || v == "" || v == 0 || v == false {
			return d
		}
		return v
	},
	"mul": func(a, b int) int { return a * b },
	"sub": func(a, b int) int { return a - b },
	"pct": func(v, p int) int { return v * p / 100 },
	"seq": func(n int) []int {
		n = max(0, min(n, 4096)) // match rules run on every commissioning: no unbounded loops
		s := make([]int, n)
		for i := range s {
			s[i] = i
		}
		return s
	},
	"json":      func(v any) string { b, _ := json.Marshal(v); return string(b) },
	"has":       hasTag,
	"contains":  func(s, sub string) bool { return strings.Contains(s, sub) },
	"hasPrefix": strings.HasPrefix,
	"matches":   func(pattern, s string) bool { ok, _ := regexp.MatchString(pattern, s); return ok },
	"gte":       func(a, b int) bool { return a >= b },
}

func (a *App) hostKeyFor(ctx context.Context, m *Machine) (priv, pub string, err error) {
	if m.hostKey != nil {
		if pemStr, err := a.sealer.Open(m.hostKey, m.ID); err == nil {
			priv = pemStr
		}
	}
	if priv == "" {
		pubk, privk, err := ed25519.GenerateKey(nil)
		if err != nil {
			return "", "", err
		}
		block, err := ssh.MarshalPrivateKey(privk, "rollcall "+m.ID)
		if err != nil {
			return "", "", err
		}
		priv = string(pem.EncodeToMemory(block))
		_ = pubk
		sealed, err := a.sealer.Seal(priv, m.ID)
		if err != nil {
			return "", "", err
		}
		if err := a.store.SetHostKey(ctx, m.ID, sealed); err != nil {
			return "", "", err
		}
	}
	signer, err := ssh.ParsePrivateKey([]byte(priv))
	if err != nil {
		return "", "", err
	}
	pub = strings.TrimSpace(string(ssh.MarshalAuthorizedKey(signer.PublicKey())))
	return priv, pub, nil
}

func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) {
	p, err := a.store.Profile(ctx, deref(m.Profile))
	if err != nil {
		return nil, fmt.Errorf("profile %q: %w", deref(m.Profile), err)
	}
	ms, err := a.store.List(ctx)
	if err != nil {
		return nil, err
	}
	var nodes []NodeCtx
	for _, x := range ms {
		if x.Profile != nil && x.NodeName != nil {
			nodes = append(nodes, nodeCtx(x))
		}
	}
	sort.Slice(nodes, func(i, j int) bool { return nodes[i].Name < nodes[j].Name })
	me := nodeCtx(m)
	if me.Name != "" {
		if me.HostKey, me.HostKeyPub, err = a.hostKeyFor(ctx, m); err != nil {
			return nil, err
		}
	}
	vars := map[string]any{}
	for k, v := range a.cfg.ClusterVars {
		vars[k] = v
	}
	for k, v := range p.Vars {
		vars[k] = v
	}
	overlays := append([]string{}, p.Overlays...)
	if defs, err := a.store.Tags(ctx); err == nil {
		for _, t := range defs {
			if hasTag(me.Tags, t.Name) {
				for _, o := range t.Overlays {
					if !hasTag(overlays, o) {
						overlays = append(overlays, o)
					}
				}
			}
		}
	}
	caPEM := ""
	if a.ca != nil {
		caPEM = string(a.ca.TrustPEM) // during a root rotation both
	}
	return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, CA: caPEM, Vars: vars, Server: a.urlFor(m),
		Now: time.Now().Format(time.RFC3339)}, nil
}

// renderOverlay produces a tar of every overlay dir of the node's profile for the stage.
func (a *App) renderOverlay(ctx context.Context, m *Machine, stage string) ([]byte, error) {
	if stage != "system" && stage != "runtime" {
		return nil, fmt.Errorf("stage must be system or runtime")
	}
	oc, err := a.buildCtx(ctx, m)
	if err != nil {
		return nil, err
	}
	var buf bytes.Buffer
	tw := tar.NewWriter(&buf)
	seen := map[string]bool{}
	for _, name := range oc.Overlays {
		root := filepath.Join(a.cfg.OverlaysDir, name, stage)
		if st, err := os.Stat(root); err != nil || !st.IsDir() {
			continue
		}
		err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
			if err != nil {
				return err
			}
			rel, _ := filepath.Rel(root, p)
			if rel == "." {
				return nil
			}
			info, err := d.Info()
			if err != nil {
				return err
			}
			if info.Mode()&fs.ModeSymlink != 0 { // follow symlinks (e.g. runtime -> system templates)
				if info, err = os.Stat(p); err != nil {
					return err
				}
			}
			hdr := &tar.Header{Name: rel, Mode: int64(secretMode(rel, info.Mode().Perm())), ModTime: info.ModTime(), Uname: "root", Gname: "root"}
			if d.IsDir() {
				if seen[rel] {
					return nil
				}
				seen[rel] = true
				hdr.Typeflag, hdr.Name = tar.TypeDir, rel+"/"
				return tw.WriteHeader(hdr)
			}
			data, err := os.ReadFile(p)
			if err != nil {
				return err
			}
			if strings.HasSuffix(rel, ".tmpl") {
				t, err := template.New(rel).Funcs(tmplFuncs).Option("missingkey=zero").Parse(string(data))
				if err != nil {
					return fmt.Errorf("%s: %w", p, err)
				}
				var out bytes.Buffer
				if err := t.Execute(&out, oc); err != nil {
					return fmt.Errorf("%s: %w", p, err)
				}
				data, hdr.Name = out.Bytes(), strings.TrimSuffix(rel, ".tmpl")
			}
			if seen[hdr.Name] { // later overlays override earlier ones; tar extraction keeps the last entry
				log.Printf("overlay %s: %s overrides an earlier file", name, hdr.Name)
			}
			seen[hdr.Name] = true
			hdr.Typeflag, hdr.Size = tar.TypeReg, int64(len(data))
			if err := tw.WriteHeader(hdr); err != nil {
				return err
			}
			_, err = tw.Write(data)
			return err
		})
		if err != nil {
			return nil, err
		}
	}
	if err := tw.Close(); err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// secretMode tightens permissions for files git cannot carry modes for.
func secretMode(rel string, mode fs.FileMode) fs.FileMode {
	base := strings.TrimSuffix(path.Base(rel), ".tmpl")
	switch {
	case base == "munge.key":
		return 0o400
	case strings.HasPrefix(base, "ssh_host_") && !strings.HasSuffix(base, ".pub"),
		base == "authorized_keys", strings.HasSuffix(base, ".key"), strings.HasSuffix(base, ".secret"):
		return 0o600
	}
	return mode
}

// renderFile renders one template path of a profile with a cluster-wide context
// (used for files the controller needs too, e.g. slurm.conf).
func (a *App) renderFile(ctx context.Context, profile, stage, rel string) ([]byte, error) {
	p, err := a.store.Profile(ctx, profile)
	if err != nil {
		return nil, err
	}
	ms, err := a.store.List(ctx)
	if err != nil {
		return nil, err
	}
	var m *Machine
	for _, x := range ms {
		if deref(x.Profile) == profile {
			m = x
			break
		}
	}
	if m == nil { // no node yet: synthesize an empty one so cluster-level templates still render
		m = &Machine{ID: "none", Profile: &p.Name}
	}
	oc, err := a.buildCtx(ctx, m)
	if err != nil {
		return nil, err
	}
	for _, name := range oc.Overlays {
		for _, cand := range []string{rel + ".tmpl", rel} {
			fp := filepath.Join(a.cfg.OverlaysDir, name, stage, cand)
			data, err := os.ReadFile(fp)
			if err != nil {
				continue
			}
			if !strings.HasSuffix(cand, ".tmpl") {
				return data, nil
			}
			t, err := template.New(rel).Funcs(tmplFuncs).Option("missingkey=zero").Parse(string(data))
			if err != nil {
				return nil, err
			}
			var out bytes.Buffer
			if err := t.Execute(&out, oc); err != nil {
				return nil, err
			}
			return out.Bytes(), nil
		}
	}
	return nil, errNotFound
}

// ---------------------------------------------------------------- PXE + dnsmasq

const statelessTmpl = `# generated by rollcall %s
# node %s  machine %s  profile %s
DEFAULT menu.c32
PROMPT 0
TIMEOUT 30
MENU TITLE %s - stateless (%s)

LABEL node
  MENU LABEL Boot %s (%s) from RAM
  MENU DEFAULT
  KERNEL images/%s/vmlinuz
  INITRD images/%s/initrd.img
  APPEND %s

LABEL recommission
  MENU LABEL Re-run commissioning
  KERNEL rollcall/vmlinuz
  INITRD rollcall/initrd.img
  APPEND %s

LABEL local
  MENU LABEL Boot from local disk
  LOCALBOOT 0
`

func (a *App) statelessAppend(m *Machine, p *Profile, token string) string {
	base := a.urlFor(m)
	parts := []string{"boot=live", "components",
		"fetch=" + base + "/images/" + p.Image + "/filesystem.squashfs",
		"hostname=" + deref(m.NodeName),
		"live-config.nocomponents=user-setup,sudo",
		"rollcall.server=" + base, "rollcall.node=" + token}
	parts = append(parts, a.agentTLSArgs(base)...)
	if p.Append != "" {
		parts = append(parts, p.Append)
	}
	return strings.Join(parts, " ")
}

func (a *App) writePXEStateless(m *Machine, p *Profile, token string) error {
	body := fmt.Sprintf(statelessTmpl, time.Now().Format(time.RFC3339), deref(m.NodeName), m.ID, p.Name,
		deref(m.NodeName), p.Name, deref(m.NodeName), p.Name, p.Image, p.Image, a.statelessAppend(m, p, token),
		a.appendFor(a.urlFor(m)))
	return a.writePXEBody(m, body)
}

// writePXEAny picks the right per-MAC content for the machine's current state.
func (a *App) writePXEAny(ctx context.Context, m *Machine) error {
	if m.Profile != nil && m.nodeToken != nil {
		p, err := a.store.Profile(ctx, *m.Profile)
		if err != nil {
			return err
		}
		token, err := a.sealer.Open(m.nodeToken, m.ID)
		if err != nil {
			return err
		}
		return a.writePXEStateless(m, p, token)
	}
	if m.State == "deployed" {
		return a.writePXEDeployed(m)
	}
	return a.writePXE(m)
}

// ---------------------------------------------------------------- handlers

func (a *App) handleProfiles(w http.ResponseWriter, r *http.Request) {
	switch r.Method {
	case http.MethodGet:
		ps, err := a.store.Profiles(r.Context())
		if err != nil {
			fail(w, 500, err)
			return
		}
		writeJSON(w, 200, ps)
	case http.MethodPost:
		var p Profile
		if err := json.NewDecoder(r.Body).Decode(&p); err != nil || p.Name == "" || p.Image == "" {
			fail(w, 400, fmt.Errorf("name and image are required"))
			return
		}
		if strings.ContainsAny(p.Name, "/ \t") || strings.ContainsAny(p.Image, "/ \t") {
			fail(w, 400, fmt.Errorf("name/image must be plain identifiers"))
			return
		}
		if err := a.store.PutProfile(r.Context(), p); err != nil {
			fail(w, 500, err)
			return
		}
		// profile changes affect every assigned node's PXE entry and overlays
		ms, _ := a.store.List(r.Context())
		for _, m := range ms {
			if deref(m.Profile) == p.Name {
				if err := a.writePXEAny(r.Context(), m); err != nil {
					log.Printf("pxe rewrite %s: %v", m.ID, err)
				}
			}
		}
		writeJSON(w, 200, p)
	}
}

func (a *App) handleProfileDelete(w http.ResponseWriter, r *http.Request) {
	if err := a.store.DeleteProfile(r.Context(), r.PathValue("name")); err != nil {
		fail(w, 409, err)
		return
	}
	writeJSON(w, 200, obj{"deleted": r.PathValue("name")})
}

func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) {
	m, err := a.store.Get(r.Context(), r.PathValue("id"))
	if err != nil {
		fail(w, 500, err)
		return
	}
	var req struct {
		Profile string `json:"profile"`
		Name    string `json:"name"`
		Boot    bool   `json:"boot"`
	}
	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Profile == "" {
		fail(w, 400, fmt.Errorf("profile is required"))
		return
	}
	req.Name = strings.ToLower(req.Name)
	if req.Name != "" && !validDomain(req.Name) {
		fail(w, 400, fmt.Errorf("name %q is not a host name (letters, digits, hyphens)", req.Name))
		return
	}
	p, err := a.store.Profile(r.Context(), req.Profile)
	if err != nil {
		fail(w, 404, fmt.Errorf("profile %s: %w", req.Profile, err))
		return
	}
	if _, err := os.Stat(filepath.Join(a.cfg.ImagesDir, p.Image, "filesystem.squashfs")); err != nil {
		fail(w, 409, fmt.Errorf("image %s is not built (%s)", p.Image, err))
		return
	}
	// keep the existing name/IPs when re-assigning an already named node
	name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP)
	if name == "" || (req.Name != "" && req.Name != name) {
		if name, mgmt, ib, err = a.allocate(r.Context(), m, p.NameFormat, req.Name); err != nil {
			fail(w, 409, err)
			return
		}
	}
	token := randomToken(16)
	sum := sha256.Sum256([]byte(token))
	sealed, err := a.sealer.Seal(token, m.ID)
	if err != nil {
		fail(w, 500, err)
		return
	}
	if err := a.store.Assign(r.Context(), m.ID, nodeAssignment{Profile: p.Name, Name: name, MgmtIP: mgmt, IBIP: ib,
		TokenHash: hex.EncodeToString(sum[:]), Token: sealed}); err != nil {
		fail(w, 500, err)
		return
	}
	m, _ = a.store.Get(r.Context(), m.ID)
	if err := a.writePXEStateless(m, p, token); err != nil {
		fail(w, 500, err)
		return
	}
	if err := a.exportDnsmasq(r.Context()); err != nil {
		log.Printf("dnsmasq export: %v", err)
	}
	_ = a.store.AddEvent(r.Context(), m.ID, "assigned", fmt.Sprintf("profile=%s name=%s mgmt=%s ib=%s", p.Name, name, mgmt, ib))
	out := obj{"id": m.ID, "state": "stateless", "profile": p.Name, "name": name, "mgmt_ip": mgmt, "ib_ip": ib}
	if req.Boot {
		out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m)
	}
	log.Printf("assigned %s -> %s as %s (%s)", m.ID, p.Name, name, mgmt)
	writeJSON(w, 200, out)
}

// pxeBoot sets bootdev pxe and power-cycles (or powers on) through the BMC.
func (a *App) pxeBoot(ctx context.Context, m *Machine) (string, bool) {
	o1, e1 := a.power(ctx, m, "bootdev", "pxe")
	if errors.Is(e1, errNoBMC) {
		return e1.Error(), false
	}
	st, _ := a.power(ctx, m, "status", "")
	var o2 string
	var e2 error
	if strings.Contains(st, "off") {
		o2, e2 = a.power(ctx, m, "on", "")
	} else {
		o2, e2 = a.power(ctx, m, "cycle", "")
	}
	detail := strings.TrimSpace(o1 + "\n" + o2)
	ev := "pxe boot"
	if e1 != nil || e2 != nil { // a lost or failed power command must not look like success
		ev = "pxe boot failed"
		for _, e := range []error{e1, e2} {
			if e != nil {
				detail = strings.TrimSpace(detail + "\n" + e.Error())
			}
		}
	}
	_ = a.store.AddEvent(ctx, m.ID, ev, detail)
	return detail, e1 == nil && e2 == nil
}

func (a *App) handleUnassign(w http.ResponseWriter, r *http.Request) {
	m, err := a.store.Get(r.Context(), r.PathValue("id"))
	if err != nil {
		fail(w, 500, err)
		return
	}
	_ = a.store.FreeMachineIPs(r.Context(), m.ID, "auto")
	a.kickIPAM()
	if err := a.store.Unassign(r.Context(), m.ID); err != nil {
		fail(w, 500, err)
		return
	}
	m, _ = a.store.Get(r.Context(), m.ID)
	_ = a.writePXE(m)
	_ = a.exportDnsmasq(r.Context())
	_ = a.store.AddEvent(r.Context(), m.ID, "unassigned", "")
	writeJSON(w, 200, obj{"id": m.ID, "state": "ready"})
}

func (a *App) serveOverlay(w http.ResponseWriter, r *http.Request, m *Machine) {
	stage := r.URL.Query().Get("stage")
	if stage == "" {
		stage = "system"
	}
	data, err := a.renderOverlay(r.Context(), m, stage)
	if err != nil {
		fail(w, 500, err)
		return
	}
	sum := sha256.Sum256(data)
	w.Header().Set("Content-Type", "application/x-tar")
	w.Header().Set("ETag", `"`+hex.EncodeToString(sum[:8])+`"`)
	w.Header().Set("X-Rollcall-Node-Name", deref(m.NodeName))
	w.Header().Set("Content-Length", strconv.Itoa(len(data)))
	_, _ = w.Write(data)
}

// handleNodeOverlay is what a booted node calls; the token comes from its kernel cmdline.
func (a *App) handleNodeOverlay(w http.ResponseWriter, r *http.Request) {
	tok := r.Header.Get("X-Rollcall-Node")
	if tok == "" {
		fail(w, 401, errors.New("X-Rollcall-Node required"))
		return
	}
	sum := sha256.Sum256([]byte(tok))
	m, err := a.store.ByNodeToken(r.Context(), hex.EncodeToString(sum[:]))
	if err != nil {
		fail(w, 401, errors.New("unknown node token"))
		return
	}
	if err := a.tokenSource(r.Context(), r, m, nil, "node"); err != nil {
		fail(w, 403, err)
		return
	}
	if m.State != "booted" {
		_ = a.store.AddEvent(r.Context(), m.ID, "booted", r.RemoteAddr)
		log.Printf("node %s (%s) booted from %s", deref(m.NodeName), m.ID, r.RemoteAddr)
	}
	_ = a.store.MarkBooted(r.Context(), m.ID)
	a.serveOverlay(w, r, m)
}

func (a *App) handleAdminOverlay(w http.ResponseWriter, r *http.Request) {
	m, err := a.store.Get(r.Context(), r.PathValue("id"))
	if err != nil {
		fail(w, 500, err)
		return
	}
	if m.Profile == nil {
		fail(w, 409, errors.New("machine has no profile"))
		return
	}
	a.serveOverlay(w, r, m)
}

func (a *App) handleRender(w http.ResponseWriter, r *http.Request) {
	stage := r.URL.Query().Get("stage")
	if stage == "" {
		stage = "system"
	}
	rel := path.Clean("/" + r.PathValue("path"))[1:] // rooted first, so ".." cannot climb out of the overlay
	if (stage != "system" && stage != "runtime") || rel == "" {
		fail(w, 400, errors.New("stage is system or runtime, path a file in the overlay"))
		return
	}
	data, err := a.renderFile(r.Context(), r.PathValue("profile"), stage, rel)
	if err != nil {
		fail(w, 500, err)
		return
	}
	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
	_, _ = w.Write(data)
}

func (a *App) handleImages(w http.ResponseWriter, r *http.Request) {
	ents, err := os.ReadDir(a.cfg.ImagesDir)
	if err != nil {
		writeJSON(w, 200, []any{})
		return
	}
	out := []obj{}
	for _, e := range ents {
		if !e.IsDir() {
			continue
		}
		info := obj{"name": e.Name()}
		for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} {
			if st, err := os.Stat(filepath.Join(a.cfg.ImagesDir, e.Name(), f)); err == nil {
				info[f] = st.Size()
				info["built"] = st.ModTime()
			}
		}
		info["complete"] = info["filesystem.squashfs"] != nil && info["vmlinuz"] != nil && info["initrd.img"] != nil
		out = append(out, info)
	}
	writeJSON(w, 200, out)
}