Stateless (Warewulf-style) provisioning: profiles, node pools, overlays, dnsmasq export, compute image

Profiles bind a RAM-booted root image to overlay templates; assigning a
machine allocates a node name and management IP, writes the per-MAC PXE
entry with a per-node token and exports static leases for dnsmasq. Nodes
pull rendered overlays (hosts, slurm.conf from inventory, gres.conf, SSH
host key, munge key) through rollcall-overlay.service; the runtime refresh
timer is the heartbeat. Adds the compute-debian13 image definition and a
generic live-build root image script, CLI and web UI for profiles/assign.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

README.md

@@ -126,6 +126,37 @@ for a hint (`bmc_hints`) and verifies it out-of-band.
 Vendor quirks (iDRAC/iLO password policies, missing empty slots) are reported in
 `bmc.errors` and visible in the UI.
 
+## Stateless nodes (Warewulf-style)
+
+For HPC compute nodes Rollcall can boot a root image straight into RAM and
+dress it with per-node files, instead of installing an OS on disk.
+
+* **Images** live in `images_dir/<name>/{vmlinuz,initrd.img,filesystem.squashfs}`
+  and are built with `images/build-rootfs.sh <name>` from
+  `images/<name>/{packages.list,hook.chroot,includes/}` (live-build, same
+  toolchain as the commissioning image). `images/compute-debian13` is a
+  Debian 13 compute node: slurmd, munge, rdma-core, NFS client, firmware.
+* **Profiles** bind an image to overlays, extra kernel parameters, a node name
+  format (`node%03d`) and variables: `rollcall profile add compute -image compute-debian13 -overlays compute`.
+* **Assigning** a commissioned machine to a profile allocates a node name and
+  IPs from `node_pools`, writes a per-MAC PXE entry that boots the image with
+  `hostname=`, `rollcall.server=` and a per-node token, and exports static
+  DHCP leases plus host names for dnsmasq (`dnsmasq_dir`, re-read on HUP via
+  `rollcall-dnsmasq.path`). `rollcall assign <id> compute -boot` also
+  power-cycles the machine through its BMC.
+* **Overlays** are directory trees under `overlays_dir/<overlay>/{system,runtime}`;
+  files ending in `.tmpl` are Go templates rendered with the node, all nodes of
+  the cluster, the profile and `cluster_vars`. The `compute` overlay generates
+  `/etc/hosts`, `slurm.conf` (node lines from the commissioning inventory:
+  sockets, cores, threads, memory, GPUs), `gres.conf`, a persistent SSH host
+  key per node (sealed in the database), root `authorized_keys` and the munge
+  key. Inside the image `rollcall-overlay.service` applies the system overlay
+  before munge/slurmd/ssh start; a timer refreshes the runtime overlay every
+  five minutes, which doubles as the node heartbeat (`booted` state, last seen).
+* The controller renders the same files: `rollcall render compute etc/slurm/slurm.conf > /etc/slurm/slurm.conf`.
+
+States: `ready → stateless → booted`; `rollcall unassign` releases name and IPs.
+
 ## Lab
 
 `lab/` documents a Proxmox SDN test setup: an isolated vnet, a PXE server VM

cmd/rollcall/client.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"archive/tar"
 	"bytes"
 	"encoding/json"
 	"flag"
@@ -8,6 +9,7 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"path/filepath"
 	"strings"
 	"time"
 )
@@ -97,8 +99,8 @@ func runClient(cmd string, args []string) error {
 		if err := c.do("GET", "/api/v1/machines", nil, &ms); err != nil {
 			return err
 		}
-		f := "%-16s %-12s %-28s %-16s %4s %7s %3s %7s %3s %2s %3s  %s\n"
-		fmt.Printf(f, "ID", "STATE", "MODEL", "SERIAL", "CPU", "RAM", "DSK", "STORAGE", "NIC", "IB", "GPU", "BMC")
+		f := "%-16s %-12s %-10s %-10s %-14s %-24s %4s %7s %3s %7s %3s %2s %3s  %s\n"
+		fmt.Printf(f, "ID", "STATE", "NODE", "PROFILE", "MGMT-IP", "MODEL", "CPU", "RAM", "DSK", "STORAGE", "NIC", "IB", "GPU", "BMC")
 		for _, m := range ms {
 			bmc := "-"
 			if m.BMC != nil {
@@ -110,7 +112,7 @@ func runClient(cmd string, args []string) error {
 				bmc = fmt.Sprintf("%s (%s,%s)", str(b["address"]), str(b["source"]), ok)
 			}
 			model := strings.TrimSpace(deref(m.Manufacturer) + " " + deref(m.Product))
-			fmt.Printf(f, m.ID, m.State, trunc(model, 28), trunc(deref(m.Serial), 16), m.CPUs, human(m.MemoryBytes),
+			fmt.Printf(f, m.ID, m.State, deref(m.NodeName), deref(m.Profile), deref(m.MgmtIP), trunc(model, 24), m.CPUs, human(m.MemoryBytes),
 				fmt.Sprint(int(m.Disks)), human(m.StorageBytes), fmt.Sprint(m.NICs), fmt.Sprint(m.IBHCAs), fmt.Sprint(m.GPUs), bmc)
 		}
 	case "show":
@@ -175,10 +177,172 @@ func runClient(cmd string, args []string) error {
 			return err
 		}
 		fmt.Println("deleted", str(d["deleted"]))
+	default:
+		return c.statelessCmd(cmd, args)
 	}
 	return nil
 }
 
+func (c *client) statelessCmd(cmd string, args []string) error {
+	switch cmd {
+	case "images":
+		var imgs []obj
+		if err := c.do("GET", "/api/v1/images", nil, &imgs); err != nil {
+			return err
+		}
+		fmt.Printf("%-24s %-9s %10s %10s %10s  %s\n", "IMAGE", "COMPLETE", "KERNEL", "INITRD", "SQUASHFS", "BUILT")
+		for _, i := range imgs {
+			fmt.Printf("%-24s %-9v %10s %10s %10s  %s\n", str(i["name"]), i["complete"], human(num(i["vmlinuz"])),
+				human(num(i["initrd.img"])), human(num(i["filesystem.squashfs"])), str(i["built"]))
+		}
+	case "profile":
+		if len(args) < 1 {
+			usage()
+		}
+		switch args[0] {
+		case "list":
+			var ps []Profile
+			if err := c.do("GET", "/api/v1/profiles", nil, &ps); err != nil {
+				return err
+			}
+			fmt.Printf("%-14s %-22s %-24s %-12s %s\n", "PROFILE", "IMAGE", "OVERLAYS", "NAMES", "APPEND")
+			for _, p := range ps {
+				fmt.Printf("%-14s %-22s %-24s %-12s %s\n", p.Name, p.Image, strings.Join(p.Overlays, ","), p.NameFormat, p.Append)
+			}
+		case "add":
+			if len(args) < 2 {
+				usage()
+			}
+			fs := flag.NewFlagSet("profile add", flag.ExitOnError)
+			img := fs.String("image", "", "image directory name (required)")
+			app := fs.String("append", "", "extra kernel parameters")
+			ov := fs.String("overlays", "", "comma-separated overlay names")
+			nf := fs.String("name-format", "node%03d", "node name format")
+			vars := fs.String("vars", "{}", "profile variables as JSON")
+			_ = fs.Parse(args[2:])
+			p := Profile{Name: args[1], Image: *img, Append: *app, NameFormat: *nf}
+			if *ov != "" {
+				p.Overlays = strings.Split(*ov, ",")
+			}
+			if err := json.Unmarshal([]byte(*vars), &p.Vars); err != nil {
+				return fmt.Errorf("-vars: %w", err)
+			}
+			var out Profile
+			if err := c.do("POST", "/api/v1/profiles", p, &out); err != nil {
+				return err
+			}
+			fmt.Printf("profile %s: image=%s overlays=%v\n", out.Name, out.Image, out.Overlays)
+		case "rm":
+			if len(args) < 2 {
+				usage()
+			}
+			var d obj
+			if err := c.do("DELETE", "/api/v1/profiles/"+args[1], nil, &d); err != nil {
+				return err
+			}
+			fmt.Println("deleted", str(d["deleted"]))
+		default:
+			usage()
+		}
+	case "assign":
+		if len(args) < 2 {
+			usage()
+		}
+		fs := flag.NewFlagSet("assign", flag.ExitOnError)
+		name := fs.String("name", "", "node name (default: next free from the profile's name format)")
+		boot := fs.Bool("boot", false, "set bootdev pxe and power-cycle via BMC")
+		_ = fs.Parse(args[2:])
+		var d obj
+		if err := c.do("POST", "/api/v1/machines/"+args[0]+"/assign", obj{"profile": args[1], "name": *name, "boot": *boot}, &d); err != nil {
+			return err
+		}
+		fmt.Printf("%s -> %s as %s mgmt=%s ib=%s\n", str(d["id"]), str(d["profile"]), str(d["name"]), str(d["mgmt_ip"]), str(d["ib_ip"]))
+		if d["ipmi"] != nil {
+			fmt.Println(str(d["ipmi"]))
+		}
+	case "unassign":
+		if len(args) < 1 {
+			usage()
+		}
+		var d obj
+		if err := c.do("POST", "/api/v1/machines/"+args[0]+"/unassign", nil, &d); err != nil {
+			return err
+		}
+		fmt.Printf("%s -> %s\n", str(d["id"]), str(d["state"]))
+	case "overlay":
+		if len(args) < 1 {
+			usage()
+		}
+		fs := flag.NewFlagSet("overlay", flag.ExitOnError)
+		stage := fs.String("stage", "system", "system or runtime")
+		extract := fs.String("extract", "", "extract into this directory instead of listing")
+		_ = fs.Parse(args[1:])
+		data, err := c.raw("GET", "/api/v1/machines/"+args[0]+"/overlay?stage="+*stage)
+		if err != nil {
+			return err
+		}
+		tr := tar.NewReader(bytes.NewReader(data))
+		for {
+			h, err := tr.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return err
+			}
+			if *extract == "" {
+				fmt.Printf("%s %8d %s\n", os.FileMode(h.Mode).String(), h.Size, h.Name)
+				continue
+			}
+			dst := filepath.Join(*extract, h.Name)
+			if h.Typeflag == tar.TypeDir {
+				_ = os.MkdirAll(dst, 0o755)
+				continue
+			}
+			_ = os.MkdirAll(filepath.Dir(dst), 0o755)
+			f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(h.Mode))
+			if err != nil {
+				return err
+			}
+			_, _ = io.Copy(f, tr)
+			f.Close()
+		}
+	case "render":
+		if len(args) < 2 {
+			usage()
+		}
+		fs := flag.NewFlagSet("render", flag.ExitOnError)
+		stage := fs.String("stage", "system", "system or runtime")
+		_ = fs.Parse(args[2:])
+		data, err := c.raw("GET", "/api/v1/render/"+args[0]+"/"+strings.TrimPrefix(args[1], "/")+"?stage="+*stage)
+		if err != nil {
+			return err
+		}
+		os.Stdout.Write(data)
+	default:
+		usage()
+	}
+	return nil
+}
+
+func (c *client) raw(method, path string) ([]byte, error) {
+	req, err := http.NewRequest(method, c.base+path, nil)
+	if err != nil {
+		return nil, err
+	}
+	req.Header.Set("Authorization", "Bearer "+c.token)
+	resp, err := c.http.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	data, _ := io.ReadAll(resp.Body)
+	if resp.StatusCode >= 400 {
+		return nil, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(data)))
+	}
+	return data, nil
+}
+
 func trunc(s string, n int) string {
 	if len(s) > n {
 		return s[:n]

cmd/rollcall/config.go

@@ -4,6 +4,8 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
+	"strings"
 )
 
 // Config is /etc/rollcall/config.json.
@@ -18,6 +20,14 @@ type Config struct {
 	CommissionAppend string    `json:"commission_append"` // kernel cmdline used in generated pxelinux files
 	IPMITool         string    `json:"ipmitool"`          // path, default "ipmitool"
 	BMCHints         []BMCHint `json:"bmc_hints"`         // out-of-band BMC mapping for machines without in-band IPMI
+
+	// stateless provisioning
+	ServerURL   string            `json:"server_url"`   // how nodes reach this server, e.g. http://10.213.77.1
+	ImagesDir   string            `json:"images_dir"`   // <images_dir>/<image>/{vmlinuz,initrd.img,filesystem.squashfs}, served under /images/
+	OverlaysDir string            `json:"overlays_dir"` // <overlays_dir>/<overlay>/{system,runtime}/... (Go templates with .tmpl suffix)
+	DnsmasqDir  string            `json:"dnsmasq_dir"`  // hosts.dhcp + hosts written here for dnsmasq (dhcp-hostsfile / addn-hosts); "" disables
+	NodePools   map[string]string `json:"node_pools"`   // {"mgmt": "10.0.0.50-10.0.0.99", "ib": "10.1.0.10-10.1.0.250"}
+	ClusterVars map[string]any    `json:"cluster_vars"` // available to overlay templates as .Vars (profile vars override)
 }
 
 // BMCHint maps a machine MAC to a known BMC (virtual lab nodes).
@@ -44,5 +54,19 @@ func loadConfig(path string) (*Config, error) {
 			return nil, fmt.Errorf("%s: %s is required", path, k.n)
 		}
 	}
+	if c.ServerURL == "" {
+		host, _, _ := strings.Cut(c.Listen, ":")
+		if host == "" || host == "0.0.0.0" {
+			return nil, fmt.Errorf("%s: server_url is required when listen is not a specific address", path)
+		}
+		c.ServerURL = "http://" + c.Listen
+	}
+	c.ServerURL = strings.TrimRight(c.ServerURL, "/")
+	if c.ImagesDir == "" {
+		c.ImagesDir = filepath.Join(filepath.Dir(c.ImageDir), "images")
+	}
+	if c.OverlaysDir == "" {
+		c.OverlaysDir = "/etc/rollcall/overlays"
+	}
 	return c, nil
 }

cmd/rollcall/go.mod

@@ -2,13 +2,16 @@ module rollcall
 
 go 1.24
 
-require github.com/jackc/pgx/v5 v5.7.5
+require (
+	github.com/jackc/pgx/v5 v5.7.5
+	golang.org/x/crypto v0.37.0
+)
 
 require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
 	github.com/jackc/puddle/v2 v2.2.2 // indirect
-	golang.org/x/crypto v0.37.0 // indirect
 	golang.org/x/sync v0.13.0 // indirect
+	golang.org/x/sys v0.32.0 // indirect
 	golang.org/x/text v0.24.0 // indirect
 )

cmd/rollcall/go.sum

@@ -20,6 +20,10 @@ golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
 golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
 golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
 golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
+golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
+golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
 golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
 golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

cmd/rollcall/main.go

@@ -30,6 +30,12 @@ func usage() {
   rollcall bootdev ID pxe|disk|cdrom|bios|none
   rollcall recommission ID [-now]
   rollcall delete ID
+  rollcall images
+  rollcall profile list | add NAME -image IMG [-append ARGS] [-overlays a,b] [-name-format node%03d] [-vars JSON] | rm NAME
+  rollcall assign ID PROFILE [-name node001] [-boot]
+  rollcall unassign ID
+  rollcall overlay ID [-stage system|runtime] [-extract DIR]
+  rollcall render PROFILE PATH [-stage system|runtime]
 env: ROLLCALL_URL (default from config listen), ROLLCALL_TOKEN, ROLLCALL_CONFIG`)
 	os.Exit(2)
 }
@@ -50,7 +56,8 @@ func main() {
 		}
 	case "keygen":
 		fmt.Println(randomToken(24))
-	case "list", "show", "events", "power", "bootdev", "recommission", "delete":
+	case "list", "show", "events", "power", "bootdev", "recommission", "delete",
+		"images", "profile", "assign", "unassign", "overlay", "render":
 		if err := runClient(cmd, args); err != nil {
 			fmt.Fprintln(os.Stderr, "error:", err)
 			os.Exit(1)

cmd/rollcall/server.go

@@ -81,7 +81,7 @@ func logging(next http.Handler) http.Handler {
 		start := time.Now()
 		rw := &statusWriter{ResponseWriter: w, code: 200}
 		next.ServeHTTP(rw, r)
-		if !strings.HasPrefix(r.URL.Path, "/rollcall/") || rw.code >= 400 {
+		if (!strings.HasPrefix(r.URL.Path, "/rollcall/") && !strings.HasPrefix(r.URL.Path, "/images/")) || rw.code >= 400 {
 			log.Printf("%s %s %s %d %s", r.RemoteAddr, r.Method, r.URL.RequestURI(), rw.code, time.Since(start).Round(time.Millisecond))
 		}
 	})
@@ -110,9 +110,20 @@ func (a *App) routes() http.Handler {
 	mux.HandleFunc("GET /api/v1/whoami", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
 		writeJSON(w, 200, map[string]any{"ok": true})
 	}))
+	// stateless provisioning
+	mux.HandleFunc("GET /api/v1/profiles", a.adminAuth(a.handleProfiles))
+	mux.HandleFunc("POST /api/v1/profiles", a.adminAuth(a.handleProfiles))
+	mux.HandleFunc("DELETE /api/v1/profiles/{name}", a.adminAuth(a.handleProfileDelete))
+	mux.HandleFunc("GET /api/v1/images", a.adminAuth(a.handleImages))
+	mux.HandleFunc("POST /api/v1/machines/{id}/assign", a.adminAuth(a.handleAssign))
+	mux.HandleFunc("POST /api/v1/machines/{id}/unassign", a.adminAuth(a.handleUnassign))
+	mux.HandleFunc("GET /api/v1/machines/{id}/overlay", a.adminAuth(a.handleAdminOverlay))
+	mux.HandleFunc("GET /api/v1/render/{profile}/{path...}", a.adminAuth(a.handleRender))
+	mux.HandleFunc("GET /api/v1/nodes/self/overlay", a.handleNodeOverlay) // node token auth inside
 	// image files for the live boot (no auth: the kernel/initrd/squashfs must be fetchable)
 	imgDir := a.cfg.ImageDir
 	mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir))))
+	mux.Handle("GET /images/", http.StripPrefix("/images/", http.FileServer(http.Dir(a.cfg.ImagesDir))))
 	// embedded web UI
 	sub, _ := fs.Sub(webFS, "web")
 	mux.Handle("GET /", http.FileServer(http.FS(sub)))
@@ -209,6 +220,11 @@ type Summary struct {
 	IBHCAs       int        `json:"ib_hcas"`
 	GPUs         int        `json:"gpus"`
 	BMC          *obj       `json:"bmc"`
+	Profile      *string    `json:"profile"`
+	NodeName     *string    `json:"node_name"`
+	MgmtIP       *string    `json:"mgmt_ip"`
+	IBIP         *string    `json:"ib_ip"`
+	BootedAt     *time.Time `json:"booted_at"`
 }
 
 func num(v any) float64 {
@@ -224,7 +240,8 @@ func summarize(m *Machine) Summary {
 		CPU: str(dig(inv, "cpu", "Model name")), CPUs: str(dig(inv, "cpu", "CPU(s)")),
 		MemoryBytes: num(dig(inv, "memory", "total_bytes")), Disks: num(dig(inv, "storage", "disk_count")),
 		StorageBytes: num(dig(inv, "storage", "total_bytes")), NICs: listLen(inv["network"]),
-		IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: listLen(inv["gpus"])}
+		IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: listLen(inv["gpus"]),
+		Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt}
 	if m.BMC != nil {
 		var b obj
 		if json.Unmarshal(m.BMC, &b) == nil {
@@ -275,6 +292,10 @@ func (a *App) writePXE(m *Machine) error {
 	man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial)
 	body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser,
 		m.ID, man, prod, ser, a.cfg.CommissionAppend)
+	return a.writePXEBody(m, body)
+}
+
+func (a *App) writePXEBody(m *Machine, body string) error {
 	for _, mac := range m.MACs {
 		p := a.pxePath(mac)
 		if err := os.WriteFile(p+".tmp", []byte(body), 0o644); err != nil {
@@ -358,7 +379,10 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	_ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail)
-	if err := a.writePXE(m); err != nil {
+	if full, err := a.store.Get(r.Context(), m.ID); err == nil {
+		m = full // carries profile / node token when this is a re-commission of a stateless node
+	}
+	if err := a.writePXEAny(r.Context(), m); err != nil {
 		log.Printf("pxe write failed for %s: %v", m.ID, err)
 		_ = a.store.AddEvent(r.Context(), m.ID, "pxe-write-failed", err.Error())
 	}

cmd/rollcall/stateless.go

@@ -0,0 +1,883 @@
+package main
+
+// Stateless (Warewulf-style) provisioning: profiles, node naming and IP pools,
+// per-node overlays rendered from templates, dnsmasq host export and the
+// per-MAC PXE entry that boots a profile's root image from RAM.
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"crypto/ed25519"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/hex"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io/fs"
+	"log"
+	"net"
+	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
+	"text/template"
+	"time"
+
+	"golang.org/x/crypto/ssh"
+)
+
+// ---------------------------------------------------------------- profiles
+
+type Profile struct {
+	Name       string         `json:"name"`
+	Image      string         `json:"image"`       // directory under images_dir: vmlinuz, initrd.img, filesystem.squashfs
+	Append     string         `json:"append"`      // extra kernel parameters
+	Overlays   []string       `json:"overlays"`    // directories under overlays_dir
+	NameFormat string         `json:"name_format"` // e.g. node%03d
+	Vars       map[string]any `json:"vars"`
+	Created    time.Time      `json:"created"`
+}
+
+func (s *Store) Profiles(ctx context.Context) ([]Profile, error) {
+	rows, err := s.pool.Query(ctx, `SELECT name, image, coalesce(append,''), overlays, coalesce(name_format,''), vars, created FROM profiles ORDER BY name`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := []Profile{}
+	for rows.Next() {
+		var p Profile
+		var ov, vars []byte
+		if err := rows.Scan(&p.Name, &p.Image, &p.Append, &ov, &p.NameFormat, &vars, &p.Created); err != nil {
+			return nil, err
+		}
+		_ = json.Unmarshal(ov, &p.Overlays)
+		_ = json.Unmarshal(vars, &p.Vars)
+		out = append(out, p)
+	}
+	return out, rows.Err()
+}
+
+func (s *Store) Profile(ctx context.Context, name string) (*Profile, error) {
+	ps, err := s.Profiles(ctx)
+	if err != nil {
+		return nil, err
+	}
+	for i := range ps {
+		if ps[i].Name == name {
+			return &ps[i], nil
+		}
+	}
+	return nil, errNotFound
+}
+
+func (s *Store) PutProfile(ctx context.Context, p Profile) error {
+	if p.Overlays == nil {
+		p.Overlays = []string{}
+	}
+	if p.Vars == nil {
+		p.Vars = map[string]any{}
+	}
+	if p.NameFormat == "" {
+		p.NameFormat = "node%03d"
+	}
+	ov, _ := json.Marshal(p.Overlays)
+	vars, _ := json.Marshal(p.Vars)
+	_, err := s.pool.Exec(ctx, `INSERT INTO profiles(name, image, append, overlays, name_format, vars) VALUES ($1,$2,$3,$4,$5,$6)
+		ON CONFLICT (name) DO UPDATE SET image=$2, append=$3, overlays=$4, name_format=$5, vars=$6`,
+		p.Name, p.Image, p.Append, ov, p.NameFormat, vars)
+	return err
+}
+
+func (s *Store) DeleteProfile(ctx context.Context, name string) error {
+	var n int
+	_ = s.pool.QueryRow(ctx, `SELECT count(*) FROM machines WHERE profile=$1`, name).Scan(&n)
+	if n > 0 {
+		return fmt.Errorf("profile %s is assigned to %d machine(s)", name, n)
+	}
+	tag, err := s.pool.Exec(ctx, `DELETE FROM profiles WHERE name=$1`, name)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() == 0 {
+		return errNotFound
+	}
+	return nil
+}
+
+// ---------------------------------------------------------------- allocation
+
+type nodeAssignment struct {
+	Profile   string
+	Name      string
+	MgmtIP    string
+	IBIP      string
+	TokenHash string
+	Token     []byte // sealed
+}
+
+func (s *Store) Assign(ctx context.Context, id string, n nodeAssignment) error {
+	tag, err := s.pool.Exec(ctx, `UPDATE machines SET profile=$2, node_name=$3, mgmt_ip=$4, ib_ip=$5,
+		node_token_hash=$6, node_token=$7, state='stateless', booted_at=NULL WHERE id=$1`,
+		id, n.Profile, n.Name, nilIfEmpty(n.MgmtIP), nilIfEmpty(n.IBIP), n.TokenHash, n.Token)
+	if err != nil {
+		return err
+	}
+	if tag.RowsAffected() == 0 {
+		return errNotFound
+	}
+	return nil
+}
+
+func (s *Store) Unassign(ctx context.Context, id string) error {
+	_, err := s.pool.Exec(ctx, `UPDATE machines SET profile=NULL, node_name=NULL, mgmt_ip=NULL, ib_ip=NULL,
+		node_token_hash=NULL, node_token=NULL, state='ready', booted_at=NULL WHERE id=$1`, id)
+	return err
+}
+
+func (s *Store) ByNodeToken(ctx context.Context, hash string) (*Machine, error) {
+	return scanMachine(s.pool.QueryRow(ctx, `SELECT `+machineCols+` FROM machines WHERE node_token_hash=$1`, hash), true)
+}
+
+func (s *Store) MarkBooted(ctx context.Context, id string) error {
+	_, err := s.pool.Exec(ctx, `UPDATE machines SET state='booted', booted_at=coalesce(booted_at, now()), last_seen=now() WHERE id=$1`, id)
+	return err
+}
+
+func (s *Store) SetHostKey(ctx context.Context, id string, sealed []byte) error {
+	_, err := s.pool.Exec(ctx, `UPDATE machines SET host_key=$2 WHERE id=$1`, id, sealed)
+	return err
+}
+
+func nilIfEmpty(s string) *string {
+	if s == "" {
+		return nil
+	}
+	return &s
+}
+
+// parsePool turns "10.0.0.50-10.0.0.99" into the ordered list of addresses.
+func parsePool(spec string) ([]string, error) {
+	lo, hi, ok := strings.Cut(strings.TrimSpace(spec), "-")
+	if !ok {
+		return nil, fmt.Errorf("pool %q: want A.B.C.D-A.B.C.D", spec)
+	}
+	a, b := net.ParseIP(strings.TrimSpace(lo)).To4(), net.ParseIP(strings.TrimSpace(hi)).To4()
+	if a == nil || b == nil {
+		return nil, fmt.Errorf("pool %q: bad address", spec)
+	}
+	ai, bi := binary.BigEndian.Uint32(a), binary.BigEndian.Uint32(b)
+	if bi < ai || bi-ai > 65536 {
+		return nil, fmt.Errorf("pool %q: bad range", spec)
+	}
+	out := make([]string, 0, bi-ai+1)
+	for i := ai; i <= bi; i++ {
+		ip := make(net.IP, 4)
+		binary.BigEndian.PutUint32(ip, i)
+		out = append(out, ip.String())
+	}
+	return out, nil
+}
+
+func (a *App) allocate(ctx context.Context, p *Profile, wantName string) (name, mgmt, ib string, err error) {
+	ms, err := a.store.List(ctx)
+	if err != nil {
+		return
+	}
+	usedName, usedIP := map[string]bool{}, map[string]bool{}
+	for _, m := range ms {
+		if m.NodeName != nil {
+			usedName[*m.NodeName] = true
+		}
+		if m.MgmtIP != nil {
+			usedIP[*m.MgmtIP] = true
+		}
+		if m.IBIP != nil {
+			usedIP[*m.IBIP] = true
+		}
+	}
+	name = wantName
+	if name == "" {
+		f := p.NameFormat
+		if f == "" {
+			f = "node%03d"
+		}
+		for i := 1; i < 100000; i++ {
+			if c := fmt.Sprintf(f, i); !usedName[c] {
+				name = c
+				break
+			}
+		}
+	} else if usedName[name] {
+		err = fmt.Errorf("node name %s already in use", name)
+		return
+	}
+	pick := func(pool string) (string, error) {
+		spec := a.cfg.NodePools[pool]
+		if spec == "" {
+			return "", nil
+		}
+		ips, err := parsePool(spec)
+		if err != nil {
+			return "", err
+		}
+		for _, ip := range ips {
+			if !usedIP[ip] {
+				usedIP[ip] = true
+				return ip, nil
+			}
+		}
+		return "", fmt.Errorf("pool %s exhausted", pool)
+	}
+	if mgmt, err = pick("mgmt"); err != nil {
+		return
+	}
+	ib, err = pick("ib")
+	return
+}
+
+// ---------------------------------------------------------------- template context
+
+type NodeCtx struct {
+	ID, Name, Profile, State string
+	BootMAC                  string
+	MACs                     []string
+	MgmtIP, IBIP             string
+	Sockets, CoresPerSocket  int
+	ThreadsPerCore, CPUs     int
+	MemMB                    int
+	GPUs                     int
+	IBGUIDs                  []string
+	HostKey, HostKeyPub      string // only filled for the node being rendered
+}
+
+type overlayCtx struct {
+	Node    NodeCtx
+	Nodes   []NodeCtx // all machines with a profile, sorted by name
+	Profile Profile
+	Vars    map[string]any // cluster_vars overlaid with profile vars
+	Server  string
+	Now     string
+}
+
+func atoi(v any) int {
+	n, _ := strconv.Atoi(strings.TrimSpace(str(v)))
+	return n
+}
+
+func nodeCtx(m *Machine) NodeCtx {
+	var inv obj
+	_ = json.Unmarshal(m.Inventory, &inv)
+	n := NodeCtx{ID: m.ID, Name: deref(m.NodeName), Profile: deref(m.Profile), State: m.State, MACs: m.MACs,
+		MgmtIP: deref(m.MgmtIP), IBIP: deref(m.IBIP)}
+	if len(m.MACs) > 0 {
+		n.BootMAC = m.MACs[0]
+	}
+	n.Sockets = atoi(dig(inv, "cpu", "Socket(s)"))
+	n.CoresPerSocket = atoi(dig(inv, "cpu", "Core(s) per socket"))
+	n.ThreadsPerCore = atoi(dig(inv, "cpu", "Thread(s) per core"))
+	n.CPUs = atoi(dig(inv, "cpu", "CPU(s)"))
+	if n.Sockets == 0 {
+		n.Sockets = 1
+	}
+	if n.CoresPerSocket == 0 {
+		n.CoresPerSocket = max(1, n.CPUs/max(1, n.ThreadsPerCore))
+	}
+	if n.ThreadsPerCore == 0 {
+		n.ThreadsPerCore = 1
+	}
+	if n.CPUs == 0 {
+		n.CPUs = n.Sockets * n.CoresPerSocket * n.ThreadsPerCore
+	}
+	n.MemMB = int(num(dig(inv, "memory", "total_bytes")) / (1024 * 1024))
+	// count real accelerators only (NVIDIA, AMD, or PCI class 0302 "3D controller"); emulated VGA is not a GPU
+	for _, g := range toList(inv["gpus"]) {
+		vid, cls := str(dig(g, "vendor_id")), str(dig(g, "class_id"))
+		if vid == "10de" || vid == "1002" || cls == "0302" || cls == "1200" {
+			n.GPUs++
+		}
+	}
+	for _, h := range toList(dig(inv, "infiniband", "hcas")) {
+		if g := str(dig(h, "node_guid")); g != "" {
+			n.IBGUIDs = append(n.IBGUIDs, g)
+		}
+	}
+	return n
+}
+
+var tmplFuncs = template.FuncMap{
+	"join":    strings.Join,
+	"upper":   strings.ToUpper,
+	"lower":   strings.ToLower,
+	"replace": strings.ReplaceAll,
+	"default": func(d, v any) any {
+		if v == nil || v == "" || v == 0 || v == false {
+			return d
+		}
+		return v
+	},
+	"mul": func(a, b int) int { return a * b },
+	"sub": func(a, b int) int { return a - b },
+	"pct": func(v, p int) int { return v * p / 100 },
+	"seq": func(n int) []int {
+		s := make([]int, n)
+		for i := range s {
+			s[i] = i
+		}
+		return s
+	},
+	"json": func(v any) string { b, _ := json.Marshal(v); return string(b) },
+}
+
+func (a *App) hostKeyFor(ctx context.Context, m *Machine) (priv, pub string, err error) {
+	if m.hostKey != nil {
+		if pemStr, err := a.sealer.Open(m.hostKey, m.ID); err == nil {
+			priv = pemStr
+		}
+	}
+	if priv == "" {
+		pubk, privk, err := ed25519.GenerateKey(nil)
+		if err != nil {
+			return "", "", err
+		}
+		block, err := ssh.MarshalPrivateKey(privk, "rollcall "+m.ID)
+		if err != nil {
+			return "", "", err
+		}
+		priv = string(pem.EncodeToMemory(block))
+		_ = pubk
+		sealed, err := a.sealer.Seal(priv, m.ID)
+		if err != nil {
+			return "", "", err
+		}
+		if err := a.store.SetHostKey(ctx, m.ID, sealed); err != nil {
+			return "", "", err
+		}
+	}
+	signer, err := ssh.ParsePrivateKey([]byte(priv))
+	if err != nil {
+		return "", "", err
+	}
+	pub = strings.TrimSpace(string(ssh.MarshalAuthorizedKey(signer.PublicKey())))
+	return priv, pub, nil
+}
+
+func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) {
+	p, err := a.store.Profile(ctx, deref(m.Profile))
+	if err != nil {
+		return nil, fmt.Errorf("profile %q: %w", deref(m.Profile), err)
+	}
+	ms, err := a.store.List(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var nodes []NodeCtx
+	for _, x := range ms {
+		if x.Profile != nil && x.NodeName != nil {
+			nodes = append(nodes, nodeCtx(x))
+		}
+	}
+	sort.Slice(nodes, func(i, j int) bool { return nodes[i].Name < nodes[j].Name })
+	me := nodeCtx(m)
+	if me.Name != "" {
+		if me.HostKey, me.HostKeyPub, err = a.hostKeyFor(ctx, m); err != nil {
+			return nil, err
+		}
+	}
+	vars := map[string]any{}
+	for k, v := range a.cfg.ClusterVars {
+		vars[k] = v
+	}
+	for k, v := range p.Vars {
+		vars[k] = v
+	}
+	return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Vars: vars, Server: a.cfg.ServerURL,
+		Now: time.Now().Format(time.RFC3339)}, nil
+}
+
+// renderOverlay produces a tar of every overlay dir of the node's profile for the stage.
+func (a *App) renderOverlay(ctx context.Context, m *Machine, stage string) ([]byte, error) {
+	if stage != "system" && stage != "runtime" {
+		return nil, fmt.Errorf("stage must be system or runtime")
+	}
+	oc, err := a.buildCtx(ctx, m)
+	if err != nil {
+		return nil, err
+	}
+	var buf bytes.Buffer
+	tw := tar.NewWriter(&buf)
+	seen := map[string]bool{}
+	for _, name := range oc.Profile.Overlays {
+		root := filepath.Join(a.cfg.OverlaysDir, name, stage)
+		if st, err := os.Stat(root); err != nil || !st.IsDir() {
+			continue
+		}
+		err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error {
+			if err != nil {
+				return err
+			}
+			rel, _ := filepath.Rel(root, p)
+			if rel == "." {
+				return nil
+			}
+			info, err := d.Info()
+			if err != nil {
+				return err
+			}
+			if info.Mode()&fs.ModeSymlink != 0 { // follow symlinks (e.g. runtime -> system templates)
+				if info, err = os.Stat(p); err != nil {
+					return err
+				}
+			}
+			hdr := &tar.Header{Name: rel, Mode: int64(secretMode(rel, info.Mode().Perm())), ModTime: info.ModTime(), Uname: "root", Gname: "root"}
+			if d.IsDir() {
+				if seen[rel] {
+					return nil
+				}
+				seen[rel] = true
+				hdr.Typeflag, hdr.Name = tar.TypeDir, rel+"/"
+				return tw.WriteHeader(hdr)
+			}
+			data, err := os.ReadFile(p)
+			if err != nil {
+				return err
+			}
+			if strings.HasSuffix(rel, ".tmpl") {
+				t, err := template.New(rel).Funcs(tmplFuncs).Option("missingkey=zero").Parse(string(data))
+				if err != nil {
+					return fmt.Errorf("%s: %w", p, err)
+				}
+				var out bytes.Buffer
+				if err := t.Execute(&out, oc); err != nil {
+					return fmt.Errorf("%s: %w", p, err)
+				}
+				data, hdr.Name = out.Bytes(), strings.TrimSuffix(rel, ".tmpl")
+			}
+			if seen[hdr.Name] { // later overlays override earlier ones; tar extraction keeps the last entry
+				log.Printf("overlay %s: %s overrides an earlier file", name, hdr.Name)
+			}
+			seen[hdr.Name] = true
+			hdr.Typeflag, hdr.Size = tar.TypeReg, int64(len(data))
+			if err := tw.WriteHeader(hdr); err != nil {
+				return err
+			}
+			_, err = tw.Write(data)
+			return err
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+	if err := tw.Close(); err != nil {
+		return nil, err
+	}
+	return buf.Bytes(), nil
+}
+
+// secretMode tightens permissions for files git cannot carry modes for.
+func secretMode(rel string, mode fs.FileMode) fs.FileMode {
+	base := strings.TrimSuffix(path.Base(rel), ".tmpl")
+	switch {
+	case base == "munge.key":
+		return 0o400
+	case strings.HasPrefix(base, "ssh_host_") && !strings.HasSuffix(base, ".pub"),
+		base == "authorized_keys", strings.HasSuffix(base, ".key"), strings.HasSuffix(base, ".secret"):
+		return 0o600
+	}
+	return mode
+}
+
+// renderFile renders one template path of a profile with a cluster-wide context
+// (used for files the controller needs too, e.g. slurm.conf).
+func (a *App) renderFile(ctx context.Context, profile, stage, rel string) ([]byte, error) {
+	p, err := a.store.Profile(ctx, profile)
+	if err != nil {
+		return nil, err
+	}
+	ms, err := a.store.List(ctx)
+	if err != nil {
+		return nil, err
+	}
+	var m *Machine
+	for _, x := range ms {
+		if deref(x.Profile) == profile {
+			m = x
+			break
+		}
+	}
+	if m == nil { // no node yet: synthesize an empty one so cluster-level templates still render
+		m = &Machine{ID: "none", Profile: &p.Name}
+	}
+	oc, err := a.buildCtx(ctx, m)
+	if err != nil {
+		return nil, err
+	}
+	for _, name := range p.Overlays {
+		for _, cand := range []string{rel + ".tmpl", rel} {
+			fp := filepath.Join(a.cfg.OverlaysDir, name, stage, cand)
+			data, err := os.ReadFile(fp)
+			if err != nil {
+				continue
+			}
+			if !strings.HasSuffix(cand, ".tmpl") {
+				return data, nil
+			}
+			t, err := template.New(rel).Funcs(tmplFuncs).Option("missingkey=zero").Parse(string(data))
+			if err != nil {
+				return nil, err
+			}
+			var out bytes.Buffer
+			if err := t.Execute(&out, oc); err != nil {
+				return nil, err
+			}
+			return out.Bytes(), nil
+		}
+	}
+	return nil, errNotFound
+}
+
+// ---------------------------------------------------------------- PXE + dnsmasq
+
+const statelessTmpl = `# generated by rollcall %s
+# node %s  machine %s  profile %s
+DEFAULT menu.c32
+PROMPT 0
+TIMEOUT 30
+MENU TITLE %s - stateless (%s)
+
+LABEL node
+  MENU LABEL Boot %s (%s) from RAM
+  MENU DEFAULT
+  KERNEL images/%s/vmlinuz
+  INITRD images/%s/initrd.img
+  APPEND %s
+
+LABEL recommission
+  MENU LABEL Re-run commissioning
+  KERNEL rollcall/vmlinuz
+  INITRD rollcall/initrd.img
+  APPEND %s
+
+LABEL local
+  MENU LABEL Boot from local disk
+  LOCALBOOT 0
+`
+
+func (a *App) statelessAppend(m *Machine, p *Profile, token string) string {
+	parts := []string{"boot=live", "components",
+		"fetch=" + a.cfg.ServerURL + "/images/" + p.Image + "/filesystem.squashfs",
+		"hostname=" + deref(m.NodeName),
+		"live-config.nocomponents=user-setup,sudo",
+		"rollcall.server=" + a.cfg.ServerURL, "rollcall.node=" + token}
+	if p.Append != "" {
+		parts = append(parts, p.Append)
+	}
+	return strings.Join(parts, " ")
+}
+
+func (a *App) writePXEStateless(m *Machine, p *Profile, token string) error {
+	body := fmt.Sprintf(statelessTmpl, time.Now().Format(time.RFC3339), deref(m.NodeName), m.ID, p.Name,
+		deref(m.NodeName), p.Name, deref(m.NodeName), p.Name, p.Image, p.Image, a.statelessAppend(m, p, token),
+		a.cfg.CommissionAppend)
+	return a.writePXEBody(m, body)
+}
+
+// writePXEAny picks the right per-MAC content for the machine's current state.
+func (a *App) writePXEAny(ctx context.Context, m *Machine) error {
+	if m.Profile != nil && m.nodeToken != nil {
+		p, err := a.store.Profile(ctx, *m.Profile)
+		if err != nil {
+			return err
+		}
+		token, err := a.sealer.Open(m.nodeToken, m.ID)
+		if err != nil {
+			return err
+		}
+		return a.writePXEStateless(m, p, token)
+	}
+	return a.writePXE(m)
+}
+
+// exportDnsmasq writes dhcp-hostsfile and addn-hosts files; a systemd path unit HUPs dnsmasq.
+func (a *App) exportDnsmasq(ctx context.Context) error {
+	if a.cfg.DnsmasqDir == "" {
+		return nil
+	}
+	ms, err := a.store.List(ctx)
+	if err != nil {
+		return err
+	}
+	domain := str(a.cfg.ClusterVars["domain"])
+	var dhcp, hosts bytes.Buffer
+	fmt.Fprintf(&dhcp, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339))
+	fmt.Fprintf(&hosts, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339))
+	for _, m := range ms {
+		if m.NodeName == nil || len(m.MACs) == 0 {
+			continue
+		}
+		name := *m.NodeName
+		if m.MgmtIP != nil {
+			fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], *m.MgmtIP, name)
+			if domain != "" {
+				fmt.Fprintf(&hosts, "%s %s.%s %s\n", *m.MgmtIP, name, domain, name)
+			} else {
+				fmt.Fprintf(&hosts, "%s %s\n", *m.MgmtIP, name)
+			}
+		}
+		if m.IBIP != nil {
+			fmt.Fprintf(&hosts, "%s %s-ib\n", *m.IBIP, name)
+		}
+	}
+	if err := os.MkdirAll(a.cfg.DnsmasqDir, 0o755); err != nil {
+		return err
+	}
+	for fn, b := range map[string]*bytes.Buffer{"hosts.dhcp": &dhcp, "hosts": &hosts} {
+		p := filepath.Join(a.cfg.DnsmasqDir, fn)
+		if err := os.WriteFile(p+".tmp", b.Bytes(), 0o644); err != nil {
+			return err
+		}
+		if err := os.Rename(p+".tmp", p); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// ---------------------------------------------------------------- handlers
+
+func (a *App) handleProfiles(w http.ResponseWriter, r *http.Request) {
+	switch r.Method {
+	case http.MethodGet:
+		ps, err := a.store.Profiles(r.Context())
+		if err != nil {
+			fail(w, 500, err)
+			return
+		}
+		writeJSON(w, 200, ps)
+	case http.MethodPost:
+		var p Profile
+		if err := json.NewDecoder(r.Body).Decode(&p); err != nil || p.Name == "" || p.Image == "" {
+			fail(w, 400, fmt.Errorf("name and image are required"))
+			return
+		}
+		if strings.ContainsAny(p.Name, "/ \t") || strings.ContainsAny(p.Image, "/ \t") {
+			fail(w, 400, fmt.Errorf("name/image must be plain identifiers"))
+			return
+		}
+		if err := a.store.PutProfile(r.Context(), p); err != nil {
+			fail(w, 500, err)
+			return
+		}
+		// profile changes affect every assigned node's PXE entry and overlays
+		ms, _ := a.store.List(r.Context())
+		for _, m := range ms {
+			if deref(m.Profile) == p.Name {
+				if err := a.writePXEAny(r.Context(), m); err != nil {
+					log.Printf("pxe rewrite %s: %v", m.ID, err)
+				}
+			}
+		}
+		writeJSON(w, 200, p)
+	}
+}
+
+func (a *App) handleProfileDelete(w http.ResponseWriter, r *http.Request) {
+	if err := a.store.DeleteProfile(r.Context(), r.PathValue("name")); err != nil {
+		fail(w, 409, err)
+		return
+	}
+	writeJSON(w, 200, obj{"deleted": r.PathValue("name")})
+}
+
+func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) {
+	m, err := a.store.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		fail(w, 500, err)
+		return
+	}
+	var req struct {
+		Profile string `json:"profile"`
+		Name    string `json:"name"`
+		Boot    bool   `json:"boot"`
+	}
+	if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Profile == "" {
+		fail(w, 400, fmt.Errorf("profile is required"))
+		return
+	}
+	p, err := a.store.Profile(r.Context(), req.Profile)
+	if err != nil {
+		fail(w, 404, fmt.Errorf("profile %s: %w", req.Profile, err))
+		return
+	}
+	if _, err := os.Stat(filepath.Join(a.cfg.ImagesDir, p.Image, "filesystem.squashfs")); err != nil {
+		fail(w, 409, fmt.Errorf("image %s is not built (%s)", p.Image, err))
+		return
+	}
+	// keep the existing name/IPs when re-assigning an already named node
+	name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP)
+	if name == "" || (req.Name != "" && req.Name != name) {
+		if name, mgmt, ib, err = a.allocate(r.Context(), p, req.Name); err != nil {
+			fail(w, 409, err)
+			return
+		}
+	}
+	token := randomToken(16)
+	sum := sha256.Sum256([]byte(token))
+	sealed, err := a.sealer.Seal(token, m.ID)
+	if err != nil {
+		fail(w, 500, err)
+		return
+	}
+	if err := a.store.Assign(r.Context(), m.ID, nodeAssignment{Profile: p.Name, Name: name, MgmtIP: mgmt, IBIP: ib,
+		TokenHash: hex.EncodeToString(sum[:]), Token: sealed}); err != nil {
+		fail(w, 500, err)
+		return
+	}
+	m, _ = a.store.Get(r.Context(), m.ID)
+	if err := a.writePXEStateless(m, p, token); err != nil {
+		fail(w, 500, err)
+		return
+	}
+	if err := a.exportDnsmasq(r.Context()); err != nil {
+		log.Printf("dnsmasq export: %v", err)
+	}
+	_ = a.store.AddEvent(r.Context(), m.ID, "assigned", fmt.Sprintf("profile=%s name=%s mgmt=%s ib=%s", p.Name, name, mgmt, ib))
+	out := obj{"id": m.ID, "state": "stateless", "profile": p.Name, "name": name, "mgmt_ip": mgmt, "ib_ip": ib}
+	if req.Boot {
+		out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m)
+	}
+	log.Printf("assigned %s -> %s as %s (%s)", m.ID, p.Name, name, mgmt)
+	writeJSON(w, 200, out)
+}
+
+// pxeBoot sets bootdev pxe and power-cycles (or powers on) through the BMC.
+func (a *App) pxeBoot(ctx context.Context, m *Machine) (string, bool) {
+	c, err := a.credsFor(m)
+	if err != nil {
+		return err.Error(), false
+	}
+	o1, e1 := a.ipmi(ctx, c, "chassis", "bootdev", "pxe")
+	st, _ := a.ipmi(ctx, c, "chassis", "power", "status")
+	var o2 string
+	var e2 error
+	if strings.Contains(st, "off") {
+		o2, e2 = a.ipmi(ctx, c, "chassis", "power", "on")
+	} else {
+		o2, e2 = a.ipmi(ctx, c, "chassis", "power", "cycle")
+	}
+	_ = a.store.AddEvent(ctx, m.ID, "pxe boot", strings.TrimSpace(o1+"\n"+o2))
+	return strings.TrimSpace(o1 + "\n" + o2), e1 == nil && e2 == nil
+}
+
+func (a *App) handleUnassign(w http.ResponseWriter, r *http.Request) {
+	m, err := a.store.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		fail(w, 500, err)
+		return
+	}
+	if err := a.store.Unassign(r.Context(), m.ID); err != nil {
+		fail(w, 500, err)
+		return
+	}
+	m, _ = a.store.Get(r.Context(), m.ID)
+	_ = a.writePXE(m)
+	_ = a.exportDnsmasq(r.Context())
+	_ = a.store.AddEvent(r.Context(), m.ID, "unassigned", "")
+	writeJSON(w, 200, obj{"id": m.ID, "state": "ready"})
+}
+
+func (a *App) serveOverlay(w http.ResponseWriter, r *http.Request, m *Machine) {
+	stage := r.URL.Query().Get("stage")
+	if stage == "" {
+		stage = "system"
+	}
+	data, err := a.renderOverlay(r.Context(), m, stage)
+	if err != nil {
+		fail(w, 500, err)
+		return
+	}
+	sum := sha256.Sum256(data)
+	w.Header().Set("Content-Type", "application/x-tar")
+	w.Header().Set("ETag", `"`+hex.EncodeToString(sum[:8])+`"`)
+	w.Header().Set("X-Rollcall-Node-Name", deref(m.NodeName))
+	w.Header().Set("Content-Length", strconv.Itoa(len(data)))
+	_, _ = w.Write(data)
+}
+
+// handleNodeOverlay is what a booted node calls; the token comes from its kernel cmdline.
+func (a *App) handleNodeOverlay(w http.ResponseWriter, r *http.Request) {
+	tok := r.Header.Get("X-Rollcall-Node")
+	if tok == "" {
+		fail(w, 401, errors.New("X-Rollcall-Node required"))
+		return
+	}
+	sum := sha256.Sum256([]byte(tok))
+	m, err := a.store.ByNodeToken(r.Context(), hex.EncodeToString(sum[:]))
+	if err != nil {
+		fail(w, 401, errors.New("unknown node token"))
+		return
+	}
+	if m.State != "booted" {
+		_ = a.store.AddEvent(r.Context(), m.ID, "booted", r.RemoteAddr)
+		log.Printf("node %s (%s) booted from %s", deref(m.NodeName), m.ID, r.RemoteAddr)
+	}
+	_ = a.store.MarkBooted(r.Context(), m.ID)
+	a.serveOverlay(w, r, m)
+}
+
+func (a *App) handleAdminOverlay(w http.ResponseWriter, r *http.Request) {
+	m, err := a.store.Get(r.Context(), r.PathValue("id"))
+	if err != nil {
+		fail(w, 500, err)
+		return
+	}
+	if m.Profile == nil {
+		fail(w, 409, errors.New("machine has no profile"))
+		return
+	}
+	a.serveOverlay(w, r, m)
+}
+
+func (a *App) handleRender(w http.ResponseWriter, r *http.Request) {
+	stage := r.URL.Query().Get("stage")
+	if stage == "" {
+		stage = "system"
+	}
+	data, err := a.renderFile(r.Context(), r.PathValue("profile"), stage, path.Clean(r.PathValue("path")))
+	if err != nil {
+		fail(w, 500, err)
+		return
+	}
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	_, _ = w.Write(data)
+}
+
+func (a *App) handleImages(w http.ResponseWriter, r *http.Request) {
+	ents, err := os.ReadDir(a.cfg.ImagesDir)
+	if err != nil {
+		writeJSON(w, 200, []any{})
+		return
+	}
+	out := []obj{}
+	for _, e := range ents {
+		if !e.IsDir() {
+			continue
+		}
+		info := obj{"name": e.Name()}
+		for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} {
+			if st, err := os.Stat(filepath.Join(a.cfg.ImagesDir, e.Name(), f)); err == nil {
+				info[f] = st.Size()
+				info["built"] = st.ModTime()
+			}
+		}
+		info["complete"] = info["filesystem.squashfs"] != nil && info["vmlinuz"] != nil && info["initrd.img"] != nil
+		out = append(out, info)
+	}
+	writeJSON(w, 200, out)
+}

cmd/rollcall/store.go

@@ -38,6 +38,24 @@ CREATE TABLE IF NOT EXISTS events (
   detail     text
 );
 CREATE INDEX IF NOT EXISTS events_machine ON events(machine_id, ts);
+CREATE TABLE IF NOT EXISTS profiles (
+  name        text PRIMARY KEY,
+  image       text NOT NULL,
+  append      text,
+  overlays    jsonb NOT NULL DEFAULT '[]',
+  name_format text,
+  vars        jsonb NOT NULL DEFAULT '{}',
+  created     timestamptz NOT NULL DEFAULT now()
+);
+ALTER TABLE machines ADD COLUMN IF NOT EXISTS profile text REFERENCES profiles(name);
+ALTER TABLE machines ADD COLUMN IF NOT EXISTS node_name text UNIQUE;
+ALTER TABLE machines ADD COLUMN IF NOT EXISTS mgmt_ip text;
+ALTER TABLE machines ADD COLUMN IF NOT EXISTS ib_ip text;
+ALTER TABLE machines ADD COLUMN IF NOT EXISTS node_token_hash text;
+ALTER TABLE machines ADD COLUMN IF NOT EXISTS node_token bytea;
+ALTER TABLE machines ADD COLUMN IF NOT EXISTS host_key bytea;
+ALTER TABLE machines ADD COLUMN IF NOT EXISTS booted_at timestamptz;
+CREATE INDEX IF NOT EXISTS machines_node_token ON machines(node_token_hash);
 `
 
 var errNotFound = errors.New("not found")
@@ -58,7 +76,14 @@ type Machine struct {
 	MACs           []string        `json:"macs"`
 	BMC            json.RawMessage `json:"bmc,omitempty"`
 	Inventory      json.RawMessage `json:"inventory,omitempty"`
+	Profile        *string         `json:"profile"`
+	NodeName       *string         `json:"node_name"`
+	MgmtIP         *string         `json:"mgmt_ip"`
+	IBIP           *string         `json:"ib_ip"`
+	BootedAt       *time.Time      `json:"booted_at"`
 	bmcSecret      []byte
+	nodeToken      []byte
+	hostKey        []byte
 }
 
 type Event struct {
@@ -79,13 +104,14 @@ func openStore(ctx context.Context, url string) (*Store, error) {
 }
 
 const machineCols = `id, uuid, serial, hostname, manufacturer, product, state, first_seen, last_seen,
-	commissioned_at, macs, bmc, bmc_secret, inventory`
+	commissioned_at, macs, bmc, bmc_secret, inventory, profile, node_name, mgmt_ip, ib_ip, node_token, host_key, booted_at`
 
 func scanMachine(row pgx.Row, full bool) (*Machine, error) {
 	m := &Machine{}
 	var macs []byte
 	err := row.Scan(&m.ID, &m.UUID, &m.Serial, &m.Hostname, &m.Manufacturer, &m.Product, &m.State,
-		&m.FirstSeen, &m.LastSeen, &m.CommissionedAt, &macs, &m.BMC, &m.bmcSecret, &m.Inventory)
+		&m.FirstSeen, &m.LastSeen, &m.CommissionedAt, &macs, &m.BMC, &m.bmcSecret, &m.Inventory,
+		&m.Profile, &m.NodeName, &m.MgmtIP, &m.IBIP, &m.nodeToken, &m.hostKey, &m.BootedAt)
 	if errors.Is(err, pgx.ErrNoRows) {
 		return nil, errNotFound
 	}
@@ -163,8 +189,10 @@ func (s *Store) UpsertCommissioned(ctx context.Context, m *Machine, secret []byt
 			return false, err
 		}
 	}
+	// a node that already has a profile goes back to 'stateless', everything else to 'ready'
 	_, err = tx.Exec(ctx, `UPDATE machines SET uuid=$2, serial=$3, hostname=$4, manufacturer=$5, product=$6,
-		state='ready', last_seen=now(), commissioned_at=now(), macs=$7, bmc=$8, bmc_secret=$9, inventory=$10 WHERE id=$1`,
+		state=CASE WHEN profile IS NULL THEN 'ready' ELSE 'stateless' END, booted_at=NULL,
+		last_seen=now(), commissioned_at=now(), macs=$7, bmc=$8, bmc_secret=$9, inventory=$10 WHERE id=$1`,
 		m.ID, m.UUID, m.Serial, m.Hostname, m.Manufacturer, m.Product, macs, m.BMC, secret, m.Inventory)
 	if err != nil {
 		return false, err

cmd/rollcall/web/app.js

@@ -35,15 +35,38 @@ $('#back').onclick = () => { current = null; $('#detail').hidden = true; $('#lis
 
 function setStatus(msg, bad) { const el = $('#status'); el.textContent = msg; el.className = bad ? 'bad' : 'muted'; }
 
+let profiles = [];
 async function loadList() {
   if (!token) return askToken();
   try {
     machines = await api('GET', '/api/v1/machines');
     setStatus(`updated ${new Date().toLocaleTimeString()}`);
     renderList();
+    loadProfiles();
   } catch (e) { if (e.message !== 'unauthorized') setStatus(e.message, true); }
 }
 
+async function loadProfiles() {
+  try {
+    profiles = await api('GET', '/api/v1/profiles');
+    const images = await api('GET', '/api/v1/images');
+    $('#profiles').innerHTML = table(['Profile', 'Image', 'Overlays', 'Names', 'Nodes', ''], profiles.map(p => [esc(p.name), esc(p.image), esc((p.overlays || []).join(', ')), esc(p.name_format), machines.filter(m => m.profile === p.name).length, `<button class="btn danger" data-rmprofile="${esc(p.name)}">rm</button>`]));
+    $('#images').innerHTML = table(['Image', 'Complete', 'Squashfs', 'Built'], images.map(i => [esc(i.name), i.complete ? '<span class="ok">yes</span>' : '<span class="bad">no</span>', human(i['filesystem.squashfs']), esc(i.built ? new Date(i.built).toLocaleString() : '')]));
+    document.querySelectorAll('[data-rmprofile]').forEach(b => b.onclick = async () => { if (confirm(`Delete profile ${b.dataset.rmprofile}?`)) { try { await api('DELETE', '/api/v1/profiles/' + b.dataset.rmprofile); } catch (e) { alert(e.message); } loadProfiles(); } });
+    const sel = $('#assign-profile'); const cur = sel.value;
+    sel.innerHTML = '<option value="">profile…</option>' + profiles.map(p => `<option value="${esc(p.name)}">${esc(p.name)} (${esc(p.image)})</option>`).join('');
+    sel.value = cur;
+  } catch (e) { /* non-fatal */ }
+}
+$('#profile-form').onsubmit = async ev => {
+  ev.preventDefault();
+  const f = new FormData(ev.target);
+  let vars = {};
+  try { vars = JSON.parse(f.get('vars') || '{}'); } catch (e) { $('#profile-out').textContent = 'vars: invalid JSON'; return; }
+  const body = { name: f.get('name'), image: f.get('image'), append: f.get('append'), name_format: f.get('name_format'), vars, overlays: (f.get('overlays') || '').split(',').map(s => s.trim()).filter(Boolean) };
+  try { await api('POST', '/api/v1/profiles', body); $('#profile-out').textContent = 'saved'; loadProfiles(); } catch (e) { $('#profile-out').textContent = e.message; }
+};
+
 function renderList() {
   const q = $('#filter').value.toLowerCase();
   const rows = machines.filter(m => !q || JSON.stringify([m.id, m.state, m.manufacturer, m.product, m.serial, m.macs, m.hostname]).toLowerCase().includes(q));
@@ -54,6 +77,7 @@ function renderList() {
     const bmc = b ? `<span class="${b.configured ? 'ok' : 'bad'}">${esc(b.address || '-')}</span> <span class="muted">${esc(b.source)}</span>` : '<span class="muted">-</span>';
     return `<tr data-id="${esc(m.id)}">
       <td class="mono">${esc(m.id)}</td><td><span class="pill ${esc(m.state)}">${esc(m.state)}</span></td>
+      <td>${esc(m.node_name || '')}${m.mgmt_ip ? `<br><span class="muted mono">${esc(m.mgmt_ip)}</span>` : ''}</td><td>${esc(m.profile || '')}</td>
       <td>${esc((m.manufacturer || '') + ' ' + (m.product || ''))}</td><td class="mono">${esc(m.serial || '')}</td>
       <td class="num">${esc(m.cpus)}</td><td class="num">${human(m.memory_bytes)}</td><td class="num">${m.disks | 0}</td>
       <td class="num">${human(m.storage_bytes)}</td><td class="num">${m.nics}</td><td class="num">${m.ib_hcas}</td><td class="num">${m.gpus}</td>
@@ -77,10 +101,17 @@ async function loadDetail(id) {
 
 function renderDetail(d) {
   const s = d.summary, inv = d.inventory || {}, sys = inv.system || {}, cpu = inv.cpu || {}, mem = inv.memory || {}, st = inv.storage || {}, b = d.bmc || {};
-  $('#d-title').textContent = `${s.id}  ${sys['system-manufacturer'] || ''} ${sys['system-product-name'] || ''}`;
+  $('#d-title').textContent = `${s.node_name ? s.node_name + ' · ' : ''}${s.id}  ${sys['system-manufacturer'] || ''} ${sys['system-product-name'] || ''}`;
   $('#d-state').textContent = s.state; $('#d-state').className = 'pill ' + s.state;
   $('#action-out').textContent = '';
+  if (s.profile) $('#assign-profile').value = s.profile;
+  $('#assign-name').value = s.node_name || '';
   const cards = [];
+  if (s.profile) cards.push(card('Stateless node', kv([
+    ['Node name', esc(s.node_name)], ['Profile', esc(s.profile)], ['Management IP', `<span class="mono">${esc(s.mgmt_ip)}</span>`],
+    ['InfiniBand IP', s.ib_ip ? `<span class="mono">${esc(s.ib_ip)}</span>` : ''], ['Booted', esc(s.booted_at ? new Date(s.booted_at).toLocaleString() : 'not yet')],
+    ['Last heartbeat', esc(s.last_seen ? ago(s.last_seen) : '')],
+  ])));
   cards.push(card('System', kv([
     ['Manufacturer', esc(sys['system-manufacturer'])], ['Product', esc(sys['system-product-name'])], ['Serial', esc(sys['system-serial-number'])],
     ['UUID', `<span class="mono">${esc(sys['system-uuid'])}</span>`], ['SKU', esc(sys['system-sku-number'])],
@@ -149,6 +180,14 @@ document.querySelectorAll('[data-power]').forEach(b => b.onclick = () => action(
 document.querySelectorAll('[data-bootdev]').forEach(b => b.onclick = () => action(async () => {
   const r = await api('POST', `/api/v1/machines/${current}/bootdev`, { device: b.dataset.bootdev }); return r.output || (r.ok ? 'ok' : 'failed');
 }, 'bootdev ' + b.dataset.bootdev));
+const doAssign = boot => action(async () => {
+  const profile = $('#assign-profile').value; if (!profile) throw new Error('pick a profile');
+  const r = await api('POST', `/api/v1/machines/${current}/assign`, { profile, name: $('#assign-name').value.trim(), boot });
+  await loadDetail(current); return `${r.name} (${r.mgmt_ip || 'no pool'})${r.ipmi ? ' · ' + r.ipmi : ''}`;
+}, boot ? 'assign + boot' : 'assign');
+$('#assign').onclick = () => doAssign(false);
+$('#assign-boot').onclick = () => { if (confirm('Assign and power-cycle this machine into the stateless image?')) doAssign(true); };
+$('#unassign').onclick = () => { if (confirm('Remove the profile? The node name and IP are released.')) action(async () => { const r = await api('POST', `/api/v1/machines/${current}/unassign`); await loadDetail(current); return 'state ' + r.state; }, 'unassign'); };
 $('#recommission').onclick = () => action(async () => { const r = await api('POST', `/api/v1/machines/${current}/recommission`); await loadDetail(current); return `state ${r.state}: next PXE boot runs commissioning`; }, 'recommission');
 $('#recommission-now').onclick = () => { if (confirm('Remove PXE override, set bootdev=pxe and power-cycle this machine via its BMC?')) action(async () => { const r = await api('POST', `/api/v1/machines/${current}/recommission?now=1`); await loadDetail(current); return r.ipmi || r.state; }, 'recommission now'); };
 $('#delete').onclick = () => { if (confirm(`Delete ${current}? It will be commissioned again as a new machine on next PXE boot.`)) action(async () => { await api('DELETE', `/api/v1/machines/${current}`); $('#back').click(); return 'deleted'; }, 'delete'); };

cmd/rollcall/web/index.html

@@ -33,13 +33,32 @@
     </div>
     <table id="machines">
       <thead><tr>
-        <th>ID</th><th>State</th><th>Model</th><th>Serial</th><th class="num">CPU</th><th class="num">RAM</th>
+        <th>ID</th><th>State</th><th>Node</th><th>Profile</th><th>Model</th><th>Serial</th><th class="num">CPU</th><th class="num">RAM</th>
         <th class="num">Disks</th><th class="num">Storage</th><th class="num">NIC</th><th class="num">IB</th><th class="num">GPU</th>
         <th>BMC</th><th>Last seen</th>
       </tr></thead>
       <tbody></tbody>
     </table>
     <p id="empty" class="muted" hidden>No machines yet. PXE-boot an unknown server: it commissions itself and appears here.</p>
+
+    <details id="profiles-box" style="margin-top:18px">
+      <summary>Profiles &amp; images (stateless provisioning)</summary>
+      <div class="grid" style="margin-top:10px">
+        <div class="card"><h3>Profiles</h3><div id="profiles"></div></div>
+        <div class="card"><h3>Images</h3><div id="images"></div></div>
+        <div class="card wide"><h3>Add / update profile</h3>
+          <form id="profile-form" class="kv" style="grid-template-columns:max-content 1fr;align-items:center">
+            <label>Name</label><input name="name" required placeholder="compute">
+            <label>Image</label><input name="image" required placeholder="compute-debian13">
+            <label>Overlays</label><input name="overlays" placeholder="compute">
+            <label>Name format</label><input name="name_format" value="node%03d">
+            <label>Kernel append</label><input name="append" placeholder="extra kernel parameters">
+            <label>Vars (JSON)</label><input name="vars" value="{}">
+            <span></span><span><button class="btn primary" type="submit">Save profile</button> <span id="profile-out" class="muted"></span></span>
+          </form>
+        </div>
+      </div>
+    </details>
   </section>
 
   <section id="detail" hidden>
@@ -61,6 +80,13 @@
       <button class="btn" data-bootdev="pxe">PXE</button>
       <button class="btn" data-bootdev="disk">Disk</button>
       <span class="sep"></span>
+      <span class="muted">Stateless:</span>
+      <select id="assign-profile"><option value="">profile…</option></select>
+      <input id="assign-name" placeholder="node name (auto)" style="width:150px;padding:5px 8px;border:1px solid var(--line);border-radius:6px;background:var(--panel);color:var(--fg)">
+      <button id="assign" class="btn">Assign</button>
+      <button id="assign-boot" class="btn" title="assign, bootdev pxe, power-cycle">Assign + boot</button>
+      <button id="unassign" class="btn warn">Unassign</button>
+      <span class="sep"></span>
       <button id="recommission" class="btn warn">Re-commission</button>
       <button id="recommission-now" class="btn warn" title="remove per-MAC PXE config, set bootdev pxe and power-cycle via BMC">Re-commission now</button>
       <button id="delete" class="btn danger">Delete</button>

cmd/rollcall/web/style.css

@@ -33,7 +33,9 @@ th { font-weight: 600; color: var(--muted); font-size: 12px; text-transform: upp
 th.num, td.num { text-align: right; font-variant-numeric: tabular-nums; }
 tbody tr { cursor: pointer; } tbody tr:hover { background: var(--hover); }
 .pill { display: inline-block; padding: 2px 8px; border-radius: 999px; background: var(--pill); font-size: 12px; }
-.pill.ready { color: var(--ok); } .pill.recommission, .pill.new { color: var(--warn); }
+.pill.ready, .pill.booted { color: var(--ok); } .pill.recommission, .pill.new, .pill.stateless { color: var(--warn); }
+.card form input { padding: 5px 8px; border: 1px solid var(--line); border-radius: 6px; background: var(--panel); color: var(--fg); font: inherit; min-width: 0; width: 100%; }
+#assign-profile { padding: 5px 8px; border: 1px solid var(--line); border-radius: 6px; background: var(--panel); color: var(--fg); font: inherit; }
 .ok { color: var(--ok); } .bad { color: var(--danger); }
 .detail-head { display: flex; gap: 12px; align-items: center; margin-bottom: 10px; }
 .actions { display: flex; gap: 6px; align-items: center; flex-wrap: wrap; padding: 10px; background: var(--panel); border: 1px solid var(--line); border-radius: 8px; margin-bottom: 14px; }

deploy/config.example.json

@@ -8,6 +8,19 @@
   "agent_token": "<AGENT-TOKEN: rollcall keygen>",
   "commission_append": "boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 rollcall.token=<AGENT-TOKEN> console=tty0 console=ttyS0,115200n8",
   "ipmitool": "ipmitool",
+  "server_url": "http://10.213.77.1",
+  "images_dir": "/srv/rollcall/images",
+  "overlays_dir": "/etc/rollcall/overlays",
+  "dnsmasq_dir": "/var/lib/rollcall/dnsmasq",
+  "node_pools": {"mgmt": "10.213.77.50-10.213.77.99"},
+  "cluster_vars": {
+    "domain": "pxe.local",
+    "cluster_name": "rollcall-lab",
+    "slurm_controller": "pxe-server",
+    "slurm_controller_ip": "10.213.77.1",
+    "partition": "compute",
+    "ssh_keys": ["ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDSAJ4owXlJreHP3sa4HstasxA5pk1ZAPi6vnzOpA5R1 ilker@t440s"]
+  },
   "bmc_hints": [
     {"mac": "bc:24:11:8e:1e:6c", "address": "10.213.77.11", "port": 623, "user": "admin", "password": "password", "comment": "pxe-node01 (VM 211, pve-vbmc)"},
     {"mac": "bc:24:11:36:90:3f", "address": "10.213.77.12", "port": 623, "user": "admin", "password": "password", "comment": "pxe-node02 (VM 212, pve-vbmc)"}

deploy/install.sh

@@ -19,7 +19,20 @@ install -m 0644 /usr/lib/syslinux/modules/bios/poweroff.c32 /srv/tftp/poweroff.c
 sed "s/@AGENT_TOKEN@/$(jq -r .agent_token /etc/rollcall/config.json)/g" pxelinux-default > /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default
 [ -e /srv/tftp/rollcall ] || ln -s /srv/rollcall/image /srv/tftp/rollcall
 [ -d /srv/tftp/efi64 ] && ln -sfn ../rollcall /srv/tftp/efi64/rollcall
+# stateless images: <images_dir>/<name>/..., reachable by pxelinux as images/<name>/
+install -d /srv/rollcall/images
+[ -e /srv/tftp/images ] || ln -s /srv/rollcall/images /srv/tftp/images
+[ -d /srv/tftp/efi64 ] && ln -sfn ../images /srv/tftp/efi64/images
+# overlay templates (server-only secrets such as munge.key live next to them and are kept)
+install -d -m 0750 -o root -g rollcall /etc/rollcall/overlays
+cp -a overlays/. /etc/rollcall/overlays/
+chgrp -R rollcall /etc/rollcall/overlays
+# dnsmasq host export + reload trigger
+install -d -m 0755 /var/lib/rollcall
+install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/dnsmasq
+install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service /etc/systemd/system/
 systemctl daemon-reload
+systemctl enable --now rollcall-dnsmasq.path >/dev/null 2>&1 || true
 systemctl enable --now rollcall.service
 systemctl restart rollcall.service
 sleep 1

deploy/overlays/compute/runtime/etc/hosts.tmpl

@@ -0,0 +1 @@
+../../system/etc/hosts.tmpl

deploy/overlays/compute/runtime/etc/slurm/slurm.conf.tmpl

@@ -0,0 +1 @@
+../../../system/etc/slurm/slurm.conf.tmpl

deploy/overlays/compute/runtime/run/rollcall/post-runtime.sh

@@ -0,0 +1,4 @@
+#!/bin/sh
+# runs when the runtime overlay changed (new nodes, partition changes)
+systemctl try-restart slurmd.service
+exit 0

deploy/overlays/compute/system/etc/hosts.tmpl

@@ -0,0 +1,11 @@
+127.0.0.1 localhost
+::1       localhost ip6-localhost ip6-loopback
+{{- if .Vars.slurm_controller_ip }}
+{{ .Vars.slurm_controller_ip }} {{ .Vars.slurm_controller }}{{ if .Vars.domain }} {{ .Vars.slurm_controller }}.{{ .Vars.domain }}{{ end }}
+{{- end }}
+{{- range .Nodes }}
+{{ .MgmtIP }} {{ .Name }}{{ if $.Vars.domain }} {{ .Name }}.{{ $.Vars.domain }}{{ end }}
+{{- if .IBIP }}
+{{ .IBIP }} {{ .Name }}-ib
+{{- end }}
+{{- end }}

deploy/overlays/compute/system/etc/slurm/gres.conf.tmpl

@@ -0,0 +1,5 @@
+# rendered by rollcall for {{ .Node.Name }}
+{{- if gt .Node.GPUs 0 }}
+AutoDetect=off
+NodeName={{ .Node.Name }} Name=gpu File=/dev/nvidia[0-{{ sub .Node.GPUs 1 }}]
+{{- end }}

deploy/overlays/compute/system/etc/slurm/slurm.conf.tmpl

@@ -0,0 +1,27 @@
+# rendered by rollcall {{ .Now }} - do not edit on the node
+ClusterName={{ default "rollcall" .Vars.cluster_name }}
+SlurmctldHost={{ .Vars.slurm_controller }}({{ .Vars.slurm_controller_ip }})
+SlurmUser=slurm
+AuthType=auth/munge
+StateSaveLocation=/var/spool/slurmctld
+SlurmdSpoolDir=/var/spool/slurmd
+SlurmctldPidFile=/run/slurmctld.pid
+SlurmdPidFile=/run/slurmd.pid
+SlurmctldLogFile=/var/log/slurm/slurmctld.log
+SlurmdLogFile=/var/log/slurm/slurmd.log
+ProctrackType=proctrack/linuxproc
+TaskPlugin=task/none
+ReturnToService=2
+SlurmctldTimeout=120
+SlurmdTimeout=300
+SchedulerType=sched/backfill
+SelectType=select/cons_tres
+SelectTypeParameters=CR_Core_Memory
+GresTypes=gpu
+MpiDefault=none
+
+# nodes: generated from the commissioning inventory
+{{- range .Nodes }}
+NodeName={{ .Name }} NodeAddr={{ .MgmtIP }} Sockets={{ .Sockets }} CoresPerSocket={{ .CoresPerSocket }} ThreadsPerCore={{ .ThreadsPerCore }} RealMemory={{ pct .MemMB 90 }}{{ if gt .GPUs 0 }} Gres=gpu:{{ .GPUs }}{{ end }} State=UNKNOWN
+{{- end }}
+PartitionName={{ default "compute" .Vars.partition }} Nodes=ALL Default=YES MaxTime=INFINITE State=UP

deploy/overlays/compute/system/etc/ssh/ssh_host_ed25519_key.pub.tmpl

@@ -0,0 +1 @@
+{{ .Node.HostKeyPub }} {{ .Node.Name }}

deploy/overlays/compute/system/etc/ssh/ssh_host_ed25519_key.tmpl

@@ -0,0 +1 @@
+{{ .Node.HostKey }}

deploy/overlays/compute/system/root/.ssh/authorized_keys.tmpl

@@ -0,0 +1,2 @@
+{{ range .Vars.ssh_keys }}{{ . }}
+{{ end }}

deploy/overlays/compute/system/run/rollcall/post-system.sh.tmpl

@@ -0,0 +1,9 @@
+#!/bin/sh
+# runs once after the system overlay is applied (node {{ .Node.Name }})
+hostnamectl set-hostname {{ .Node.Name }} 2>/dev/null || hostname {{ .Node.Name }}
+chown munge:munge /etc/munge/munge.key && chmod 0400 /etc/munge/munge.key
+chmod 0600 /etc/ssh/ssh_host_ed25519_key; chmod 0644 /etc/ssh/ssh_host_ed25519_key.pub
+chmod 0700 /root/.ssh; chmod 0600 /root/.ssh/authorized_keys
+install -d -m 0755 /var/spool/slurmd /var/log/slurm
+systemctl daemon-reload
+exit 0

deploy/rollcall-dnsmasq.path

@@ -0,0 +1,9 @@
+[Unit]
+Description=rollcall: reload dnsmasq when node host files change
+
+[Path]
+PathChanged=/var/lib/rollcall/dnsmasq/hosts.dhcp
+PathChanged=/var/lib/rollcall/dnsmasq/hosts
+
+[Install]
+WantedBy=multi-user.target

deploy/rollcall-dnsmasq.service

@@ -0,0 +1,6 @@
+[Unit]
+Description=rollcall: HUP dnsmasq to re-read dhcp-hostsfile / addn-hosts
+
+[Service]
+Type=oneshot
+ExecStart=/bin/systemctl kill -s HUP dnsmasq.service

deploy/rollcall.service

@@ -16,7 +16,7 @@ NoNewPrivileges=yes
 ProtectSystem=strict
 ProtectHome=yes
 PrivateTmp=yes
-ReadWritePaths=/etc/rollcall /srv/tftp/pxelinux.cfg
+ReadWritePaths=/etc/rollcall /srv/tftp/pxelinux.cfg /var/lib/rollcall
 
 [Install]
 WantedBy=multi-user.target

images/build-rootfs.sh

@@ -0,0 +1,55 @@
+#!/bin/bash
+# Build a stateless (RAM-booted) root image with live-build.
+#
+#   build-rootfs.sh <image-name>
+#
+# Reads images/<image-name>/{packages.list,hook.chroot,includes/} and writes
+# <IMAGES_DIR>/<image-name>/{vmlinuz,initrd.img,filesystem.squashfs}, which
+# rollcall serves under /images/<image-name>/ and pxelinux loads via TFTP.
+set -euo pipefail
+NAME=${1:?image name}
+SRC=$(cd "$(dirname "$0")" && pwd)/$NAME
+[ -d "$SRC" ] || { echo "no such image definition: $SRC" >&2; exit 1; }
+IMAGES_DIR=${IMAGES_DIR:-/srv/rollcall/images}
+BUILD=${BUILD:-/srv/rollcall/build-$NAME}
+MIRROR=${MIRROR:-http://deb.debian.org/debian}
+DIST=${DIST:-trixie}
+AUTHKEYS=${AUTHKEYS:-/home/ilker/.ssh/authorized_keys}
+OUT=$IMAGES_DIR/$NAME
+
+mkdir -p "$BUILD" "$OUT"
+cd "$BUILD"
+if [ -d config ]; then lb clean --purge >/dev/null 2>&1 || true; rm -rf config; fi
+
+lb config \
+  --distribution "$DIST" --architectures amd64 \
+  --archive-areas "main contrib non-free non-free-firmware" \
+  --mirror-bootstrap "$MIRROR" --mirror-binary "$MIRROR" \
+  --binary-images iso-hybrid --bootloaders syslinux \
+  --debian-installer none --memtest none \
+  --firmware-chroot true --firmware-binary false \
+  --apt-recommends false \
+  --bootappend-live "boot=live components" \
+  --linux-flavours amd64 \
+  --iso-volume "$NAME" --iso-application "rollcall $NAME" \
+  --cache-packages true --cache-indices false >/dev/null
+
+install -m 0644 "$SRC/packages.list" "config/package-lists/$NAME.list.chroot"
+mkdir -p config/includes.chroot config/hooks/live
+if [ -d "$SRC/includes" ]; then cp -a "$SRC/includes/." config/includes.chroot/; fi
+if [ -r "$AUTHKEYS" ]; then
+  install -d -m 0700 config/includes.chroot/root/.ssh
+  install -m 0600 "$AUTHKEYS" config/includes.chroot/root/.ssh/authorized_keys
+fi
+if [ -f "$SRC/hook.chroot" ]; then install -m 0755 "$SRC/hook.chroot" "config/hooks/live/0100-$NAME.hook.chroot"; fi
+
+echo "=== lb build $NAME started $(date)"
+lb build 2>&1 | tee "$BUILD/build.log" | grep -E "^P: (Begin|Build)|^E:" || true
+
+for f in vmlinuz initrd.img filesystem.squashfs; do
+  src=$(ls binary/live/$f* 2>/dev/null | head -1)
+  [ -n "$src" ] || { echo "missing $f in binary/live (see $BUILD/build.log)" >&2; exit 1; }
+  install -m 0644 "$src" "$OUT/$f.new" && mv -f "$OUT/$f.new" "$OUT/$f"
+done
+ls -la "$OUT"
+echo "=== done $(date)"

images/compute-debian13/hook.chroot

@@ -0,0 +1,11 @@
+#!/bin/sh
+# live-build chroot hook for the compute image
+set -e
+systemctl enable rollcall-overlay.service rollcall-overlay-runtime.timer
+systemctl enable munge.service slurmd.service ssh.service
+systemctl disable apt-daily.timer apt-daily-upgrade.timer 2>/dev/null || true
+sed -i 's/^#\?PermitRootLogin.*/PermitRootLogin prohibit-password/' /etc/ssh/sshd_config
+# host keys come from the overlay; do not let the package generate random ones first
+rm -f /etc/ssh/ssh_host_*
+install -d -m 0755 -o root -g root /var/spool/slurmd /var/log/slurm
+echo rollcall > /etc/hostname

images/compute-debian13/includes/etc/systemd/system/munge.service.d/rollcall.conf

@@ -0,0 +1,3 @@
+[Unit]
+After=rollcall-overlay.service
+Requires=rollcall-overlay.service

images/compute-debian13/includes/etc/systemd/system/rollcall-overlay-runtime.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=rollcall: refresh node runtime overlay
+After=rollcall-overlay.service
+ConditionKernelCommandLine=rollcall.node
+
+[Service]
+Type=oneshot
+ExecStart=/usr/local/sbin/rollcall-overlay runtime

images/compute-debian13/includes/etc/systemd/system/rollcall-overlay-runtime.timer

@@ -0,0 +1,10 @@
+[Unit]
+Description=rollcall: periodic runtime overlay refresh (also the node heartbeat)
+
+[Timer]
+OnBootSec=2min
+OnUnitActiveSec=5min
+RandomizedDelaySec=30s
+
+[Install]
+WantedBy=timers.target

images/compute-debian13/includes/etc/systemd/system/rollcall-overlay.service

@@ -0,0 +1,15 @@
+[Unit]
+Description=rollcall: apply node system overlay
+After=network-online.target
+Wants=network-online.target
+Before=munge.service slurmd.service ssh.service
+ConditionKernelCommandLine=rollcall.node
+
+[Service]
+Type=oneshot
+RemainAfterExit=yes
+ExecStart=/usr/local/sbin/rollcall-overlay system
+StandardOutput=journal+console
+
+[Install]
+WantedBy=multi-user.target

images/compute-debian13/includes/etc/systemd/system/slurmd.service.d/rollcall.conf

@@ -0,0 +1,7 @@
+[Unit]
+After=rollcall-overlay.service munge.service
+Requires=rollcall-overlay.service
+
+[Service]
+Restart=on-failure
+RestartSec=10

images/compute-debian13/includes/etc/systemd/system/ssh.service.d/rollcall.conf

@@ -0,0 +1,2 @@
+[Unit]
+After=rollcall-overlay.service

images/compute-debian13/includes/usr/local/sbin/rollcall-overlay

@@ -0,0 +1,40 @@
+#!/bin/sh
+# rollcall-overlay <system|runtime>
+# Fetches this node's overlay tarball from the rollcall server (identified by
+# the rollcall.node token on the kernel command line), applies it over / when
+# it changed, then runs /run/rollcall/post-<stage>.sh if the overlay shipped one.
+set -eu
+stage=${1:-system}
+SERVER=""; TOKEN=""
+for t in $(cat /proc/cmdline); do
+  case $t in
+    rollcall.server=*) SERVER=${t#*=} ;;
+    rollcall.node=*)   TOKEN=${t#*=} ;;
+  esac
+done
+if [ -z "$SERVER" ] || [ -z "$TOKEN" ]; then
+  echo "rollcall-overlay: rollcall.server/rollcall.node not on cmdline, nothing to do"
+  exit 0
+fi
+mkdir -p /run/rollcall
+tmp=/run/rollcall/$stage.tar.new
+i=0
+until curl -fsS --connect-timeout 5 -H "X-Rollcall-Node: $TOKEN" \
+      "$SERVER/api/v1/nodes/self/overlay?stage=$stage" -o "$tmp"; do
+  i=$((i+1))
+  if [ "$i" -ge 60 ]; then echo "rollcall-overlay: server unreachable" >&2; exit 1; fi
+  sleep 5
+done
+new=$(sha256sum "$tmp" | cut -d' ' -f1)
+old=$(cat "/run/rollcall/$stage.sha" 2>/dev/null || true)
+if [ "$new" = "$old" ]; then
+  rm -f "$tmp"
+  exit 0
+fi
+tar -C / -xpf "$tmp" --no-same-owner --overwrite
+mv -f "$tmp" "/run/rollcall/$stage.tar"
+echo "$new" > "/run/rollcall/$stage.sha"
+echo "rollcall-overlay: applied $stage overlay ($new)"
+if [ -f "/run/rollcall/post-$stage.sh" ]; then
+  sh "/run/rollcall/post-$stage.sh" || echo "rollcall-overlay: post-$stage.sh failed" >&2
+fi

images/compute-debian13/packages.list

@@ -0,0 +1,35 @@
+linux-image-amd64
+live-boot
+live-config
+live-config-systemd
+systemd-sysv
+systemd-timesyncd
+kmod
+pciutils
+ethtool
+iproute2
+isc-dhcp-client
+curl
+ca-certificates
+openssh-server
+openssh-client
+less
+vim-tiny
+procps
+numactl
+hwloc-nox
+ipmitool
+munge
+slurmd
+slurm-client
+rdma-core
+ibverbs-utils
+infiniband-diags
+nfs-common
+firmware-linux
+firmware-bnx2
+firmware-bnx2x
+firmware-qlogic
+firmware-realtek
+amd64-microcode
+intel-microcode

lab/cloud-init/pxe-server-user.yaml

@@ -41,6 +41,9 @@ write_files:
       dhcp-boot=tag:bios,pxelinux.0
       dhcp-boot=tag:efi64,efi64/syslinux.efi
       log-dhcp
+      # rollcall writes static node leases / names here and HUPs dnsmasq
+      dhcp-hostsfile=/var/lib/rollcall/dnsmasq/hosts.dhcp
+      addn-hosts=/var/lib/rollcall/dnsmasq/hosts
   - path: /srv/tftp/pxelinux.cfg/default
     content: |
       DEFAULT menu.c32