Files
rollcall/cmd/rollcall/server.go
T
Ilker ManapandClaude Fable 5.1 7fe7c26e22
CI / agent (push) Successful in 3s
CI / site (push) Successful in 3s
CI / go (push) Failing after 36s
Stateless (Warewulf-style) provisioning: profiles, node pools, overlays, dnsmasq export, compute image
Profiles bind a RAM-booted root image to overlay templates; assigning a
machine allocates a node name and management IP, writes the per-MAC PXE
entry with a per-node token and exports static leases for dnsmasq. Nodes
pull rendered overlays (hosts, slurm.conf from inventory, gres.conf, SSH
host key, munge key) through rollcall-overlay.service; the runtime refresh
timer is the heartbeat. Adds the compute-debian13 image definition and a
generic live-build root image script, CLI and web UI for profiles/assign.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
2026-10-27 02:00:14 +01:00

558 lines
16 KiB
Go

package main
import (
"context"
"crypto/subtle"
"embed"
"encoding/json"
"errors"
"fmt"
"io"
"io/fs"
"log"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"time"
)
//go:embed web
var webFS embed.FS
type App struct {
cfg *Config
store *Store
sealer *Sealer
}
type apiError struct {
Error string `json:"error"`
}
func writeJSON(w http.ResponseWriter, code int, v any) {
w.Header().Set("Content-Type", "application/json")
w.WriteHeader(code)
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
_ = enc.Encode(v)
}
func fail(w http.ResponseWriter, code int, err error) {
if errors.Is(err, errNotFound) {
code = http.StatusNotFound
}
writeJSON(w, code, apiError{err.Error()})
}
func tokenEqual(a, b string) bool {
return b != "" && subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
}
func (a *App) isAdmin(r *http.Request) bool {
h := r.Header.Get("Authorization")
return strings.HasPrefix(h, "Bearer ") && tokenEqual(strings.TrimPrefix(h, "Bearer "), a.cfg.AdminToken)
}
func (a *App) adminAuth(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if !a.isAdmin(r) {
w.Header().Set("WWW-Authenticate", `Bearer realm="rollcall"`)
fail(w, http.StatusUnauthorized, errors.New("admin token required"))
return
}
next(w, r)
}
}
func (a *App) agentAuth(next http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if a.cfg.AgentToken != "" && !tokenEqual(r.Header.Get("X-Rollcall-Token"), a.cfg.AgentToken) && !a.isAdmin(r) {
fail(w, http.StatusUnauthorized, errors.New("agent token required"))
return
}
next(w, r)
}
}
func logging(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
rw := &statusWriter{ResponseWriter: w, code: 200}
next.ServeHTTP(rw, r)
if (!strings.HasPrefix(r.URL.Path, "/rollcall/") && !strings.HasPrefix(r.URL.Path, "/images/")) || rw.code >= 400 {
log.Printf("%s %s %s %d %s", r.RemoteAddr, r.Method, r.URL.RequestURI(), rw.code, time.Since(start).Round(time.Millisecond))
}
})
}
type statusWriter struct {
http.ResponseWriter
code int
}
func (s *statusWriter) WriteHeader(c int) { s.code = c; s.ResponseWriter.WriteHeader(c) }
func (a *App) routes() http.Handler {
mux := http.NewServeMux()
// agent endpoints
mux.HandleFunc("POST /api/v1/commission", a.agentAuth(a.handleCommission))
mux.HandleFunc("GET /api/v1/bmc-hint", a.agentAuth(a.handleHint))
// management endpoints
mux.HandleFunc("GET /api/v1/machines", a.adminAuth(a.handleList))
mux.HandleFunc("GET /api/v1/machines/{id}", a.adminAuth(a.handleGet))
mux.HandleFunc("DELETE /api/v1/machines/{id}", a.adminAuth(a.handleDelete))
mux.HandleFunc("POST /api/v1/machines/{id}/recommission", a.adminAuth(a.handleRecommission))
mux.HandleFunc("GET /api/v1/machines/{id}/power", a.adminAuth(a.handlePower))
mux.HandleFunc("POST /api/v1/machines/{id}/power", a.adminAuth(a.handlePower))
mux.HandleFunc("POST /api/v1/machines/{id}/bootdev", a.adminAuth(a.handleBootdev))
mux.HandleFunc("GET /api/v1/whoami", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
writeJSON(w, 200, map[string]any{"ok": true})
}))
// stateless provisioning
mux.HandleFunc("GET /api/v1/profiles", a.adminAuth(a.handleProfiles))
mux.HandleFunc("POST /api/v1/profiles", a.adminAuth(a.handleProfiles))
mux.HandleFunc("DELETE /api/v1/profiles/{name}", a.adminAuth(a.handleProfileDelete))
mux.HandleFunc("GET /api/v1/images", a.adminAuth(a.handleImages))
mux.HandleFunc("POST /api/v1/machines/{id}/assign", a.adminAuth(a.handleAssign))
mux.HandleFunc("POST /api/v1/machines/{id}/unassign", a.adminAuth(a.handleUnassign))
mux.HandleFunc("GET /api/v1/machines/{id}/overlay", a.adminAuth(a.handleAdminOverlay))
mux.HandleFunc("GET /api/v1/render/{profile}/{path...}", a.adminAuth(a.handleRender))
mux.HandleFunc("GET /api/v1/nodes/self/overlay", a.handleNodeOverlay) // node token auth inside
// image files for the live boot (no auth: the kernel/initrd/squashfs must be fetchable)
imgDir := a.cfg.ImageDir
mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir))))
mux.Handle("GET /images/", http.StripPrefix("/images/", http.FileServer(http.Dir(a.cfg.ImagesDir))))
// embedded web UI
sub, _ := fs.Sub(webFS, "web")
mux.Handle("GET /", http.FileServer(http.FS(sub)))
return logging(mux)
}
// ---------------------------------------------------------------- helpers over inventory JSON
type obj = map[string]any
func dig(m any, path ...string) any {
cur := m
for _, p := range path {
mm, ok := cur.(obj)
if !ok {
return nil
}
cur = mm[p]
}
return cur
}
func str(v any) string {
switch t := v.(type) {
case nil:
return ""
case string:
return t
case float64:
return fmt.Sprintf("%v", t)
default:
return fmt.Sprintf("%v", t)
}
}
func strPtr(v any) *string {
s := str(v)
if s == "" {
return nil
}
return &s
}
func listLen(v any) int {
if l, ok := v.([]any); ok {
return len(l)
}
return 0
}
func normMAC(s string) string { return strings.ToLower(strings.ReplaceAll(s, "-", ":")) }
var nonAlnum = regexp.MustCompile(`[^a-z0-9]`)
func machineID(uuid, serial string, macs []string) string {
u := strings.ToLower(uuid)
if u != "" && u != "00000000-0000-0000-0000-000000000000" && !strings.Contains(u, "not ") {
u = strings.ReplaceAll(u, "-", "")
if len(u) > 12 {
u = u[:12]
}
return "m-" + u
}
if s := nonAlnum.ReplaceAllString(strings.ToLower(serial), ""); s != "" && !strings.Contains(strings.ToLower(serial), "not ") {
if len(s) > 16 {
s = s[:16]
}
return "m-" + s
}
if len(macs) > 0 {
return "m-" + strings.ReplaceAll(macs[0], ":", "")
}
return "m-" + randomToken(6)
}
// Summary is the list-view projection of a machine (no secrets, no full inventory).
type Summary struct {
ID string `json:"id"`
State string `json:"state"`
Hostname *string `json:"hostname"`
Serial *string `json:"serial"`
UUID *string `json:"uuid"`
Manufacturer *string `json:"manufacturer"`
Product *string `json:"product"`
FirstSeen time.Time `json:"first_seen"`
LastSeen *time.Time `json:"last_seen"`
MACs []string `json:"macs"`
CPU string `json:"cpu"`
CPUs string `json:"cpus"`
MemoryBytes float64 `json:"memory_bytes"`
Disks float64 `json:"disks"`
StorageBytes float64 `json:"storage_bytes"`
NICs int `json:"nics"`
IBHCAs int `json:"ib_hcas"`
GPUs int `json:"gpus"`
BMC *obj `json:"bmc"`
Profile *string `json:"profile"`
NodeName *string `json:"node_name"`
MgmtIP *string `json:"mgmt_ip"`
IBIP *string `json:"ib_ip"`
BootedAt *time.Time `json:"booted_at"`
}
func num(v any) float64 {
f, _ := v.(float64)
return f
}
func summarize(m *Machine) Summary {
var inv obj
_ = json.Unmarshal(m.Inventory, &inv)
s := Summary{ID: m.ID, State: m.State, Hostname: m.Hostname, Serial: m.Serial, UUID: m.UUID,
Manufacturer: m.Manufacturer, Product: m.Product, FirstSeen: m.FirstSeen, LastSeen: m.LastSeen, MACs: m.MACs,
CPU: str(dig(inv, "cpu", "Model name")), CPUs: str(dig(inv, "cpu", "CPU(s)")),
MemoryBytes: num(dig(inv, "memory", "total_bytes")), Disks: num(dig(inv, "storage", "disk_count")),
StorageBytes: num(dig(inv, "storage", "total_bytes")), NICs: listLen(inv["network"]),
IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: listLen(inv["gpus"]),
Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt}
if m.BMC != nil {
var b obj
if json.Unmarshal(m.BMC, &b) == nil {
s.BMC = &obj{"source": b["source"], "address": b["address"], "user": b["user"],
"configured": b["configured"], "verified_oob": b["verified_oob"]}
}
}
return s
}
// ---------------------------------------------------------------- pxelinux per-MAC files
const commissionedTmpl = `# generated by rollcall %s
# machine %s %s %s serial=%s
DEFAULT menu.c32
PROMPT 0
TIMEOUT 150
MENU TITLE %s - commissioned (%s %s, S/N %s)
LABEL poweroff
MENU LABEL Commissioned: power off (default in 15s)
MENU DEFAULT
KERNEL poweroff.c32
LABEL recommission
MENU LABEL Re-run commissioning
KERNEL rollcall/vmlinuz
INITRD rollcall/initrd.img
APPEND %s
LABEL local
MENU LABEL Boot from local disk
LOCALBOOT 0
`
func (a *App) pxePath(mac string) string {
return filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-"+strings.ReplaceAll(normMAC(mac), ":", "-"))
}
func deref(p *string) string {
if p == nil {
return ""
}
return *p
}
func (a *App) writePXE(m *Machine) error {
man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial)
body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser,
m.ID, man, prod, ser, a.cfg.CommissionAppend)
return a.writePXEBody(m, body)
}
func (a *App) writePXEBody(m *Machine, body string) error {
for _, mac := range m.MACs {
p := a.pxePath(mac)
if err := os.WriteFile(p+".tmp", []byte(body), 0o644); err != nil {
return err
}
if err := os.Rename(p+".tmp", p); err != nil {
return err
}
log.Printf("wrote %s", p)
}
return nil
}
func (a *App) removePXE(m *Machine) {
for _, mac := range m.MACs {
if err := os.Remove(a.pxePath(mac)); err == nil {
log.Printf("removed %s", a.pxePath(mac))
}
}
}
// ---------------------------------------------------------------- handlers
func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 32<<20))
if err != nil {
fail(w, 400, err)
return
}
var inv obj
if err := json.Unmarshal(body, &inv); err != nil {
fail(w, 400, fmt.Errorf("bad json: %w", err))
return
}
ident, _ := inv["identity"].(obj)
var macs []string
if l, ok := ident["macs"].([]any); ok {
for _, v := range l {
if s := normMAC(str(v)); s != "" {
macs = append(macs, s)
}
}
}
if boot := normMAC(str(ident["boot_mac"])); boot != "" {
for i, mac := range macs {
if mac == boot && i != 0 {
macs = append([]string{boot}, append(macs[:i], macs[i+1:]...)...)
break
}
}
}
m := &Machine{
ID: machineID(str(ident["uuid"]), str(ident["serial"]), macs),
UUID: strPtr(ident["uuid"]),
Serial: strPtr(ident["serial"]),
Hostname: strPtr(inv["hostname"]),
Manufacturer: strPtr(dig(inv, "system", "system-manufacturer")),
Product: strPtr(dig(inv, "system", "system-product-name")),
MACs: macs,
}
// Pull the BMC password out of the payload before anything is persisted.
var secret []byte
detail := "bmc=none"
if bmc, ok := inv["bmc"].(obj); ok {
if pw := str(bmc["password"]); pw != "" {
secret, err = a.sealer.Seal(pw, m.ID)
if err != nil {
fail(w, 500, err)
return
}
bmc["password"] = "<stored encrypted>"
}
m.BMC, _ = json.Marshal(bmc)
detail = fmt.Sprintf("bmc=%s:%s configured=%v", str(bmc["source"]), str(bmc["address"]), bmc["configured"])
}
m.Inventory, _ = json.Marshal(inv)
isNew, err := a.store.UpsertCommissioned(r.Context(), m, secret)
if err != nil {
fail(w, 500, err)
return
}
_ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail)
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
m = full // carries profile / node token when this is a re-commission of a stateless node
}
if err := a.writePXEAny(r.Context(), m); err != nil {
log.Printf("pxe write failed for %s: %v", m.ID, err)
_ = a.store.AddEvent(r.Context(), m.ID, "pxe-write-failed", err.Error())
}
log.Printf("commissioned %s new=%v (%s %s, serial %s, %d macs) %s", m.ID, isNew, deref(m.Manufacturer),
deref(m.Product), deref(m.Serial), len(macs), detail)
writeJSON(w, 200, map[string]any{"id": m.ID, "state": "ready", "new": isNew})
}
func (a *App) handleHint(w http.ResponseWriter, r *http.Request) {
want := map[string]bool{}
for _, m := range strings.Split(r.URL.Query().Get("macs"), ",") {
want[normMAC(m)] = true
}
for _, h := range a.cfg.BMCHints {
if want[normMAC(h.MAC)] {
port := h.Port
if port == 0 {
port = 623
}
writeJSON(w, 200, obj{"address": h.Address, "port": port, "user": h.User, "password": h.Password})
return
}
}
writeJSON(w, 200, obj{})
}
func (a *App) handleList(w http.ResponseWriter, r *http.Request) {
ms, err := a.store.List(r.Context())
if err != nil {
fail(w, 500, err)
return
}
out := make([]Summary, 0, len(ms))
for _, m := range ms {
out = append(out, summarize(m))
}
writeJSON(w, 200, out)
}
func (a *App) handleGet(w http.ResponseWriter, r *http.Request) {
m, err := a.store.Get(r.Context(), r.PathValue("id"))
if err != nil {
fail(w, 500, err)
return
}
evs, _ := a.store.Events(r.Context(), m.ID)
var bmc, inv any
_ = json.Unmarshal(m.BMC, &bmc)
_ = json.Unmarshal(m.Inventory, &inv)
writeJSON(w, 200, obj{"summary": summarize(m), "bmc": bmc, "has_bmc_secret": m.bmcSecret != nil,
"inventory": inv, "events": evs})
}
func (a *App) handleDelete(w http.ResponseWriter, r *http.Request) {
m, err := a.store.Get(r.Context(), r.PathValue("id"))
if err != nil {
fail(w, 500, err)
return
}
a.removePXE(m)
if err := a.store.Delete(r.Context(), m.ID); err != nil {
fail(w, 500, err)
return
}
log.Printf("deleted %s", m.ID)
writeJSON(w, 200, obj{"deleted": m.ID})
}
func (a *App) handleRecommission(w http.ResponseWriter, r *http.Request) {
m, err := a.store.Get(r.Context(), r.PathValue("id"))
if err != nil {
fail(w, 500, err)
return
}
a.removePXE(m)
_ = a.store.SetState(r.Context(), m.ID, "recommission")
_ = a.store.AddEvent(r.Context(), m.ID, "recommission requested", "")
// optional: power-cycle it right away when the caller asks (?now=1)
out := obj{"id": m.ID, "state": "recommission"}
if r.URL.Query().Get("now") == "1" {
if c, err := a.credsFor(m); err == nil {
o, ierr := a.ipmi(r.Context(), c, "chassis", "bootdev", "pxe")
o2, ierr2 := a.ipmi(r.Context(), c, "chassis", "power", "cycle")
if ierr2 != nil && strings.Contains(o2, "off") { // cycle fails when off; power on instead
o2, ierr2 = a.ipmi(r.Context(), c, "chassis", "power", "on")
}
out["ipmi"] = strings.TrimSpace(o + "\n" + o2)
out["ipmi_ok"] = ierr == nil && ierr2 == nil
} else {
out["ipmi"] = err.Error()
}
}
writeJSON(w, 200, out)
}
func (a *App) handlePower(w http.ResponseWriter, r *http.Request) {
m, err := a.store.Get(r.Context(), r.PathValue("id"))
if err != nil {
fail(w, 500, err)
return
}
action := "status"
if r.Method == http.MethodPost {
var req struct {
Action string `json:"action"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
fail(w, 400, err)
return
}
action = req.Action
}
args, ok := powerActions[action]
if !ok {
fail(w, 400, fmt.Errorf("unknown power action %q", action))
return
}
c, err := a.credsFor(m)
if err != nil {
fail(w, 409, err)
return
}
out, err := a.ipmi(r.Context(), c, args...)
if action != "status" {
_ = a.store.AddEvent(r.Context(), m.ID, "power "+action, out)
}
writeJSON(w, 200, obj{"id": m.ID, "action": action, "ok": err == nil, "output": out})
}
func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) {
m, err := a.store.Get(r.Context(), r.PathValue("id"))
if err != nil {
fail(w, 500, err)
return
}
var req struct {
Device string `json:"device"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || !bootDevices[req.Device] {
fail(w, 400, fmt.Errorf("device must be one of pxe, disk, cdrom, bios, none, safe"))
return
}
c, err := a.credsFor(m)
if err != nil {
fail(w, 409, err)
return
}
out, err := a.ipmi(r.Context(), c, "chassis", "bootdev", req.Device)
_ = a.store.AddEvent(r.Context(), m.ID, "bootdev "+req.Device, out)
writeJSON(w, 200, obj{"id": m.ID, "device": req.Device, "ok": err == nil, "output": out})
}
func serve(cfgPath string) error {
cfg, err := loadConfig(cfgPath)
if err != nil {
return err
}
sealer, err := loadOrCreateKey(cfg.MasterKeyFile)
if err != nil {
return err
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
store, err := openStore(ctx, cfg.DatabaseURL)
cancel()
if err != nil {
return fmt.Errorf("database: %w", err)
}
app := &App{cfg: cfg, store: store, sealer: sealer}
srv := &http.Server{Addr: cfg.Listen, Handler: app.routes(), ReadHeaderTimeout: 10 * time.Second}
log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "")
return srv.ListenAndServe()
}