diff --git a/README.md b/README.md index bc9cb56..937e63c 100644 --- a/README.md +++ b/README.md @@ -126,6 +126,37 @@ for a hint (`bmc_hints`) and verifies it out-of-band. Vendor quirks (iDRAC/iLO password policies, missing empty slots) are reported in `bmc.errors` and visible in the UI. +## Stateless nodes (Warewulf-style) + +For HPC compute nodes Rollcall can boot a root image straight into RAM and +dress it with per-node files, instead of installing an OS on disk. + +* **Images** live in `images_dir//{vmlinuz,initrd.img,filesystem.squashfs}` + and are built with `images/build-rootfs.sh ` from + `images//{packages.list,hook.chroot,includes/}` (live-build, same + toolchain as the commissioning image). `images/compute-debian13` is a + Debian 13 compute node: slurmd, munge, rdma-core, NFS client, firmware. +* **Profiles** bind an image to overlays, extra kernel parameters, a node name + format (`node%03d`) and variables: `rollcall profile add compute -image compute-debian13 -overlays compute`. +* **Assigning** a commissioned machine to a profile allocates a node name and + IPs from `node_pools`, writes a per-MAC PXE entry that boots the image with + `hostname=`, `rollcall.server=` and a per-node token, and exports static + DHCP leases plus host names for dnsmasq (`dnsmasq_dir`, re-read on HUP via + `rollcall-dnsmasq.path`). `rollcall assign compute -boot` also + power-cycles the machine through its BMC. +* **Overlays** are directory trees under `overlays_dir//{system,runtime}`; + files ending in `.tmpl` are Go templates rendered with the node, all nodes of + the cluster, the profile and `cluster_vars`. The `compute` overlay generates + `/etc/hosts`, `slurm.conf` (node lines from the commissioning inventory: + sockets, cores, threads, memory, GPUs), `gres.conf`, a persistent SSH host + key per node (sealed in the database), root `authorized_keys` and the munge + key. Inside the image `rollcall-overlay.service` applies the system overlay + before munge/slurmd/ssh start; a timer refreshes the runtime overlay every + five minutes, which doubles as the node heartbeat (`booted` state, last seen). +* The controller renders the same files: `rollcall render compute etc/slurm/slurm.conf > /etc/slurm/slurm.conf`. + +States: `ready → stateless → booted`; `rollcall unassign` releases name and IPs. + ## Lab `lab/` documents a Proxmox SDN test setup: an isolated vnet, a PXE server VM diff --git a/cmd/rollcall/client.go b/cmd/rollcall/client.go index 8719bf3..c024564 100644 --- a/cmd/rollcall/client.go +++ b/cmd/rollcall/client.go @@ -1,6 +1,7 @@ package main import ( + "archive/tar" "bytes" "encoding/json" "flag" @@ -8,6 +9,7 @@ import ( "io" "net/http" "os" + "path/filepath" "strings" "time" ) @@ -97,8 +99,8 @@ func runClient(cmd string, args []string) error { if err := c.do("GET", "/api/v1/machines", nil, &ms); err != nil { return err } - f := "%-16s %-12s %-28s %-16s %4s %7s %3s %7s %3s %2s %3s %s\n" - fmt.Printf(f, "ID", "STATE", "MODEL", "SERIAL", "CPU", "RAM", "DSK", "STORAGE", "NIC", "IB", "GPU", "BMC") + f := "%-16s %-12s %-10s %-10s %-14s %-24s %4s %7s %3s %7s %3s %2s %3s %s\n" + fmt.Printf(f, "ID", "STATE", "NODE", "PROFILE", "MGMT-IP", "MODEL", "CPU", "RAM", "DSK", "STORAGE", "NIC", "IB", "GPU", "BMC") for _, m := range ms { bmc := "-" if m.BMC != nil { @@ -110,7 +112,7 @@ func runClient(cmd string, args []string) error { bmc = fmt.Sprintf("%s (%s,%s)", str(b["address"]), str(b["source"]), ok) } model := strings.TrimSpace(deref(m.Manufacturer) + " " + deref(m.Product)) - fmt.Printf(f, m.ID, m.State, trunc(model, 28), trunc(deref(m.Serial), 16), m.CPUs, human(m.MemoryBytes), + fmt.Printf(f, m.ID, m.State, deref(m.NodeName), deref(m.Profile), deref(m.MgmtIP), trunc(model, 24), m.CPUs, human(m.MemoryBytes), fmt.Sprint(int(m.Disks)), human(m.StorageBytes), fmt.Sprint(m.NICs), fmt.Sprint(m.IBHCAs), fmt.Sprint(m.GPUs), bmc) } case "show": @@ -175,10 +177,172 @@ func runClient(cmd string, args []string) error { return err } fmt.Println("deleted", str(d["deleted"])) + default: + return c.statelessCmd(cmd, args) } return nil } +func (c *client) statelessCmd(cmd string, args []string) error { + switch cmd { + case "images": + var imgs []obj + if err := c.do("GET", "/api/v1/images", nil, &imgs); err != nil { + return err + } + fmt.Printf("%-24s %-9s %10s %10s %10s %s\n", "IMAGE", "COMPLETE", "KERNEL", "INITRD", "SQUASHFS", "BUILT") + for _, i := range imgs { + fmt.Printf("%-24s %-9v %10s %10s %10s %s\n", str(i["name"]), i["complete"], human(num(i["vmlinuz"])), + human(num(i["initrd.img"])), human(num(i["filesystem.squashfs"])), str(i["built"])) + } + case "profile": + if len(args) < 1 { + usage() + } + switch args[0] { + case "list": + var ps []Profile + if err := c.do("GET", "/api/v1/profiles", nil, &ps); err != nil { + return err + } + fmt.Printf("%-14s %-22s %-24s %-12s %s\n", "PROFILE", "IMAGE", "OVERLAYS", "NAMES", "APPEND") + for _, p := range ps { + fmt.Printf("%-14s %-22s %-24s %-12s %s\n", p.Name, p.Image, strings.Join(p.Overlays, ","), p.NameFormat, p.Append) + } + case "add": + if len(args) < 2 { + usage() + } + fs := flag.NewFlagSet("profile add", flag.ExitOnError) + img := fs.String("image", "", "image directory name (required)") + app := fs.String("append", "", "extra kernel parameters") + ov := fs.String("overlays", "", "comma-separated overlay names") + nf := fs.String("name-format", "node%03d", "node name format") + vars := fs.String("vars", "{}", "profile variables as JSON") + _ = fs.Parse(args[2:]) + p := Profile{Name: args[1], Image: *img, Append: *app, NameFormat: *nf} + if *ov != "" { + p.Overlays = strings.Split(*ov, ",") + } + if err := json.Unmarshal([]byte(*vars), &p.Vars); err != nil { + return fmt.Errorf("-vars: %w", err) + } + var out Profile + if err := c.do("POST", "/api/v1/profiles", p, &out); err != nil { + return err + } + fmt.Printf("profile %s: image=%s overlays=%v\n", out.Name, out.Image, out.Overlays) + case "rm": + if len(args) < 2 { + usage() + } + var d obj + if err := c.do("DELETE", "/api/v1/profiles/"+args[1], nil, &d); err != nil { + return err + } + fmt.Println("deleted", str(d["deleted"])) + default: + usage() + } + case "assign": + if len(args) < 2 { + usage() + } + fs := flag.NewFlagSet("assign", flag.ExitOnError) + name := fs.String("name", "", "node name (default: next free from the profile's name format)") + boot := fs.Bool("boot", false, "set bootdev pxe and power-cycle via BMC") + _ = fs.Parse(args[2:]) + var d obj + if err := c.do("POST", "/api/v1/machines/"+args[0]+"/assign", obj{"profile": args[1], "name": *name, "boot": *boot}, &d); err != nil { + return err + } + fmt.Printf("%s -> %s as %s mgmt=%s ib=%s\n", str(d["id"]), str(d["profile"]), str(d["name"]), str(d["mgmt_ip"]), str(d["ib_ip"])) + if d["ipmi"] != nil { + fmt.Println(str(d["ipmi"])) + } + case "unassign": + if len(args) < 1 { + usage() + } + var d obj + if err := c.do("POST", "/api/v1/machines/"+args[0]+"/unassign", nil, &d); err != nil { + return err + } + fmt.Printf("%s -> %s\n", str(d["id"]), str(d["state"])) + case "overlay": + if len(args) < 1 { + usage() + } + fs := flag.NewFlagSet("overlay", flag.ExitOnError) + stage := fs.String("stage", "system", "system or runtime") + extract := fs.String("extract", "", "extract into this directory instead of listing") + _ = fs.Parse(args[1:]) + data, err := c.raw("GET", "/api/v1/machines/"+args[0]+"/overlay?stage="+*stage) + if err != nil { + return err + } + tr := tar.NewReader(bytes.NewReader(data)) + for { + h, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + if *extract == "" { + fmt.Printf("%s %8d %s\n", os.FileMode(h.Mode).String(), h.Size, h.Name) + continue + } + dst := filepath.Join(*extract, h.Name) + if h.Typeflag == tar.TypeDir { + _ = os.MkdirAll(dst, 0o755) + continue + } + _ = os.MkdirAll(filepath.Dir(dst), 0o755) + f, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, os.FileMode(h.Mode)) + if err != nil { + return err + } + _, _ = io.Copy(f, tr) + f.Close() + } + case "render": + if len(args) < 2 { + usage() + } + fs := flag.NewFlagSet("render", flag.ExitOnError) + stage := fs.String("stage", "system", "system or runtime") + _ = fs.Parse(args[2:]) + data, err := c.raw("GET", "/api/v1/render/"+args[0]+"/"+strings.TrimPrefix(args[1], "/")+"?stage="+*stage) + if err != nil { + return err + } + os.Stdout.Write(data) + default: + usage() + } + return nil +} + +func (c *client) raw(method, path string) ([]byte, error) { + req, err := http.NewRequest(method, c.base+path, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Bearer "+c.token) + resp, err := c.http.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 400 { + return nil, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(data))) + } + return data, nil +} + func trunc(s string, n int) string { if len(s) > n { return s[:n] diff --git a/cmd/rollcall/config.go b/cmd/rollcall/config.go index 63fdade..4341199 100644 --- a/cmd/rollcall/config.go +++ b/cmd/rollcall/config.go @@ -4,6 +4,8 @@ import ( "encoding/json" "fmt" "os" + "path/filepath" + "strings" ) // Config is /etc/rollcall/config.json. @@ -18,6 +20,14 @@ type Config struct { CommissionAppend string `json:"commission_append"` // kernel cmdline used in generated pxelinux files IPMITool string `json:"ipmitool"` // path, default "ipmitool" BMCHints []BMCHint `json:"bmc_hints"` // out-of-band BMC mapping for machines without in-band IPMI + + // stateless provisioning + ServerURL string `json:"server_url"` // how nodes reach this server, e.g. http://10.213.77.1 + ImagesDir string `json:"images_dir"` // //{vmlinuz,initrd.img,filesystem.squashfs}, served under /images/ + OverlaysDir string `json:"overlays_dir"` // //{system,runtime}/... (Go templates with .tmpl suffix) + DnsmasqDir string `json:"dnsmasq_dir"` // hosts.dhcp + hosts written here for dnsmasq (dhcp-hostsfile / addn-hosts); "" disables + NodePools map[string]string `json:"node_pools"` // {"mgmt": "10.0.0.50-10.0.0.99", "ib": "10.1.0.10-10.1.0.250"} + ClusterVars map[string]any `json:"cluster_vars"` // available to overlay templates as .Vars (profile vars override) } // BMCHint maps a machine MAC to a known BMC (virtual lab nodes). @@ -44,5 +54,19 @@ func loadConfig(path string) (*Config, error) { return nil, fmt.Errorf("%s: %s is required", path, k.n) } } + if c.ServerURL == "" { + host, _, _ := strings.Cut(c.Listen, ":") + if host == "" || host == "0.0.0.0" { + return nil, fmt.Errorf("%s: server_url is required when listen is not a specific address", path) + } + c.ServerURL = "http://" + c.Listen + } + c.ServerURL = strings.TrimRight(c.ServerURL, "/") + if c.ImagesDir == "" { + c.ImagesDir = filepath.Join(filepath.Dir(c.ImageDir), "images") + } + if c.OverlaysDir == "" { + c.OverlaysDir = "/etc/rollcall/overlays" + } return c, nil } diff --git a/cmd/rollcall/go.mod b/cmd/rollcall/go.mod index 7d7ee22..7ef15e6 100644 --- a/cmd/rollcall/go.mod +++ b/cmd/rollcall/go.mod @@ -2,13 +2,16 @@ module rollcall go 1.24 -require github.com/jackc/pgx/v5 v5.7.5 +require ( + github.com/jackc/pgx/v5 v5.7.5 + golang.org/x/crypto v0.37.0 +) require ( github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect github.com/jackc/puddle/v2 v2.2.2 // indirect - golang.org/x/crypto v0.37.0 // indirect golang.org/x/sync v0.13.0 // indirect + golang.org/x/sys v0.32.0 // indirect golang.org/x/text v0.24.0 // indirect ) diff --git a/cmd/rollcall/go.sum b/cmd/rollcall/go.sum index 85a678b..f600d5d 100644 --- a/cmd/rollcall/go.sum +++ b/cmd/rollcall/go.sum @@ -20,6 +20,10 @@ golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index a6b6044..308acea 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -30,6 +30,12 @@ func usage() { rollcall bootdev ID pxe|disk|cdrom|bios|none rollcall recommission ID [-now] rollcall delete ID + rollcall images + rollcall profile list | add NAME -image IMG [-append ARGS] [-overlays a,b] [-name-format node%03d] [-vars JSON] | rm NAME + rollcall assign ID PROFILE [-name node001] [-boot] + rollcall unassign ID + rollcall overlay ID [-stage system|runtime] [-extract DIR] + rollcall render PROFILE PATH [-stage system|runtime] env: ROLLCALL_URL (default from config listen), ROLLCALL_TOKEN, ROLLCALL_CONFIG`) os.Exit(2) } @@ -50,7 +56,8 @@ func main() { } case "keygen": fmt.Println(randomToken(24)) - case "list", "show", "events", "power", "bootdev", "recommission", "delete": + case "list", "show", "events", "power", "bootdev", "recommission", "delete", + "images", "profile", "assign", "unassign", "overlay", "render": if err := runClient(cmd, args); err != nil { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index b819dc5..6e61ec7 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -81,7 +81,7 @@ func logging(next http.Handler) http.Handler { start := time.Now() rw := &statusWriter{ResponseWriter: w, code: 200} next.ServeHTTP(rw, r) - if !strings.HasPrefix(r.URL.Path, "/rollcall/") || rw.code >= 400 { + if (!strings.HasPrefix(r.URL.Path, "/rollcall/") && !strings.HasPrefix(r.URL.Path, "/images/")) || rw.code >= 400 { log.Printf("%s %s %s %d %s", r.RemoteAddr, r.Method, r.URL.RequestURI(), rw.code, time.Since(start).Round(time.Millisecond)) } }) @@ -110,9 +110,20 @@ func (a *App) routes() http.Handler { mux.HandleFunc("GET /api/v1/whoami", a.adminAuth(func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"ok": true}) })) + // stateless provisioning + mux.HandleFunc("GET /api/v1/profiles", a.adminAuth(a.handleProfiles)) + mux.HandleFunc("POST /api/v1/profiles", a.adminAuth(a.handleProfiles)) + mux.HandleFunc("DELETE /api/v1/profiles/{name}", a.adminAuth(a.handleProfileDelete)) + mux.HandleFunc("GET /api/v1/images", a.adminAuth(a.handleImages)) + mux.HandleFunc("POST /api/v1/machines/{id}/assign", a.adminAuth(a.handleAssign)) + mux.HandleFunc("POST /api/v1/machines/{id}/unassign", a.adminAuth(a.handleUnassign)) + mux.HandleFunc("GET /api/v1/machines/{id}/overlay", a.adminAuth(a.handleAdminOverlay)) + mux.HandleFunc("GET /api/v1/render/{profile}/{path...}", a.adminAuth(a.handleRender)) + mux.HandleFunc("GET /api/v1/nodes/self/overlay", a.handleNodeOverlay) // node token auth inside // image files for the live boot (no auth: the kernel/initrd/squashfs must be fetchable) imgDir := a.cfg.ImageDir mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir)))) + mux.Handle("GET /images/", http.StripPrefix("/images/", http.FileServer(http.Dir(a.cfg.ImagesDir)))) // embedded web UI sub, _ := fs.Sub(webFS, "web") mux.Handle("GET /", http.FileServer(http.FS(sub))) @@ -209,6 +220,11 @@ type Summary struct { IBHCAs int `json:"ib_hcas"` GPUs int `json:"gpus"` BMC *obj `json:"bmc"` + Profile *string `json:"profile"` + NodeName *string `json:"node_name"` + MgmtIP *string `json:"mgmt_ip"` + IBIP *string `json:"ib_ip"` + BootedAt *time.Time `json:"booted_at"` } func num(v any) float64 { @@ -224,7 +240,8 @@ func summarize(m *Machine) Summary { CPU: str(dig(inv, "cpu", "Model name")), CPUs: str(dig(inv, "cpu", "CPU(s)")), MemoryBytes: num(dig(inv, "memory", "total_bytes")), Disks: num(dig(inv, "storage", "disk_count")), StorageBytes: num(dig(inv, "storage", "total_bytes")), NICs: listLen(inv["network"]), - IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: listLen(inv["gpus"])} + IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: listLen(inv["gpus"]), + Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt} if m.BMC != nil { var b obj if json.Unmarshal(m.BMC, &b) == nil { @@ -275,6 +292,10 @@ func (a *App) writePXE(m *Machine) error { man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial) body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser, m.ID, man, prod, ser, a.cfg.CommissionAppend) + return a.writePXEBody(m, body) +} + +func (a *App) writePXEBody(m *Machine, body string) error { for _, mac := range m.MACs { p := a.pxePath(mac) if err := os.WriteFile(p+".tmp", []byte(body), 0o644); err != nil { @@ -358,7 +379,10 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) { return } _ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail) - if err := a.writePXE(m); err != nil { + if full, err := a.store.Get(r.Context(), m.ID); err == nil { + m = full // carries profile / node token when this is a re-commission of a stateless node + } + if err := a.writePXEAny(r.Context(), m); err != nil { log.Printf("pxe write failed for %s: %v", m.ID, err) _ = a.store.AddEvent(r.Context(), m.ID, "pxe-write-failed", err.Error()) } diff --git a/cmd/rollcall/stateless.go b/cmd/rollcall/stateless.go new file mode 100644 index 0000000..6548b72 --- /dev/null +++ b/cmd/rollcall/stateless.go @@ -0,0 +1,883 @@ +package main + +// Stateless (Warewulf-style) provisioning: profiles, node naming and IP pools, +// per-node overlays rendered from templates, dnsmasq host export and the +// per-MAC PXE entry that boots a profile's root image from RAM. + +import ( + "archive/tar" + "bytes" + "context" + "crypto/ed25519" + "crypto/sha256" + "encoding/binary" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "io/fs" + "log" + "net" + "net/http" + "os" + "path" + "path/filepath" + "sort" + "strconv" + "strings" + "text/template" + "time" + + "golang.org/x/crypto/ssh" +) + +// ---------------------------------------------------------------- profiles + +type Profile struct { + Name string `json:"name"` + Image string `json:"image"` // directory under images_dir: vmlinuz, initrd.img, filesystem.squashfs + Append string `json:"append"` // extra kernel parameters + Overlays []string `json:"overlays"` // directories under overlays_dir + NameFormat string `json:"name_format"` // e.g. node%03d + Vars map[string]any `json:"vars"` + Created time.Time `json:"created"` +} + +func (s *Store) Profiles(ctx context.Context) ([]Profile, error) { + rows, err := s.pool.Query(ctx, `SELECT name, image, coalesce(append,''), overlays, coalesce(name_format,''), vars, created FROM profiles ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []Profile{} + for rows.Next() { + var p Profile + var ov, vars []byte + if err := rows.Scan(&p.Name, &p.Image, &p.Append, &ov, &p.NameFormat, &vars, &p.Created); err != nil { + return nil, err + } + _ = json.Unmarshal(ov, &p.Overlays) + _ = json.Unmarshal(vars, &p.Vars) + out = append(out, p) + } + return out, rows.Err() +} + +func (s *Store) Profile(ctx context.Context, name string) (*Profile, error) { + ps, err := s.Profiles(ctx) + if err != nil { + return nil, err + } + for i := range ps { + if ps[i].Name == name { + return &ps[i], nil + } + } + return nil, errNotFound +} + +func (s *Store) PutProfile(ctx context.Context, p Profile) error { + if p.Overlays == nil { + p.Overlays = []string{} + } + if p.Vars == nil { + p.Vars = map[string]any{} + } + if p.NameFormat == "" { + p.NameFormat = "node%03d" + } + ov, _ := json.Marshal(p.Overlays) + vars, _ := json.Marshal(p.Vars) + _, err := s.pool.Exec(ctx, `INSERT INTO profiles(name, image, append, overlays, name_format, vars) VALUES ($1,$2,$3,$4,$5,$6) + ON CONFLICT (name) DO UPDATE SET image=$2, append=$3, overlays=$4, name_format=$5, vars=$6`, + p.Name, p.Image, p.Append, ov, p.NameFormat, vars) + return err +} + +func (s *Store) DeleteProfile(ctx context.Context, name string) error { + var n int + _ = s.pool.QueryRow(ctx, `SELECT count(*) FROM machines WHERE profile=$1`, name).Scan(&n) + if n > 0 { + return fmt.Errorf("profile %s is assigned to %d machine(s)", name, n) + } + tag, err := s.pool.Exec(ctx, `DELETE FROM profiles WHERE name=$1`, name) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return errNotFound + } + return nil +} + +// ---------------------------------------------------------------- allocation + +type nodeAssignment struct { + Profile string + Name string + MgmtIP string + IBIP string + TokenHash string + Token []byte // sealed +} + +func (s *Store) Assign(ctx context.Context, id string, n nodeAssignment) error { + tag, err := s.pool.Exec(ctx, `UPDATE machines SET profile=$2, node_name=$3, mgmt_ip=$4, ib_ip=$5, + node_token_hash=$6, node_token=$7, state='stateless', booted_at=NULL WHERE id=$1`, + id, n.Profile, n.Name, nilIfEmpty(n.MgmtIP), nilIfEmpty(n.IBIP), n.TokenHash, n.Token) + if err != nil { + return err + } + if tag.RowsAffected() == 0 { + return errNotFound + } + return nil +} + +func (s *Store) Unassign(ctx context.Context, id string) error { + _, err := s.pool.Exec(ctx, `UPDATE machines SET profile=NULL, node_name=NULL, mgmt_ip=NULL, ib_ip=NULL, + node_token_hash=NULL, node_token=NULL, state='ready', booted_at=NULL WHERE id=$1`, id) + return err +} + +func (s *Store) ByNodeToken(ctx context.Context, hash string) (*Machine, error) { + return scanMachine(s.pool.QueryRow(ctx, `SELECT `+machineCols+` FROM machines WHERE node_token_hash=$1`, hash), true) +} + +func (s *Store) MarkBooted(ctx context.Context, id string) error { + _, err := s.pool.Exec(ctx, `UPDATE machines SET state='booted', booted_at=coalesce(booted_at, now()), last_seen=now() WHERE id=$1`, id) + return err +} + +func (s *Store) SetHostKey(ctx context.Context, id string, sealed []byte) error { + _, err := s.pool.Exec(ctx, `UPDATE machines SET host_key=$2 WHERE id=$1`, id, sealed) + return err +} + +func nilIfEmpty(s string) *string { + if s == "" { + return nil + } + return &s +} + +// parsePool turns "10.0.0.50-10.0.0.99" into the ordered list of addresses. +func parsePool(spec string) ([]string, error) { + lo, hi, ok := strings.Cut(strings.TrimSpace(spec), "-") + if !ok { + return nil, fmt.Errorf("pool %q: want A.B.C.D-A.B.C.D", spec) + } + a, b := net.ParseIP(strings.TrimSpace(lo)).To4(), net.ParseIP(strings.TrimSpace(hi)).To4() + if a == nil || b == nil { + return nil, fmt.Errorf("pool %q: bad address", spec) + } + ai, bi := binary.BigEndian.Uint32(a), binary.BigEndian.Uint32(b) + if bi < ai || bi-ai > 65536 { + return nil, fmt.Errorf("pool %q: bad range", spec) + } + out := make([]string, 0, bi-ai+1) + for i := ai; i <= bi; i++ { + ip := make(net.IP, 4) + binary.BigEndian.PutUint32(ip, i) + out = append(out, ip.String()) + } + return out, nil +} + +func (a *App) allocate(ctx context.Context, p *Profile, wantName string) (name, mgmt, ib string, err error) { + ms, err := a.store.List(ctx) + if err != nil { + return + } + usedName, usedIP := map[string]bool{}, map[string]bool{} + for _, m := range ms { + if m.NodeName != nil { + usedName[*m.NodeName] = true + } + if m.MgmtIP != nil { + usedIP[*m.MgmtIP] = true + } + if m.IBIP != nil { + usedIP[*m.IBIP] = true + } + } + name = wantName + if name == "" { + f := p.NameFormat + if f == "" { + f = "node%03d" + } + for i := 1; i < 100000; i++ { + if c := fmt.Sprintf(f, i); !usedName[c] { + name = c + break + } + } + } else if usedName[name] { + err = fmt.Errorf("node name %s already in use", name) + return + } + pick := func(pool string) (string, error) { + spec := a.cfg.NodePools[pool] + if spec == "" { + return "", nil + } + ips, err := parsePool(spec) + if err != nil { + return "", err + } + for _, ip := range ips { + if !usedIP[ip] { + usedIP[ip] = true + return ip, nil + } + } + return "", fmt.Errorf("pool %s exhausted", pool) + } + if mgmt, err = pick("mgmt"); err != nil { + return + } + ib, err = pick("ib") + return +} + +// ---------------------------------------------------------------- template context + +type NodeCtx struct { + ID, Name, Profile, State string + BootMAC string + MACs []string + MgmtIP, IBIP string + Sockets, CoresPerSocket int + ThreadsPerCore, CPUs int + MemMB int + GPUs int + IBGUIDs []string + HostKey, HostKeyPub string // only filled for the node being rendered +} + +type overlayCtx struct { + Node NodeCtx + Nodes []NodeCtx // all machines with a profile, sorted by name + Profile Profile + Vars map[string]any // cluster_vars overlaid with profile vars + Server string + Now string +} + +func atoi(v any) int { + n, _ := strconv.Atoi(strings.TrimSpace(str(v))) + return n +} + +func nodeCtx(m *Machine) NodeCtx { + var inv obj + _ = json.Unmarshal(m.Inventory, &inv) + n := NodeCtx{ID: m.ID, Name: deref(m.NodeName), Profile: deref(m.Profile), State: m.State, MACs: m.MACs, + MgmtIP: deref(m.MgmtIP), IBIP: deref(m.IBIP)} + if len(m.MACs) > 0 { + n.BootMAC = m.MACs[0] + } + n.Sockets = atoi(dig(inv, "cpu", "Socket(s)")) + n.CoresPerSocket = atoi(dig(inv, "cpu", "Core(s) per socket")) + n.ThreadsPerCore = atoi(dig(inv, "cpu", "Thread(s) per core")) + n.CPUs = atoi(dig(inv, "cpu", "CPU(s)")) + if n.Sockets == 0 { + n.Sockets = 1 + } + if n.CoresPerSocket == 0 { + n.CoresPerSocket = max(1, n.CPUs/max(1, n.ThreadsPerCore)) + } + if n.ThreadsPerCore == 0 { + n.ThreadsPerCore = 1 + } + if n.CPUs == 0 { + n.CPUs = n.Sockets * n.CoresPerSocket * n.ThreadsPerCore + } + n.MemMB = int(num(dig(inv, "memory", "total_bytes")) / (1024 * 1024)) + // count real accelerators only (NVIDIA, AMD, or PCI class 0302 "3D controller"); emulated VGA is not a GPU + for _, g := range toList(inv["gpus"]) { + vid, cls := str(dig(g, "vendor_id")), str(dig(g, "class_id")) + if vid == "10de" || vid == "1002" || cls == "0302" || cls == "1200" { + n.GPUs++ + } + } + for _, h := range toList(dig(inv, "infiniband", "hcas")) { + if g := str(dig(h, "node_guid")); g != "" { + n.IBGUIDs = append(n.IBGUIDs, g) + } + } + return n +} + +var tmplFuncs = template.FuncMap{ + "join": strings.Join, + "upper": strings.ToUpper, + "lower": strings.ToLower, + "replace": strings.ReplaceAll, + "default": func(d, v any) any { + if v == nil || v == "" || v == 0 || v == false { + return d + } + return v + }, + "mul": func(a, b int) int { return a * b }, + "sub": func(a, b int) int { return a - b }, + "pct": func(v, p int) int { return v * p / 100 }, + "seq": func(n int) []int { + s := make([]int, n) + for i := range s { + s[i] = i + } + return s + }, + "json": func(v any) string { b, _ := json.Marshal(v); return string(b) }, +} + +func (a *App) hostKeyFor(ctx context.Context, m *Machine) (priv, pub string, err error) { + if m.hostKey != nil { + if pemStr, err := a.sealer.Open(m.hostKey, m.ID); err == nil { + priv = pemStr + } + } + if priv == "" { + pubk, privk, err := ed25519.GenerateKey(nil) + if err != nil { + return "", "", err + } + block, err := ssh.MarshalPrivateKey(privk, "rollcall "+m.ID) + if err != nil { + return "", "", err + } + priv = string(pem.EncodeToMemory(block)) + _ = pubk + sealed, err := a.sealer.Seal(priv, m.ID) + if err != nil { + return "", "", err + } + if err := a.store.SetHostKey(ctx, m.ID, sealed); err != nil { + return "", "", err + } + } + signer, err := ssh.ParsePrivateKey([]byte(priv)) + if err != nil { + return "", "", err + } + pub = strings.TrimSpace(string(ssh.MarshalAuthorizedKey(signer.PublicKey()))) + return priv, pub, nil +} + +func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) { + p, err := a.store.Profile(ctx, deref(m.Profile)) + if err != nil { + return nil, fmt.Errorf("profile %q: %w", deref(m.Profile), err) + } + ms, err := a.store.List(ctx) + if err != nil { + return nil, err + } + var nodes []NodeCtx + for _, x := range ms { + if x.Profile != nil && x.NodeName != nil { + nodes = append(nodes, nodeCtx(x)) + } + } + sort.Slice(nodes, func(i, j int) bool { return nodes[i].Name < nodes[j].Name }) + me := nodeCtx(m) + if me.Name != "" { + if me.HostKey, me.HostKeyPub, err = a.hostKeyFor(ctx, m); err != nil { + return nil, err + } + } + vars := map[string]any{} + for k, v := range a.cfg.ClusterVars { + vars[k] = v + } + for k, v := range p.Vars { + vars[k] = v + } + return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Vars: vars, Server: a.cfg.ServerURL, + Now: time.Now().Format(time.RFC3339)}, nil +} + +// renderOverlay produces a tar of every overlay dir of the node's profile for the stage. +func (a *App) renderOverlay(ctx context.Context, m *Machine, stage string) ([]byte, error) { + if stage != "system" && stage != "runtime" { + return nil, fmt.Errorf("stage must be system or runtime") + } + oc, err := a.buildCtx(ctx, m) + if err != nil { + return nil, err + } + var buf bytes.Buffer + tw := tar.NewWriter(&buf) + seen := map[string]bool{} + for _, name := range oc.Profile.Overlays { + root := filepath.Join(a.cfg.OverlaysDir, name, stage) + if st, err := os.Stat(root); err != nil || !st.IsDir() { + continue + } + err := filepath.WalkDir(root, func(p string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + rel, _ := filepath.Rel(root, p) + if rel == "." { + return nil + } + info, err := d.Info() + if err != nil { + return err + } + if info.Mode()&fs.ModeSymlink != 0 { // follow symlinks (e.g. runtime -> system templates) + if info, err = os.Stat(p); err != nil { + return err + } + } + hdr := &tar.Header{Name: rel, Mode: int64(secretMode(rel, info.Mode().Perm())), ModTime: info.ModTime(), Uname: "root", Gname: "root"} + if d.IsDir() { + if seen[rel] { + return nil + } + seen[rel] = true + hdr.Typeflag, hdr.Name = tar.TypeDir, rel+"/" + return tw.WriteHeader(hdr) + } + data, err := os.ReadFile(p) + if err != nil { + return err + } + if strings.HasSuffix(rel, ".tmpl") { + t, err := template.New(rel).Funcs(tmplFuncs).Option("missingkey=zero").Parse(string(data)) + if err != nil { + return fmt.Errorf("%s: %w", p, err) + } + var out bytes.Buffer + if err := t.Execute(&out, oc); err != nil { + return fmt.Errorf("%s: %w", p, err) + } + data, hdr.Name = out.Bytes(), strings.TrimSuffix(rel, ".tmpl") + } + if seen[hdr.Name] { // later overlays override earlier ones; tar extraction keeps the last entry + log.Printf("overlay %s: %s overrides an earlier file", name, hdr.Name) + } + seen[hdr.Name] = true + hdr.Typeflag, hdr.Size = tar.TypeReg, int64(len(data)) + if err := tw.WriteHeader(hdr); err != nil { + return err + } + _, err = tw.Write(data) + return err + }) + if err != nil { + return nil, err + } + } + if err := tw.Close(); err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +// secretMode tightens permissions for files git cannot carry modes for. +func secretMode(rel string, mode fs.FileMode) fs.FileMode { + base := strings.TrimSuffix(path.Base(rel), ".tmpl") + switch { + case base == "munge.key": + return 0o400 + case strings.HasPrefix(base, "ssh_host_") && !strings.HasSuffix(base, ".pub"), + base == "authorized_keys", strings.HasSuffix(base, ".key"), strings.HasSuffix(base, ".secret"): + return 0o600 + } + return mode +} + +// renderFile renders one template path of a profile with a cluster-wide context +// (used for files the controller needs too, e.g. slurm.conf). +func (a *App) renderFile(ctx context.Context, profile, stage, rel string) ([]byte, error) { + p, err := a.store.Profile(ctx, profile) + if err != nil { + return nil, err + } + ms, err := a.store.List(ctx) + if err != nil { + return nil, err + } + var m *Machine + for _, x := range ms { + if deref(x.Profile) == profile { + m = x + break + } + } + if m == nil { // no node yet: synthesize an empty one so cluster-level templates still render + m = &Machine{ID: "none", Profile: &p.Name} + } + oc, err := a.buildCtx(ctx, m) + if err != nil { + return nil, err + } + for _, name := range p.Overlays { + for _, cand := range []string{rel + ".tmpl", rel} { + fp := filepath.Join(a.cfg.OverlaysDir, name, stage, cand) + data, err := os.ReadFile(fp) + if err != nil { + continue + } + if !strings.HasSuffix(cand, ".tmpl") { + return data, nil + } + t, err := template.New(rel).Funcs(tmplFuncs).Option("missingkey=zero").Parse(string(data)) + if err != nil { + return nil, err + } + var out bytes.Buffer + if err := t.Execute(&out, oc); err != nil { + return nil, err + } + return out.Bytes(), nil + } + } + return nil, errNotFound +} + +// ---------------------------------------------------------------- PXE + dnsmasq + +const statelessTmpl = `# generated by rollcall %s +# node %s machine %s profile %s +DEFAULT menu.c32 +PROMPT 0 +TIMEOUT 30 +MENU TITLE %s - stateless (%s) + +LABEL node + MENU LABEL Boot %s (%s) from RAM + MENU DEFAULT + KERNEL images/%s/vmlinuz + INITRD images/%s/initrd.img + APPEND %s + +LABEL recommission + MENU LABEL Re-run commissioning + KERNEL rollcall/vmlinuz + INITRD rollcall/initrd.img + APPEND %s + +LABEL local + MENU LABEL Boot from local disk + LOCALBOOT 0 +` + +func (a *App) statelessAppend(m *Machine, p *Profile, token string) string { + parts := []string{"boot=live", "components", + "fetch=" + a.cfg.ServerURL + "/images/" + p.Image + "/filesystem.squashfs", + "hostname=" + deref(m.NodeName), + "live-config.nocomponents=user-setup,sudo", + "rollcall.server=" + a.cfg.ServerURL, "rollcall.node=" + token} + if p.Append != "" { + parts = append(parts, p.Append) + } + return strings.Join(parts, " ") +} + +func (a *App) writePXEStateless(m *Machine, p *Profile, token string) error { + body := fmt.Sprintf(statelessTmpl, time.Now().Format(time.RFC3339), deref(m.NodeName), m.ID, p.Name, + deref(m.NodeName), p.Name, deref(m.NodeName), p.Name, p.Image, p.Image, a.statelessAppend(m, p, token), + a.cfg.CommissionAppend) + return a.writePXEBody(m, body) +} + +// writePXEAny picks the right per-MAC content for the machine's current state. +func (a *App) writePXEAny(ctx context.Context, m *Machine) error { + if m.Profile != nil && m.nodeToken != nil { + p, err := a.store.Profile(ctx, *m.Profile) + if err != nil { + return err + } + token, err := a.sealer.Open(m.nodeToken, m.ID) + if err != nil { + return err + } + return a.writePXEStateless(m, p, token) + } + return a.writePXE(m) +} + +// exportDnsmasq writes dhcp-hostsfile and addn-hosts files; a systemd path unit HUPs dnsmasq. +func (a *App) exportDnsmasq(ctx context.Context) error { + if a.cfg.DnsmasqDir == "" { + return nil + } + ms, err := a.store.List(ctx) + if err != nil { + return err + } + domain := str(a.cfg.ClusterVars["domain"]) + var dhcp, hosts bytes.Buffer + fmt.Fprintf(&dhcp, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339)) + fmt.Fprintf(&hosts, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339)) + for _, m := range ms { + if m.NodeName == nil || len(m.MACs) == 0 { + continue + } + name := *m.NodeName + if m.MgmtIP != nil { + fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], *m.MgmtIP, name) + if domain != "" { + fmt.Fprintf(&hosts, "%s %s.%s %s\n", *m.MgmtIP, name, domain, name) + } else { + fmt.Fprintf(&hosts, "%s %s\n", *m.MgmtIP, name) + } + } + if m.IBIP != nil { + fmt.Fprintf(&hosts, "%s %s-ib\n", *m.IBIP, name) + } + } + if err := os.MkdirAll(a.cfg.DnsmasqDir, 0o755); err != nil { + return err + } + for fn, b := range map[string]*bytes.Buffer{"hosts.dhcp": &dhcp, "hosts": &hosts} { + p := filepath.Join(a.cfg.DnsmasqDir, fn) + if err := os.WriteFile(p+".tmp", b.Bytes(), 0o644); err != nil { + return err + } + if err := os.Rename(p+".tmp", p); err != nil { + return err + } + } + return nil +} + +// ---------------------------------------------------------------- handlers + +func (a *App) handleProfiles(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + ps, err := a.store.Profiles(r.Context()) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, ps) + case http.MethodPost: + var p Profile + if err := json.NewDecoder(r.Body).Decode(&p); err != nil || p.Name == "" || p.Image == "" { + fail(w, 400, fmt.Errorf("name and image are required")) + return + } + if strings.ContainsAny(p.Name, "/ \t") || strings.ContainsAny(p.Image, "/ \t") { + fail(w, 400, fmt.Errorf("name/image must be plain identifiers")) + return + } + if err := a.store.PutProfile(r.Context(), p); err != nil { + fail(w, 500, err) + return + } + // profile changes affect every assigned node's PXE entry and overlays + ms, _ := a.store.List(r.Context()) + for _, m := range ms { + if deref(m.Profile) == p.Name { + if err := a.writePXEAny(r.Context(), m); err != nil { + log.Printf("pxe rewrite %s: %v", m.ID, err) + } + } + } + writeJSON(w, 200, p) + } +} + +func (a *App) handleProfileDelete(w http.ResponseWriter, r *http.Request) { + if err := a.store.DeleteProfile(r.Context(), r.PathValue("name")); err != nil { + fail(w, 409, err) + return + } + writeJSON(w, 200, obj{"deleted": r.PathValue("name")}) +} + +func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) { + m, err := a.store.Get(r.Context(), r.PathValue("id")) + if err != nil { + fail(w, 500, err) + return + } + var req struct { + Profile string `json:"profile"` + Name string `json:"name"` + Boot bool `json:"boot"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Profile == "" { + fail(w, 400, fmt.Errorf("profile is required")) + return + } + p, err := a.store.Profile(r.Context(), req.Profile) + if err != nil { + fail(w, 404, fmt.Errorf("profile %s: %w", req.Profile, err)) + return + } + if _, err := os.Stat(filepath.Join(a.cfg.ImagesDir, p.Image, "filesystem.squashfs")); err != nil { + fail(w, 409, fmt.Errorf("image %s is not built (%s)", p.Image, err)) + return + } + // keep the existing name/IPs when re-assigning an already named node + name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP) + if name == "" || (req.Name != "" && req.Name != name) { + if name, mgmt, ib, err = a.allocate(r.Context(), p, req.Name); err != nil { + fail(w, 409, err) + return + } + } + token := randomToken(16) + sum := sha256.Sum256([]byte(token)) + sealed, err := a.sealer.Seal(token, m.ID) + if err != nil { + fail(w, 500, err) + return + } + if err := a.store.Assign(r.Context(), m.ID, nodeAssignment{Profile: p.Name, Name: name, MgmtIP: mgmt, IBIP: ib, + TokenHash: hex.EncodeToString(sum[:]), Token: sealed}); err != nil { + fail(w, 500, err) + return + } + m, _ = a.store.Get(r.Context(), m.ID) + if err := a.writePXEStateless(m, p, token); err != nil { + fail(w, 500, err) + return + } + if err := a.exportDnsmasq(r.Context()); err != nil { + log.Printf("dnsmasq export: %v", err) + } + _ = a.store.AddEvent(r.Context(), m.ID, "assigned", fmt.Sprintf("profile=%s name=%s mgmt=%s ib=%s", p.Name, name, mgmt, ib)) + out := obj{"id": m.ID, "state": "stateless", "profile": p.Name, "name": name, "mgmt_ip": mgmt, "ib_ip": ib} + if req.Boot { + out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m) + } + log.Printf("assigned %s -> %s as %s (%s)", m.ID, p.Name, name, mgmt) + writeJSON(w, 200, out) +} + +// pxeBoot sets bootdev pxe and power-cycles (or powers on) through the BMC. +func (a *App) pxeBoot(ctx context.Context, m *Machine) (string, bool) { + c, err := a.credsFor(m) + if err != nil { + return err.Error(), false + } + o1, e1 := a.ipmi(ctx, c, "chassis", "bootdev", "pxe") + st, _ := a.ipmi(ctx, c, "chassis", "power", "status") + var o2 string + var e2 error + if strings.Contains(st, "off") { + o2, e2 = a.ipmi(ctx, c, "chassis", "power", "on") + } else { + o2, e2 = a.ipmi(ctx, c, "chassis", "power", "cycle") + } + _ = a.store.AddEvent(ctx, m.ID, "pxe boot", strings.TrimSpace(o1+"\n"+o2)) + return strings.TrimSpace(o1 + "\n" + o2), e1 == nil && e2 == nil +} + +func (a *App) handleUnassign(w http.ResponseWriter, r *http.Request) { + m, err := a.store.Get(r.Context(), r.PathValue("id")) + if err != nil { + fail(w, 500, err) + return + } + if err := a.store.Unassign(r.Context(), m.ID); err != nil { + fail(w, 500, err) + return + } + m, _ = a.store.Get(r.Context(), m.ID) + _ = a.writePXE(m) + _ = a.exportDnsmasq(r.Context()) + _ = a.store.AddEvent(r.Context(), m.ID, "unassigned", "") + writeJSON(w, 200, obj{"id": m.ID, "state": "ready"}) +} + +func (a *App) serveOverlay(w http.ResponseWriter, r *http.Request, m *Machine) { + stage := r.URL.Query().Get("stage") + if stage == "" { + stage = "system" + } + data, err := a.renderOverlay(r.Context(), m, stage) + if err != nil { + fail(w, 500, err) + return + } + sum := sha256.Sum256(data) + w.Header().Set("Content-Type", "application/x-tar") + w.Header().Set("ETag", `"`+hex.EncodeToString(sum[:8])+`"`) + w.Header().Set("X-Rollcall-Node-Name", deref(m.NodeName)) + w.Header().Set("Content-Length", strconv.Itoa(len(data))) + _, _ = w.Write(data) +} + +// handleNodeOverlay is what a booted node calls; the token comes from its kernel cmdline. +func (a *App) handleNodeOverlay(w http.ResponseWriter, r *http.Request) { + tok := r.Header.Get("X-Rollcall-Node") + if tok == "" { + fail(w, 401, errors.New("X-Rollcall-Node required")) + return + } + sum := sha256.Sum256([]byte(tok)) + m, err := a.store.ByNodeToken(r.Context(), hex.EncodeToString(sum[:])) + if err != nil { + fail(w, 401, errors.New("unknown node token")) + return + } + if m.State != "booted" { + _ = a.store.AddEvent(r.Context(), m.ID, "booted", r.RemoteAddr) + log.Printf("node %s (%s) booted from %s", deref(m.NodeName), m.ID, r.RemoteAddr) + } + _ = a.store.MarkBooted(r.Context(), m.ID) + a.serveOverlay(w, r, m) +} + +func (a *App) handleAdminOverlay(w http.ResponseWriter, r *http.Request) { + m, err := a.store.Get(r.Context(), r.PathValue("id")) + if err != nil { + fail(w, 500, err) + return + } + if m.Profile == nil { + fail(w, 409, errors.New("machine has no profile")) + return + } + a.serveOverlay(w, r, m) +} + +func (a *App) handleRender(w http.ResponseWriter, r *http.Request) { + stage := r.URL.Query().Get("stage") + if stage == "" { + stage = "system" + } + data, err := a.renderFile(r.Context(), r.PathValue("profile"), stage, path.Clean(r.PathValue("path"))) + if err != nil { + fail(w, 500, err) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + _, _ = w.Write(data) +} + +func (a *App) handleImages(w http.ResponseWriter, r *http.Request) { + ents, err := os.ReadDir(a.cfg.ImagesDir) + if err != nil { + writeJSON(w, 200, []any{}) + return + } + out := []obj{} + for _, e := range ents { + if !e.IsDir() { + continue + } + info := obj{"name": e.Name()} + for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} { + if st, err := os.Stat(filepath.Join(a.cfg.ImagesDir, e.Name(), f)); err == nil { + info[f] = st.Size() + info["built"] = st.ModTime() + } + } + info["complete"] = info["filesystem.squashfs"] != nil && info["vmlinuz"] != nil && info["initrd.img"] != nil + out = append(out, info) + } + writeJSON(w, 200, out) +} diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index 05b9956..f8b0790 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -38,6 +38,24 @@ CREATE TABLE IF NOT EXISTS events ( detail text ); CREATE INDEX IF NOT EXISTS events_machine ON events(machine_id, ts); +CREATE TABLE IF NOT EXISTS profiles ( + name text PRIMARY KEY, + image text NOT NULL, + append text, + overlays jsonb NOT NULL DEFAULT '[]', + name_format text, + vars jsonb NOT NULL DEFAULT '{}', + created timestamptz NOT NULL DEFAULT now() +); +ALTER TABLE machines ADD COLUMN IF NOT EXISTS profile text REFERENCES profiles(name); +ALTER TABLE machines ADD COLUMN IF NOT EXISTS node_name text UNIQUE; +ALTER TABLE machines ADD COLUMN IF NOT EXISTS mgmt_ip text; +ALTER TABLE machines ADD COLUMN IF NOT EXISTS ib_ip text; +ALTER TABLE machines ADD COLUMN IF NOT EXISTS node_token_hash text; +ALTER TABLE machines ADD COLUMN IF NOT EXISTS node_token bytea; +ALTER TABLE machines ADD COLUMN IF NOT EXISTS host_key bytea; +ALTER TABLE machines ADD COLUMN IF NOT EXISTS booted_at timestamptz; +CREATE INDEX IF NOT EXISTS machines_node_token ON machines(node_token_hash); ` var errNotFound = errors.New("not found") @@ -58,7 +76,14 @@ type Machine struct { MACs []string `json:"macs"` BMC json.RawMessage `json:"bmc,omitempty"` Inventory json.RawMessage `json:"inventory,omitempty"` + Profile *string `json:"profile"` + NodeName *string `json:"node_name"` + MgmtIP *string `json:"mgmt_ip"` + IBIP *string `json:"ib_ip"` + BootedAt *time.Time `json:"booted_at"` bmcSecret []byte + nodeToken []byte + hostKey []byte } type Event struct { @@ -79,13 +104,14 @@ func openStore(ctx context.Context, url string) (*Store, error) { } const machineCols = `id, uuid, serial, hostname, manufacturer, product, state, first_seen, last_seen, - commissioned_at, macs, bmc, bmc_secret, inventory` + commissioned_at, macs, bmc, bmc_secret, inventory, profile, node_name, mgmt_ip, ib_ip, node_token, host_key, booted_at` func scanMachine(row pgx.Row, full bool) (*Machine, error) { m := &Machine{} var macs []byte err := row.Scan(&m.ID, &m.UUID, &m.Serial, &m.Hostname, &m.Manufacturer, &m.Product, &m.State, - &m.FirstSeen, &m.LastSeen, &m.CommissionedAt, &macs, &m.BMC, &m.bmcSecret, &m.Inventory) + &m.FirstSeen, &m.LastSeen, &m.CommissionedAt, &macs, &m.BMC, &m.bmcSecret, &m.Inventory, + &m.Profile, &m.NodeName, &m.MgmtIP, &m.IBIP, &m.nodeToken, &m.hostKey, &m.BootedAt) if errors.Is(err, pgx.ErrNoRows) { return nil, errNotFound } @@ -163,8 +189,10 @@ func (s *Store) UpsertCommissioned(ctx context.Context, m *Machine, secret []byt return false, err } } + // a node that already has a profile goes back to 'stateless', everything else to 'ready' _, err = tx.Exec(ctx, `UPDATE machines SET uuid=$2, serial=$3, hostname=$4, manufacturer=$5, product=$6, - state='ready', last_seen=now(), commissioned_at=now(), macs=$7, bmc=$8, bmc_secret=$9, inventory=$10 WHERE id=$1`, + state=CASE WHEN profile IS NULL THEN 'ready' ELSE 'stateless' END, booted_at=NULL, + last_seen=now(), commissioned_at=now(), macs=$7, bmc=$8, bmc_secret=$9, inventory=$10 WHERE id=$1`, m.ID, m.UUID, m.Serial, m.Hostname, m.Manufacturer, m.Product, macs, m.BMC, secret, m.Inventory) if err != nil { return false, err diff --git a/cmd/rollcall/web/app.js b/cmd/rollcall/web/app.js index 1043fa1..50f3aa5 100644 --- a/cmd/rollcall/web/app.js +++ b/cmd/rollcall/web/app.js @@ -35,15 +35,38 @@ $('#back').onclick = () => { current = null; $('#detail').hidden = true; $('#lis function setStatus(msg, bad) { const el = $('#status'); el.textContent = msg; el.className = bad ? 'bad' : 'muted'; } +let profiles = []; async function loadList() { if (!token) return askToken(); try { machines = await api('GET', '/api/v1/machines'); setStatus(`updated ${new Date().toLocaleTimeString()}`); renderList(); + loadProfiles(); } catch (e) { if (e.message !== 'unauthorized') setStatus(e.message, true); } } +async function loadProfiles() { + try { + profiles = await api('GET', '/api/v1/profiles'); + const images = await api('GET', '/api/v1/images'); + $('#profiles').innerHTML = table(['Profile', 'Image', 'Overlays', 'Names', 'Nodes', ''], profiles.map(p => [esc(p.name), esc(p.image), esc((p.overlays || []).join(', ')), esc(p.name_format), machines.filter(m => m.profile === p.name).length, ``])); + $('#images').innerHTML = table(['Image', 'Complete', 'Squashfs', 'Built'], images.map(i => [esc(i.name), i.complete ? 'yes' : 'no', human(i['filesystem.squashfs']), esc(i.built ? new Date(i.built).toLocaleString() : '')])); + document.querySelectorAll('[data-rmprofile]').forEach(b => b.onclick = async () => { if (confirm(`Delete profile ${b.dataset.rmprofile}?`)) { try { await api('DELETE', '/api/v1/profiles/' + b.dataset.rmprofile); } catch (e) { alert(e.message); } loadProfiles(); } }); + const sel = $('#assign-profile'); const cur = sel.value; + sel.innerHTML = '' + profiles.map(p => ``).join(''); + sel.value = cur; + } catch (e) { /* non-fatal */ } +} +$('#profile-form').onsubmit = async ev => { + ev.preventDefault(); + const f = new FormData(ev.target); + let vars = {}; + try { vars = JSON.parse(f.get('vars') || '{}'); } catch (e) { $('#profile-out').textContent = 'vars: invalid JSON'; return; } + const body = { name: f.get('name'), image: f.get('image'), append: f.get('append'), name_format: f.get('name_format'), vars, overlays: (f.get('overlays') || '').split(',').map(s => s.trim()).filter(Boolean) }; + try { await api('POST', '/api/v1/profiles', body); $('#profile-out').textContent = 'saved'; loadProfiles(); } catch (e) { $('#profile-out').textContent = e.message; } +}; + function renderList() { const q = $('#filter').value.toLowerCase(); const rows = machines.filter(m => !q || JSON.stringify([m.id, m.state, m.manufacturer, m.product, m.serial, m.macs, m.hostname]).toLowerCase().includes(q)); @@ -54,6 +77,7 @@ function renderList() { const bmc = b ? `${esc(b.address || '-')} ${esc(b.source)}` : '-'; return ` ${esc(m.id)}${esc(m.state)} + ${esc(m.node_name || '')}${m.mgmt_ip ? `
${esc(m.mgmt_ip)}` : ''}${esc(m.profile || '')} ${esc((m.manufacturer || '') + ' ' + (m.product || ''))}${esc(m.serial || '')} ${esc(m.cpus)}${human(m.memory_bytes)}${m.disks | 0} ${human(m.storage_bytes)}${m.nics}${m.ib_hcas}${m.gpus} @@ -77,10 +101,17 @@ async function loadDetail(id) { function renderDetail(d) { const s = d.summary, inv = d.inventory || {}, sys = inv.system || {}, cpu = inv.cpu || {}, mem = inv.memory || {}, st = inv.storage || {}, b = d.bmc || {}; - $('#d-title').textContent = `${s.id} ${sys['system-manufacturer'] || ''} ${sys['system-product-name'] || ''}`; + $('#d-title').textContent = `${s.node_name ? s.node_name + ' · ' : ''}${s.id} ${sys['system-manufacturer'] || ''} ${sys['system-product-name'] || ''}`; $('#d-state').textContent = s.state; $('#d-state').className = 'pill ' + s.state; $('#action-out').textContent = ''; + if (s.profile) $('#assign-profile').value = s.profile; + $('#assign-name').value = s.node_name || ''; const cards = []; + if (s.profile) cards.push(card('Stateless node', kv([ + ['Node name', esc(s.node_name)], ['Profile', esc(s.profile)], ['Management IP', `${esc(s.mgmt_ip)}`], + ['InfiniBand IP', s.ib_ip ? `${esc(s.ib_ip)}` : ''], ['Booted', esc(s.booted_at ? new Date(s.booted_at).toLocaleString() : 'not yet')], + ['Last heartbeat', esc(s.last_seen ? ago(s.last_seen) : '')], + ]))); cards.push(card('System', kv([ ['Manufacturer', esc(sys['system-manufacturer'])], ['Product', esc(sys['system-product-name'])], ['Serial', esc(sys['system-serial-number'])], ['UUID', `${esc(sys['system-uuid'])}`], ['SKU', esc(sys['system-sku-number'])], @@ -149,6 +180,14 @@ document.querySelectorAll('[data-power]').forEach(b => b.onclick = () => action( document.querySelectorAll('[data-bootdev]').forEach(b => b.onclick = () => action(async () => { const r = await api('POST', `/api/v1/machines/${current}/bootdev`, { device: b.dataset.bootdev }); return r.output || (r.ok ? 'ok' : 'failed'); }, 'bootdev ' + b.dataset.bootdev)); +const doAssign = boot => action(async () => { + const profile = $('#assign-profile').value; if (!profile) throw new Error('pick a profile'); + const r = await api('POST', `/api/v1/machines/${current}/assign`, { profile, name: $('#assign-name').value.trim(), boot }); + await loadDetail(current); return `${r.name} (${r.mgmt_ip || 'no pool'})${r.ipmi ? ' · ' + r.ipmi : ''}`; +}, boot ? 'assign + boot' : 'assign'); +$('#assign').onclick = () => doAssign(false); +$('#assign-boot').onclick = () => { if (confirm('Assign and power-cycle this machine into the stateless image?')) doAssign(true); }; +$('#unassign').onclick = () => { if (confirm('Remove the profile? The node name and IP are released.')) action(async () => { const r = await api('POST', `/api/v1/machines/${current}/unassign`); await loadDetail(current); return 'state ' + r.state; }, 'unassign'); }; $('#recommission').onclick = () => action(async () => { const r = await api('POST', `/api/v1/machines/${current}/recommission`); await loadDetail(current); return `state ${r.state}: next PXE boot runs commissioning`; }, 'recommission'); $('#recommission-now').onclick = () => { if (confirm('Remove PXE override, set bootdev=pxe and power-cycle this machine via its BMC?')) action(async () => { const r = await api('POST', `/api/v1/machines/${current}/recommission?now=1`); await loadDetail(current); return r.ipmi || r.state; }, 'recommission now'); }; $('#delete').onclick = () => { if (confirm(`Delete ${current}? It will be commissioned again as a new machine on next PXE boot.`)) action(async () => { await api('DELETE', `/api/v1/machines/${current}`); $('#back').click(); return 'deleted'; }, 'delete'); }; diff --git a/cmd/rollcall/web/index.html b/cmd/rollcall/web/index.html index 5a074ad..b54ba33 100644 --- a/cmd/rollcall/web/index.html +++ b/cmd/rollcall/web/index.html @@ -33,13 +33,32 @@ - +
IDStateModelSerialCPURAMIDStateNodeProfileModelSerialCPURAM DisksStorageNICIBGPU BMCLast seen
+ +
+ Profiles & images (stateless provisioning) +
+

Profiles

+

Images

+

Add / update profile

+
+ + + + + + + +
+
+
+