From-scratch install: deploy/bootstrap.sh and rollcall doctor

bootstrap.sh sets up a region on a fresh Debian 13 host (packages,
PostgreSQL, TFTP tree, config, services, verified commissioning image,
provisioning subnet with ranges/DHCP, optional NAT, first OS image).
rollcall doctor checks an installation and says what to fix, including a
DHCPDISCOVER probe for foreign DHCP servers. pxelinux-default no longer
hard-codes the lab address.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

CHANGELOG.md

@@ -73,6 +73,12 @@ publish a tag that has no section here.
 - Admin client certificates: `rollcall admin-cert request NAME` (key stays on the client, CSR only),
   `list`, `revoke`; the region accepts them as admin over HTTPS; CLI/console use them from
   `client.json`, the web UI works without a token when the browser presents one (`.p12` written).
+- `deploy/bootstrap.sh`: a region on a fresh Debian 13 host in one command (packages, PostgreSQL,
+  TFTP tree, generated config, services, commissioning image downloaded and verified or built,
+  provisioning subnet with ranges and DHCP, optional NAT, first OS image).
+- `rollcall doctor`: checks configuration, database, listeners, certificates, TFTP files, image
+  checksums, dnsmasq, DNS as machines see it, clock, foreign DHCP servers on the provisioning
+  network (DHCPDISCOVER probe), rack controllers, OS images and IPAM, with what to do.
 - `rollcall version`; CD-built binary carries `git describe`.
 - Release notes from `CHANGELOG.md`.
 

cmd/rollcall/doctor.go

@@ -0,0 +1,484 @@
+package main
+
+// rollcall doctor: checks an installation from the region host and says what
+// to do about what is missing. Local checks read the configuration; checks of
+// the running state ask the region's API with its own admin token.
+
+import (
+	"bufio"
+	"context"
+	"crypto/rand"
+	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/binary"
+	"encoding/hex"
+	"flag"
+	"fmt"
+	"io"
+	"net"
+	"net/http"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+)
+
+type doctor struct {
+	fails, warns int
+}
+
+func (d *doctor) ok(name, detail string) {
+	fmt.Printf("  \033[32mOK  \033[0m %-24s %s\n", name, detail)
+}
+func (d *doctor) warn(name, detail, fix string) {
+	d.warns++
+	fmt.Printf("  \033[33mWARN\033[0m %-24s %s\n", name, detail)
+	if fix != "" {
+		fmt.Printf("       %-24s -> %s\n", "", fix)
+	}
+}
+func (d *doctor) fail(name, detail, fix string) {
+	d.fails++
+	fmt.Printf("  \033[31mFAIL\033[0m %-24s %s\n", name, detail)
+	if fix != "" {
+		fmt.Printf("       %-24s -> %s\n", "", fix)
+	}
+}
+
+func doctorCmd(args []string) error {
+	fs := flag.NewFlagSet("doctor", flag.ExitOnError)
+	cfgPath := fs.String("config", envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"), "config file")
+	deep := fs.Bool("deep", false, "also verify the commissioning image checksums")
+	noProbe := fs.Bool("no-dhcp-probe", false, "do not send a DHCPDISCOVER on the provisioning interface")
+	_ = fs.Parse(args)
+	d := &doctor{}
+	fmt.Println("rollcall doctor", version)
+
+	cfg, err := loadConfig(*cfgPath)
+	if err != nil {
+		d.fail("config", err.Error(), "deploy/bootstrap.sh writes one; or see deploy/config.example.json")
+		return d.done()
+	}
+	d.ok("config", *cfgPath)
+	srvHost := ""
+	if u, err := urlHost(cfg.ServerURL); err == nil {
+		srvHost = u
+	}
+	iface := ifaceWithAddr(srvHost)
+	if iface == "" {
+		d.warn("server_url", cfg.ServerURL+" is not an address of this host", "machines reach the region there; set server_url to this host's address on the provisioning network")
+	} else {
+		d.ok("server_url", fmt.Sprintf("%s on %s", cfg.ServerURL, iface))
+	}
+
+	// database
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	st, err := openStore(ctx, cfg.DatabaseURL)
+	cancel()
+	if err != nil {
+		d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)")
+	} else {
+		d.ok("database", "connected, schema current")
+		st.pool.Close()
+	}
+
+	// listeners
+	ports := listeningPorts()
+	for _, p := range []struct {
+		proto string
+		port  int
+		what  string
+	}{{"udp", 67, "DHCP (dnsmasq)"}, {"udp", 69, "TFTP (dnsmasq)"}, {"tcp", portOf(cfg.Listen, 80), "HTTP (rollcall)"}, {"tcp", portOf(cfg.TLSListen, 443), "HTTPS (rollcall)"}} {
+		if p.port == 0 {
+			continue
+		}
+		if ports[fmt.Sprintf("%s/%d", p.proto, p.port)] {
+			d.ok(fmt.Sprintf("%s/%d", p.proto, p.port), p.what)
+		} else if p.port == 67 {
+			d.warn("udp/67", "nothing listens for DHCP", "no VLAN with dhcp on this rack yet (rollcall vlan add FABRIC VID -dhcp -rack "+cfg.RackName+"), or dnsmasq is down")
+		} else {
+			d.fail(fmt.Sprintf("%s/%d", p.proto, p.port), "nothing listens for "+p.what, "systemctl status rollcall dnsmasq")
+		}
+	}
+	hc := &http.Client{Timeout: 5 * time.Second}
+	if resp, err := hc.Get("http://127.0.0.1" + localPort(cfg.Listen) + "/ca.pem"); err != nil || resp.StatusCode != 200 {
+		d.fail("http", fmt.Sprintf("GET /ca.pem: %v", errOrStatus(err, resp)), "journalctl -u rollcall")
+	} else {
+		resp.Body.Close()
+		d.ok("http", "region answers")
+	}
+
+	// certificates
+	if cb, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil {
+		if c, err := parseCertPEM(cb); err == nil {
+			left := time.Until(c.NotAfter)
+			switch {
+			case left < 90*24*time.Hour:
+				d.warn("CA", "expires "+c.NotAfter.Format("2006-01-02"), "plan rollcall ca rotate")
+			default:
+				d.ok("CA", fmt.Sprintf("until %s, delegation %v", c.NotAfter.Format("2006-01-02"), (&CA{Cert: c}).canDelegate()))
+			}
+			if cfg.TLSListen != "" {
+				pool := x509.NewCertPool()
+				pool.AddCert(c)
+				conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", "127.0.0.1"+localPort(cfg.TLSListen),
+					&tls.Config{RootCAs: pool, ServerName: srvHost})
+				if err != nil {
+					d.fail("https", err.Error(), "the region certificate should cover "+srvHost+" (tls_names); journalctl -u rollcall")
+				} else {
+					leaf := conn.ConnectionState().PeerCertificates[0]
+					conn.Close()
+					d.ok("https", fmt.Sprintf("certificate for %s until %s", srvHost, leaf.NotAfter.Format("2006-01-02")))
+				}
+			}
+		}
+	} else {
+		d.fail("CA", "no CA yet", "it is created when rollcall serve starts")
+	}
+
+	// TFTP + image
+	missing := []string{}
+	for _, f := range []string{"pxelinux.0", "ldlinux.c32", "menu.c32", "libcom32.c32", "libutil.c32", "efi64/syslinux.efi", "efi64/ldlinux.e64", "pxelinux.cfg/default"} {
+		if _, err := os.Stat(filepath.Join(cfg.TFTPRoot, f)); err != nil {
+			missing = append(missing, f)
+		}
+	}
+	if len(missing) > 0 {
+		d.fail("tftp", "missing in "+cfg.TFTPRoot+": "+strings.Join(missing, " "), "deploy/bootstrap.sh (step 3) copies them from pxelinux/syslinux-efi")
+	} else {
+		d.ok("tftp", cfg.TFTPRoot+" (BIOS + UEFI)")
+	}
+	img := []string{}
+	for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} {
+		if st, err := os.Stat(filepath.Join(cfg.ImageDir, f)); err != nil || st.Size() == 0 {
+			img = append(img, f)
+		}
+	}
+	switch {
+	case len(img) > 0:
+		d.fail("image", "commissioning image incomplete, missing "+strings.Join(img, " "), "deploy/bootstrap.sh --image-url URL or --build-image")
+	case *deep:
+		if bad := verifySums(cfg.ImageDir); bad == "no SHA256SUMS next to the image" {
+			d.warn("image", bad, "cd "+cfg.ImageDir+" && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS")
+		} else if bad != "" {
+			d.fail("image", bad, "download the image again")
+		} else {
+			d.ok("image", "checksums match SHA256SUMS")
+		}
+	default:
+		d.ok("image", cfg.ImageDir)
+	}
+
+	// dnsmasq
+	if out, _ := exec.Command("systemctl", "is-active", "dnsmasq").Output(); strings.TrimSpace(string(out)) != "active" {
+		d.fail("dnsmasq", "not running", "journalctl -u dnsmasq -n 20")
+	} else if _, err := os.Stat(filepath.Join(cfg.DnsmasqDir, "rollcall.conf.rejected")); err == nil {
+		d.fail("dnsmasq", "the generated configuration was rejected; the last working one is in use",
+			"journalctl -u rollcall-dnsmasq; a hand-written DHCP/TFTP file in /etc/dnsmasq.d usually conflicts")
+	} else {
+		d.ok("dnsmasq", "running")
+	}
+
+	// DNS: outside, and through dnsmasq as the machines see it
+	if _, err := net.DefaultResolver.LookupHost(context.Background(), "deb.debian.org"); err != nil {
+		d.fail("dns (host)", err.Error(), "this host cannot resolve; check /etc/resolv.conf / resolvectl")
+	} else {
+		d.ok("dns (host)", "resolves outside names")
+	}
+	if srvHost != "" && ports["udp/53"] {
+		r := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
+			return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "udp", net.JoinHostPort(srvHost, "53"))
+		}}
+		if _, err := r.LookupHost(context.Background(), "deb.debian.org"); err != nil {
+			d.fail("dns (machines)", "dnsmasq on "+srvHost+" does not resolve outside names: "+err.Error(),
+				"the generated dnsmasq configuration needs an upstream (rollcall >= 0.2 writes one); check its # upstream section")
+		} else {
+			d.ok("dns (machines)", "dnsmasq on "+srvHost+" forwards outside names")
+		}
+	}
+
+	// time
+	if out, err := exec.Command("timedatectl", "show", "-p", "NTPSynchronized", "--value").Output(); err == nil {
+		if strings.TrimSpace(string(out)) == "yes" {
+			d.ok("time", "synchronised")
+		} else {
+			d.warn("time", "clock not synchronised", "certificates and tokens depend on it: systemctl enable --now systemd-timesyncd")
+		}
+	}
+
+	// other DHCP servers on the provisioning network
+	if iface != "" && !*noProbe {
+		servers, err := dhcpProbe(iface, 4*time.Second)
+		switch {
+		case err != nil:
+			d.warn("dhcp probe", err.Error(), "run as root; or --no-dhcp-probe")
+		default:
+			var foreign []string
+			for _, s := range servers {
+				if ifaceWithAddr(s) == "" {
+					foreign = append(foreign, s)
+				}
+			}
+			switch {
+			case len(foreign) > 0:
+				d.fail("dhcp probe", "another DHCP server answers on "+iface+": "+strings.Join(foreign, ", "),
+					"machines may PXE-boot from it; switch it off for this VLAN, or use --no-dhcp and let it hand out next-server "+srvHost)
+			case len(servers) == 0:
+				d.warn("dhcp probe", "no DHCP server answered on "+iface, "is a VLAN with dhcp assigned to this rack? (rollcall vlan add ... -dhcp)")
+			default:
+				d.ok("dhcp probe", "only this host answers on "+iface)
+			}
+		}
+	}
+
+	// running state through the API
+	if c, err := newClient(); err == nil {
+		var racks []RackController
+		if err := c.do("GET", "/api/v1/controllers", nil, &racks); err == nil {
+			regionVer := version
+			for _, r := range racks {
+				if r.Kind == "embedded" && r.Version != "" {
+					regionVer = r.Version // what the region runs (this binary may differ)
+				}
+			}
+			for _, r := range racks {
+				switch {
+				case !r.Online:
+					d.fail("rack "+r.Name, "offline since "+timeStr(r.LastSeen), "on that host: systemctl status rollcall-rack")
+				case r.Version != regionVer && r.Version != "":
+					d.warn("rack "+r.Name, "runs "+r.Version+", region "+regionVer, "it updates itself within minutes (rollcall-rack-update.path); see /var/lib/rollcall/update/last-result there")
+				default:
+					d.ok("rack "+r.Name, fmt.Sprintf("online, %d machines", r.Machines))
+				}
+			}
+		}
+		var oss []map[string]any
+		if err := c.do("GET", "/api/v1/os", nil, &oss); err == nil {
+			if len(oss) == 0 {
+				d.warn("os images", "none", "runuser -u rollcall -- rollcall os sync debian-13")
+			} else {
+				d.ok("os images", fmt.Sprintf("%d in the catalog", len(oss)))
+			}
+		}
+		var ipam map[string]any
+		if err := c.do("GET", "/api/v1/ipam", nil, &ipam); err == nil && ipam["provider"] != "" && ipam["provider"] != nil {
+			last, _ := ipam["last_sync"].(map[string]any)
+			if e, _ := last["error"].(string); e != "" {
+				d.fail("ipam", e, "check ipam.url/token_file/ca_file; rollcall ipam sync")
+			} else {
+				d.ok("ipam", fmt.Sprintf("%v in sync", ipam["provider"]))
+			}
+		}
+	} else {
+		d.warn("api", err.Error(), "")
+	}
+	return d.done()
+}
+
+func (d *doctor) done() error {
+	fmt.Printf("\n%d failed, %d warnings\n", d.fails, d.warns)
+	if d.fails > 0 {
+		os.Exit(1)
+	}
+	return nil
+}
+
+func errOrStatus(err error, resp *http.Response) string {
+	if err != nil {
+		return err.Error()
+	}
+	return resp.Status
+}
+
+func urlHost(u string) (string, error) {
+	u = strings.TrimPrefix(strings.TrimPrefix(u, "http://"), "https://")
+	u = strings.SplitN(u, "/", 2)[0]
+	if h, _, err := net.SplitHostPort(u); err == nil {
+		return strings.Trim(h, "[]"), nil
+	}
+	return strings.Trim(u, "[]"), nil
+}
+
+// ifaceWithAddr: the interface holding ip on this host, "" if none.
+func ifaceWithAddr(ip string) string {
+	want := net.ParseIP(ip)
+	if want == nil {
+		return ""
+	}
+	ifs, _ := net.Interfaces()
+	for _, i := range ifs {
+		addrs, _ := i.Addrs()
+		for _, a := range addrs {
+			if n, ok := a.(*net.IPNet); ok && n.IP.Equal(want) {
+				return i.Name
+			}
+		}
+	}
+	return ""
+}
+
+func portOf(listen string, def int) int {
+	if listen == "" {
+		return 0
+	}
+	_, p, err := net.SplitHostPort(listen)
+	if err != nil {
+		return def
+	}
+	n, _ := strconv.Atoi(p)
+	return n
+}
+
+func localPort(listen string) string {
+	if p := portOf(listen, 0); p != 0 && p != 80 && p != 443 {
+		return ":" + strconv.Itoa(p)
+	}
+	if p := portOf(listen, 0); p == 443 {
+		return ":443"
+	}
+	return ""
+}
+
+// listeningPorts reads /proc/net: "tcp/80", "udp/67" ...
+func listeningPorts() map[string]bool {
+	out := map[string]bool{}
+	for _, f := range []string{"tcp", "tcp6", "udp", "udp6"} {
+		fh, err := os.Open("/proc/net/" + f)
+		if err != nil {
+			continue
+		}
+		sc := bufio.NewScanner(fh)
+		sc.Scan()
+		for sc.Scan() {
+			fl := strings.Fields(sc.Text())
+			if len(fl) < 4 {
+				continue
+			}
+			if strings.HasPrefix(f, "tcp") && fl[3] != "0A" { // LISTEN
+				continue
+			}
+			i := strings.LastIndex(fl[1], ":")
+			p, err := strconv.ParseUint(fl[1][i+1:], 16, 16)
+			if err == nil {
+				out[strings.TrimSuffix(f, "6")+"/"+strconv.Itoa(int(p))] = true
+			}
+		}
+		fh.Close()
+	}
+	return out
+}
+
+func verifySums(dir string) string {
+	b, err := os.ReadFile(filepath.Join(dir, "SHA256SUMS"))
+	if err != nil {
+		return "no SHA256SUMS next to the image"
+	}
+	for _, line := range strings.Split(string(b), "\n") {
+		f := strings.Fields(line)
+		if len(f) != 2 {
+			continue
+		}
+		name := strings.TrimPrefix(f[1], "*")
+		fh, err := os.Open(filepath.Join(dir, name))
+		if err != nil {
+			return name + ": " + err.Error()
+		}
+		h := sha256.New()
+		_, _ = io.Copy(h, fh)
+		fh.Close()
+		if hex.EncodeToString(h.Sum(nil)) != f[0] {
+			return name + ": checksum mismatch"
+		}
+	}
+	return ""
+}
+
+// dhcpProbe broadcasts a DHCPDISCOVER on iface and returns the server
+// identifiers that offer an address. Needs root (port 68, SO_BINDTODEVICE).
+func dhcpProbe(iface string, wait time.Duration) ([]string, error) {
+	lc := net.ListenConfig{Control: func(_, _ string, c syscall.RawConn) error {
+		var serr error
+		err := c.Control(func(fd uintptr) {
+			serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)
+			if serr == nil {
+				serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
+			}
+			if serr == nil {
+				serr = bindToDevice(int(fd), iface)
+			}
+		})
+		if err != nil {
+			return err
+		}
+		return serr
+	}}
+	pc, err := lc.ListenPacket(context.Background(), "udp4", "0.0.0.0:68")
+	if err != nil {
+		return nil, fmt.Errorf("cannot listen on udp/68: %v", err)
+	}
+	defer pc.Close()
+	if _, err := net.InterfaceByName(iface); err != nil {
+		return nil, err
+	}
+	// a locally administered MAC so the probe never takes a real host's lease
+	mac := make([]byte, 6)
+	_, _ = rand.Read(mac)
+	mac[0] = (mac[0] | 0x02) & 0xfe
+	xid := make([]byte, 4)
+	_, _ = rand.Read(xid)
+	p := make([]byte, 240)
+	p[0], p[1], p[2] = 1, 1, 6 // BOOTREQUEST, ethernet, hlen
+	copy(p[4:8], xid)
+	binary.BigEndian.PutUint16(p[10:12], 0x8000) // broadcast reply
+	copy(p[28:34], mac)
+	copy(p[236:240], []byte{99, 130, 83, 99})
+	p = append(p, 53, 1, 1, 55, 3, 1, 3, 6, 255) // DISCOVER; ask router, DNS
+	if _, err := pc.WriteTo(p, &net.UDPAddr{IP: net.IPv4bcast, Port: 67}); err != nil {
+		return nil, err
+	}
+	seen := map[string]bool{}
+	var out []string
+	_ = pc.SetReadDeadline(time.Now().Add(wait))
+	buf := make([]byte, 1500)
+	for {
+		n, _, err := pc.ReadFrom(buf)
+		if err != nil {
+			break
+		}
+		r := buf[:n]
+		if n < 240 || r[0] != 2 || string(r[4:8]) != string(xid) {
+			continue
+		}
+		srv := ""
+		for i := 240; i < n; {
+			o := r[i]
+			if o == 255 {
+				break
+			}
+			if o == 0 {
+				i++
+				continue
+			}
+			if i+1 >= n {
+				break
+			}
+			l := int(r[i+1])
+			if o == 54 && l == 4 && i+6 <= n {
+				srv = net.IP(r[i+2 : i+6]).String()
+			}
+			i += 2 + l
+		}
+		if srv == "" {
+			srv = net.IP(r[20:24]).String() // siaddr
+		}
+		if !seen[srv] {
+			seen[srv] = true
+			out = append(out, srv)
+		}
+	}
+	return out, nil
+}

cmd/rollcall/doctor_linux.go

@@ -0,0 +1,9 @@
+//go:build linux
+
+package main
+
+import "syscall"
+
+func bindToDevice(fd int, iface string) error {
+	return syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, iface)
+}

cmd/rollcall/doctor_other.go

@@ -0,0 +1,7 @@
+//go:build !linux
+
+package main
+
+import "errors"
+
+func bindToDevice(int, string) error { return errors.New("the DHCP probe needs Linux") }

cmd/rollcall/main.go

@@ -27,6 +27,7 @@ func usage() {
   rollcall                                     in a terminal: the full-screen console
   rollcall tui [-dump WxH -page P -machine ID -plain]
   rollcall serve [-config PATH]                region: API, database, web UI, embedded rack controller
+  rollcall doctor [-deep] [-no-dhcp-probe]     check this installation and say what to fix
   rollcall rack [-config /etc/rollcall/rack.json]   a rack controller next to the machines
   rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80]   enroll a rack controller (writes rack.json + certificate)
   rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR]       the installation's certificate authority (region host)
@@ -107,6 +108,11 @@ func main() {
 			fmt.Fprintln(os.Stderr, "error:", err)
 			os.Exit(1)
 		}
+	case "doctor":
+		if err := doctorCmd(args); err != nil {
+			fmt.Fprintln(os.Stderr, "error:", err)
+			os.Exit(1)
+		}
 	case "tui", "console":
 		if err := tuiDump(args); err != nil {
 			fmt.Fprintln(os.Stderr, "error:", err)

deploy/bootstrap.sh

@@ -0,0 +1,240 @@
+#!/bin/bash
+# Rollcall region on a fresh Debian 13 host, in one go. Run as root from a
+# checkout (deploy/) or an unpacked release (rollcall + deploy/ + agent/).
+#
+#   ./bootstrap.sh --iface eth1 [--name NAME] [--domain D] [--gateway IP | --nat UPLINK]
+#                  [--image-url URL | --build-image] [--os debian-13] [--no-dhcp] [--yes]
+#
+#   --iface       interface on the provisioning network; it must already have
+#                 its IPv4 address (this host's address there becomes the server URL)
+#   --name        rack controller name of this host (default: short hostname)
+#   --domain      DNS domain handed to machines (default: rollcall.internal)
+#   --gateway     default router for machines on the provisioning network
+#   --nat UPLINK  make this host the router: forward + masquerade out of UPLINK
+#                 (default when --gateway is not given: the interface of the default route)
+#   --image-url   directory with vmlinuz, initrd.img, filesystem.squashfs and SHA256SUMS
+#   --build-image build the commissioning image here with live-build (~20 min)
+#   --os NAME     import one OS image now (e.g. debian-13); the nightly sync does the rest
+#   --no-dhcp     model the network but do not serve DHCP (another DHCP server is in charge)
+#
+# Safe to run again: existing configuration, database and image are kept.
+set -euo pipefail
+HERE=$(cd "$(dirname "$0")" && pwd)
+ROOT=$(cd "$HERE/.." && pwd)
+IFACE="" NAME=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --iface) IFACE=$2; shift ;;
+    --name) NAME=$2; shift ;;
+    --domain) DOMAIN=$2; shift ;;
+    --gateway) GATEWAY=$2; shift ;;
+    --nat) NAT=$2; shift ;;
+    --image-url) IMAGE_URL=${2%/}; shift ;;
+    --build-image) BUILD=1 ;;
+    --os) OS=$2; shift ;;
+    --no-dhcp) DHCP=0 ;;
+    --yes|-y) YES=1 ;;
+    -h|--help) sed -n '2,24p' "$0"; exit 0 ;;
+    *) echo "unknown option $1 (--help)" >&2; exit 2 ;;
+  esac
+  shift
+done
+
+say() { printf '\n\033[1m== %s\033[0m\n' "$*"; }
+ok()  { printf '   \033[32mOK\033[0m %s\n' "$*"; }
+die() { printf '   \033[31mERROR\033[0m %s\n' "$*" >&2; exit 1; }
+[ "$(id -u)" = 0 ] || die "run as root"
+. /etc/os-release
+[ "$ID" = debian ] || echo "note: tested on Debian 13; this is $PRETTY_NAME" >&2
+
+# ------------------------------------------------------------------ 0. what and where
+say "0/9  provisioning network"
+if [ -z "$IFACE" ]; then
+  def=$(ip -4 route show default | awk '{print $5; exit}')
+  cand=$(ip -4 -o addr show scope global | awk -v d="$def" '$2 != d {print $2}' | sort -u)
+  [ "$(echo "$cand" | grep -c .)" = 1 ] || die "say which interface faces the machines: --iface IFACE (candidates: $(echo $cand))"
+  IFACE=$cand
+fi
+CIDR_ADDR=$(ip -4 -o addr show dev "$IFACE" scope global | awk '{print $4; exit}')
+[ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; configure it first (netplan / ifupdown)"
+read -r ADDR NET PREFIX FIRST <<<"$(python3 - "$CIDR_ADDR" <<'P'
+import ipaddress, sys
+i = ipaddress.ip_interface(sys.argv[1])
+print(i.ip, i.network, i.network.prefixlen, i.network.network_address)
+P
+)"
+[ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; create the subnet and ranges by hand"
+off() { python3 -c "import ipaddress,sys;print(ipaddress.ip_address('$FIRST')+int(sys.argv[1]))" "$1"; }
+UPLINK=$(ip -4 route show default | awk '{print $5; exit}')
+if [ -z "$GATEWAY" ]; then
+  GATEWAY=$ADDR
+  NAT=${NAT:-$UPLINK}
+fi
+[ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not the provisioning interface"
+SERVER=http://$ADDR
+cat <<EOF
+   interface   $IFACE ($CIDR_ADDR), subnet $NET
+   server      $SERVER   (rack controller "$NAME")
+   machines    gateway $GATEWAY${NAT:+ (NAT out of $NAT)}, DNS $ADDR, domain $DOMAIN, DHCP $([ $DHCP = 1 ] && echo on || echo off)
+   ranges      reserved $(off 1)-$(off 20), static $(off 50)-$(off 99), dynamic $(off 100)-$(off 200)
+EOF
+if [ $YES = 0 ] && [ -t 0 ]; then
+  read -r -p "   continue? [y/N] " a; case "$a" in y|Y|yes) ;; *) exit 1 ;; esac
+fi
+
+# ------------------------------------------------------------------ 1. packages
+say "1/9  packages"
+export DEBIAN_FRONTEND=noninteractive
+PKGS="postgresql dnsmasq pxelinux syslinux-common syslinux-efi qemu-utils ipmitool jq zstd curl ca-certificates python3 nftables"
+[ $BUILD = 1 ] && PKGS="$PKGS live-build debootstrap squashfs-tools xorriso"
+apt-get update -q >/dev/null
+apt-get install -yq $PKGS >/dev/null
+# until rollcall writes its configuration, dnsmasq must not grab port 53 on every address
+install -d /etc/dnsmasq.d
+[ -e /etc/dnsmasq.d/00-rollcall-base.conf ] || printf '# rollcall bootstrap: bind only where configured (systemd-resolved keeps 127.0.0.53)\nbind-dynamic\n' > /etc/dnsmasq.d/00-rollcall-base.conf
+systemctl restart dnsmasq || true
+ok "$(echo $PKGS | wc -w) packages"
+
+# ------------------------------------------------------------------ 2. database
+say "2/9  PostgreSQL"
+systemctl enable --now postgresql >/dev/null
+CONF=/etc/rollcall/config.json
+if [ -f "$CONF" ]; then
+  ok "config exists, database settings kept"
+else
+  PGPASS=$(head -c 24 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28)
+  if [ "$(su postgres -c "psql -tAc \"select 1 from pg_roles where rolname='rollcall'\"")" = 1 ]; then
+    su postgres -c "psql -q -c \"ALTER ROLE rollcall PASSWORD '$PGPASS'\""
+  else
+    su postgres -c "psql -q -c \"CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'\""
+  fi
+  [ "$(su postgres -c "psql -tAc \"select 1 from pg_database where datname='rollcall'\"")" = 1 ] || su postgres -c "psql -q -c 'CREATE DATABASE rollcall OWNER rollcall'"
+  ok "role and database rollcall"
+fi
+
+# ------------------------------------------------------------------ 3. TFTP
+say "3/9  TFTP boot files"
+T=/srv/tftp
+install -d $T $T/efi64 $T/pxelinux.cfg
+cp -u /usr/lib/PXELINUX/pxelinux.0 $T/
+for m in ldlinux.c32 libcom32.c32 libutil.c32 menu.c32; do cp -u /usr/lib/syslinux/modules/bios/$m $T/; done
+cp -u /usr/lib/SYSLINUX.EFI/efi64/syslinux.efi $T/efi64/
+for m in ldlinux.e64 libcom32.c32 libutil.c32 menu.c32; do cp -u /usr/lib/syslinux/modules/efi64/$m $T/efi64/; done
+ln -sfn ../pxelinux.cfg $T/efi64/pxelinux.cfg
+ok "BIOS (pxelinux.0) and UEFI (efi64/syslinux.efi)"
+
+# ------------------------------------------------------------------ 4. binary
+say "4/9  rollcall binary"
+BIN=""
+for b in "$ROOT/rollcall" "$HERE/rollcall"; do [ -x "$b" ] && BIN=$b && break; done
+if [ -z "$BIN" ] && [ -d "$ROOT/cmd/rollcall" ] && command -v go >/dev/null; then
+  (cd "$ROOT/cmd/rollcall" && CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=$(git describe --tags --always --dirty 2>/dev/null || echo dev)" -o "$HERE/rollcall" .)
+  BIN=$HERE/rollcall
+fi
+[ -n "$BIN" ] || die "no rollcall binary next to this script (unpack a release, or build: cd cmd/rollcall && go build)"
+[ "$BIN" = "$HERE/rollcall" ] || cp "$BIN" "$HERE/rollcall"
+ok "$("$HERE/rollcall" version)"
+
+# ------------------------------------------------------------------ 5. config
+say "5/9  configuration"
+getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall
+install -d -m 0750 -o root -g rollcall /etc/rollcall
+if [ ! -f "$CONF" ]; then
+  AGENT=$("$HERE/rollcall" keygen) ADMIN=$("$HERE/rollcall" keygen)
+  FQDN=$(hostname -f 2>/dev/null || hostname)
+  python3 - "$CONF" <<P
+import json, sys
+c = json.load(open("$HERE/config.example.json"))
+srv = "$SERVER"
+c.update({
+  "listen": ":80", "server_url": srv,
+  "database_url": "postgres://rollcall:$PGPASS@127.0.0.1:5432/rollcall?sslmode=disable",
+  "admin_token": "$ADMIN", "agent_token": "$AGENT",
+  "commission_append": f"boot=live components fetch={srv}/rollcall/filesystem.squashfs rollcall.server={srv} noautologin rollcall.token=$AGENT console=tty0 console=ttyS0,115200n8",
+  "rack_name": "$NAME", "tls_listen": ":443", "tls_names": sorted({"$FQDN", "$ADDR"}), "admin_http": "loopback",
+  "node_pools": {}, "bmc_hints": [],
+})
+c["cluster_vars"].update({"domain": "$DOMAIN", "cluster_name": "$NAME", "slurm_controller": "$NAME", "slurm_controller_ip": "$ADDR", "ssh_keys": []})
+c["ipam"]["provider"] = ""
+json.dump(c, open(sys.argv[1], "w"), indent=2)
+P
+  chown root:rollcall "$CONF"; chmod 0640 "$CONF"
+  ok "$CONF (tokens generated; admin_http loopback: the web UI is HTTPS-only)"
+else
+  ok "$CONF kept"
+fi
+
+# ------------------------------------------------------------------ 6. install
+say "6/9  services"
+install -d /srv/rollcall/image
+(cd "$HERE" && ./install.sh >/dev/null)
+for i in $(seq 1 30); do curl -sf -o /dev/null http://127.0.0.1/ca.pem && break; sleep 1; done
+curl -sf -o /dev/null http://127.0.0.1/ca.pem || die "rollcall did not come up: journalctl -u rollcall"
+ok "rollcall running, CA $(rollcall ca show | awk '/fingerprint/{print $2}')"
+
+# ------------------------------------------------------------------ 7. commissioning image
+say "7/9  commissioning image"
+IMG=/srv/rollcall/image
+if [ -s $IMG/vmlinuz ] && [ -s $IMG/initrd.img ] && [ -s $IMG/filesystem.squashfs ]; then
+  ok "present, kept"
+elif [ -n "$IMAGE_URL" ]; then
+  tmp=$(mktemp -d -p /srv/rollcall)
+  curl -fsSL -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS" || die "$IMAGE_URL/SHA256SUMS missing"
+  for f in vmlinuz initrd.img filesystem.squashfs; do curl -fL --progress-bar -o "$tmp/$f" "$IMAGE_URL/$f"; done
+  (cd "$tmp" && grep -E ' (\*)?(vmlinuz|initrd.img|filesystem.squashfs)$' SHA256SUMS | sha256sum -c --quiet -) || die "checksum mismatch"
+  mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rmdir "$tmp" 2>/dev/null || rm -rf "$tmp"
+  chmod 0644 $IMG/*
+  ok "downloaded and verified from $IMAGE_URL"
+elif [ $BUILD = 1 ]; then
+  [ -x "$ROOT/agent/build-image.sh" ] || die "agent/build-image.sh not found next to deploy/"
+  OUT=$IMG "$ROOT/agent/build-image.sh"
+  (cd $IMG && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS)
+  ok "built"
+else
+  echo "   no image yet: rerun with --image-url URL or --build-image (machines cannot commission without it)" >&2
+fi
+
+# ------------------------------------------------------------------ 8. network model
+say "8/9  network model"
+FAB=provisioning
+rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null
+if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then
+  rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null
+  rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null
+  rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null
+  rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null
+  ok "subnet $NET with ranges"
+else
+  ok "subnet $NET exists, kept"
+fi
+if [ $DHCP = 1 ]; then
+  rollcall vlan add "$FAB" 0 -dhcp -rack "$NAME" >/dev/null
+  ok "DHCP/PXE on $IFACE by rack controller $NAME"
+fi
+if [ -n "$NAT" ]; then
+  echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf
+  sysctl -q -p /etc/sysctl.d/60-rollcall-forward.conf
+  install -d /etc/nftables.d
+  printf 'table ip rollcall_nat {\n  chain postrouting {\n    type nat hook postrouting priority 100;\n    oifname "%s" ip saddr %s masquerade\n  }\n}\n' "$NAT" "$NET" > /etc/nftables.d/rollcall-nat.nft
+  grep -q 'include "/etc/nftables.d/\*.nft"' /etc/nftables.conf 2>/dev/null || echo 'include "/etc/nftables.d/*.nft"' >> /etc/nftables.conf
+  nft -c -f /etc/nftables.conf && systemctl enable --now nftables >/dev/null && systemctl reload nftables
+  ok "forwarding + masquerade $NET out of $NAT"
+fi
+
+# ------------------------------------------------------------------ 9. OS images
+say "9/9  OS images"
+if [ -n "$OS" ]; then
+  runuser -u rollcall -- rollcall os sync "$OS" || echo "   os sync $OS failed; the nightly timer retries" >&2
+else
+  echo "   none imported now; rollcall-os-sync.timer fetches the catalog nightly (or: runuser -u rollcall -- rollcall os sync debian-13)"
+fi
+
+cat <<EOF
+
+Done. Rollcall region "$NAME" on $SERVER
+  web UI      https://$ADDR  (CA: rollcall ca show; trust /ca.pem in your browser)
+  admin       a certificate for your laptop: rollcall admin-cert request NAME (needs the token once:
+              jq -r .admin_token $CONF)
+  machines    set them to PXE boot on the network of $IFACE; unknown ones commission themselves
+  check       rollcall doctor
+EOF

deploy/install.sh

@@ -16,7 +16,7 @@ chown rollcall:rollcall /etc/rollcall
 # rollcall writes per-MAC pxelinux configs
 chgrp rollcall /srv/tftp/pxelinux.cfg && chmod 2775 /srv/tftp/pxelinux.cfg
 install -m 0644 /usr/lib/syslinux/modules/bios/poweroff.c32 /srv/tftp/poweroff.c32
-sed "s/@AGENT_TOKEN@/$(jq -r .agent_token /etc/rollcall/config.json)/g" pxelinux-default > /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default
+sed -e "s/@AGENT_TOKEN@/$(jq -r .agent_token /etc/rollcall/config.json)/g" -e "s#@SERVER@#$(jq -r .server_url /etc/rollcall/config.json)#g" pxelinux-default > /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default
 [ -e /srv/tftp/rollcall ] || ln -s /srv/rollcall/image /srv/tftp/rollcall
 [ -d /srv/tftp/efi64 ] && ln -sfn ../rollcall /srv/tftp/efi64/rollcall
 # stateless images: <images_dir>/<name>/..., reachable by pxelinux as images/<name>/

deploy/pxelinux-default

@@ -3,20 +3,20 @@
 DEFAULT menu.c32
 PROMPT 0
 TIMEOUT 80
-MENU TITLE pxe-server (10.213.77.1) - unknown machine: commissioning in 8s
+MENU TITLE rollcall (@SERVER@) - unknown machine: commissioning in 8s
 
 LABEL commission
   MENU LABEL Commission this machine (inventory + BMC account)
   MENU DEFAULT
   KERNEL rollcall/vmlinuz
   INITRD rollcall/initrd.img
-  APPEND boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 noautologin rollcall.token=@AGENT_TOKEN@ console=tty0 console=ttyS0,115200n8
+  APPEND boot=live components fetch=@SERVER@/rollcall/filesystem.squashfs rollcall.server=@SERVER@ noautologin rollcall.token=@AGENT_TOKEN@ console=tty0 console=ttyS0,115200n8
 
 LABEL commission-debug
   MENU LABEL Commission, but stay up afterwards (ssh root@<ip>)
   KERNEL rollcall/vmlinuz
   INITRD rollcall/initrd.img
-  APPEND boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 noautologin rollcall.token=@AGENT_TOKEN@ rollcall.nopoweroff console=tty0 console=ttyS0,115200n8
+  APPEND boot=live components fetch=@SERVER@/rollcall/filesystem.squashfs rollcall.server=@SERVER@ noautologin rollcall.token=@AGENT_TOKEN@ rollcall.nopoweroff console=tty0 console=ttyS0,115200n8
 
 LABEL netbootxyz
   MENU LABEL netboot.xyz (needs internet via NAT)