From 01cc53120c775df21ac177ca4ee396e68ee33924 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 28 Oct 2026 18:08:30 +0100 Subject: [PATCH] From-scratch install: deploy/bootstrap.sh and rollcall doctor bootstrap.sh sets up a region on a fresh Debian 13 host (packages, PostgreSQL, TFTP tree, config, services, verified commissioning image, provisioning subnet with ranges/DHCP, optional NAT, first OS image). rollcall doctor checks an installation and says what to fix, including a DHCPDISCOVER probe for foreign DHCP servers. pxelinux-default no longer hard-codes the lab address. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 6 + cmd/rollcall/doctor.go | 484 +++++++++++++++++++++++++++++++++++ cmd/rollcall/doctor_linux.go | 9 + cmd/rollcall/doctor_other.go | 7 + cmd/rollcall/main.go | 6 + deploy/bootstrap.sh | 240 +++++++++++++++++ deploy/install.sh | 2 +- deploy/pxelinux-default | 6 +- 8 files changed, 756 insertions(+), 4 deletions(-) create mode 100644 cmd/rollcall/doctor.go create mode 100644 cmd/rollcall/doctor_linux.go create mode 100644 cmd/rollcall/doctor_other.go create mode 100755 deploy/bootstrap.sh diff --git a/CHANGELOG.md b/CHANGELOG.md index 81e2653..cde2538 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,6 +73,12 @@ publish a tag that has no section here. - Admin client certificates: `rollcall admin-cert request NAME` (key stays on the client, CSR only), `list`, `revoke`; the region accepts them as admin over HTTPS; CLI/console use them from `client.json`, the web UI works without a token when the browser presents one (`.p12` written). +- `deploy/bootstrap.sh`: a region on a fresh Debian 13 host in one command (packages, PostgreSQL, + TFTP tree, generated config, services, commissioning image downloaded and verified or built, + provisioning subnet with ranges and DHCP, optional NAT, first OS image). +- `rollcall doctor`: checks configuration, database, listeners, certificates, TFTP files, image + checksums, dnsmasq, DNS as machines see it, clock, foreign DHCP servers on the provisioning + network (DHCPDISCOVER probe), rack controllers, OS images and IPAM, with what to do. - `rollcall version`; CD-built binary carries `git describe`. - Release notes from `CHANGELOG.md`. diff --git a/cmd/rollcall/doctor.go b/cmd/rollcall/doctor.go new file mode 100644 index 0000000..19755ab --- /dev/null +++ b/cmd/rollcall/doctor.go @@ -0,0 +1,484 @@ +package main + +// rollcall doctor: checks an installation from the region host and says what +// to do about what is missing. Local checks read the configuration; checks of +// the running state ask the region's API with its own admin token. + +import ( + "bufio" + "context" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/binary" + "encoding/hex" + "flag" + "fmt" + "io" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" +) + +type doctor struct { + fails, warns int +} + +func (d *doctor) ok(name, detail string) { + fmt.Printf(" \033[32mOK \033[0m %-24s %s\n", name, detail) +} +func (d *doctor) warn(name, detail, fix string) { + d.warns++ + fmt.Printf(" \033[33mWARN\033[0m %-24s %s\n", name, detail) + if fix != "" { + fmt.Printf(" %-24s -> %s\n", "", fix) + } +} +func (d *doctor) fail(name, detail, fix string) { + d.fails++ + fmt.Printf(" \033[31mFAIL\033[0m %-24s %s\n", name, detail) + if fix != "" { + fmt.Printf(" %-24s -> %s\n", "", fix) + } +} + +func doctorCmd(args []string) error { + fs := flag.NewFlagSet("doctor", flag.ExitOnError) + cfgPath := fs.String("config", envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"), "config file") + deep := fs.Bool("deep", false, "also verify the commissioning image checksums") + noProbe := fs.Bool("no-dhcp-probe", false, "do not send a DHCPDISCOVER on the provisioning interface") + _ = fs.Parse(args) + d := &doctor{} + fmt.Println("rollcall doctor", version) + + cfg, err := loadConfig(*cfgPath) + if err != nil { + d.fail("config", err.Error(), "deploy/bootstrap.sh writes one; or see deploy/config.example.json") + return d.done() + } + d.ok("config", *cfgPath) + srvHost := "" + if u, err := urlHost(cfg.ServerURL); err == nil { + srvHost = u + } + iface := ifaceWithAddr(srvHost) + if iface == "" { + d.warn("server_url", cfg.ServerURL+" is not an address of this host", "machines reach the region there; set server_url to this host's address on the provisioning network") + } else { + d.ok("server_url", fmt.Sprintf("%s on %s", cfg.ServerURL, iface)) + } + + // database + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + st, err := openStore(ctx, cfg.DatabaseURL) + cancel() + if err != nil { + d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)") + } else { + d.ok("database", "connected, schema current") + st.pool.Close() + } + + // listeners + ports := listeningPorts() + for _, p := range []struct { + proto string + port int + what string + }{{"udp", 67, "DHCP (dnsmasq)"}, {"udp", 69, "TFTP (dnsmasq)"}, {"tcp", portOf(cfg.Listen, 80), "HTTP (rollcall)"}, {"tcp", portOf(cfg.TLSListen, 443), "HTTPS (rollcall)"}} { + if p.port == 0 { + continue + } + if ports[fmt.Sprintf("%s/%d", p.proto, p.port)] { + d.ok(fmt.Sprintf("%s/%d", p.proto, p.port), p.what) + } else if p.port == 67 { + d.warn("udp/67", "nothing listens for DHCP", "no VLAN with dhcp on this rack yet (rollcall vlan add FABRIC VID -dhcp -rack "+cfg.RackName+"), or dnsmasq is down") + } else { + d.fail(fmt.Sprintf("%s/%d", p.proto, p.port), "nothing listens for "+p.what, "systemctl status rollcall dnsmasq") + } + } + hc := &http.Client{Timeout: 5 * time.Second} + if resp, err := hc.Get("http://127.0.0.1" + localPort(cfg.Listen) + "/ca.pem"); err != nil || resp.StatusCode != 200 { + d.fail("http", fmt.Sprintf("GET /ca.pem: %v", errOrStatus(err, resp)), "journalctl -u rollcall") + } else { + resp.Body.Close() + d.ok("http", "region answers") + } + + // certificates + if cb, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil { + if c, err := parseCertPEM(cb); err == nil { + left := time.Until(c.NotAfter) + switch { + case left < 90*24*time.Hour: + d.warn("CA", "expires "+c.NotAfter.Format("2006-01-02"), "plan rollcall ca rotate") + default: + d.ok("CA", fmt.Sprintf("until %s, delegation %v", c.NotAfter.Format("2006-01-02"), (&CA{Cert: c}).canDelegate())) + } + if cfg.TLSListen != "" { + pool := x509.NewCertPool() + pool.AddCert(c) + conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", "127.0.0.1"+localPort(cfg.TLSListen), + &tls.Config{RootCAs: pool, ServerName: srvHost}) + if err != nil { + d.fail("https", err.Error(), "the region certificate should cover "+srvHost+" (tls_names); journalctl -u rollcall") + } else { + leaf := conn.ConnectionState().PeerCertificates[0] + conn.Close() + d.ok("https", fmt.Sprintf("certificate for %s until %s", srvHost, leaf.NotAfter.Format("2006-01-02"))) + } + } + } + } else { + d.fail("CA", "no CA yet", "it is created when rollcall serve starts") + } + + // TFTP + image + missing := []string{} + for _, f := range []string{"pxelinux.0", "ldlinux.c32", "menu.c32", "libcom32.c32", "libutil.c32", "efi64/syslinux.efi", "efi64/ldlinux.e64", "pxelinux.cfg/default"} { + if _, err := os.Stat(filepath.Join(cfg.TFTPRoot, f)); err != nil { + missing = append(missing, f) + } + } + if len(missing) > 0 { + d.fail("tftp", "missing in "+cfg.TFTPRoot+": "+strings.Join(missing, " "), "deploy/bootstrap.sh (step 3) copies them from pxelinux/syslinux-efi") + } else { + d.ok("tftp", cfg.TFTPRoot+" (BIOS + UEFI)") + } + img := []string{} + for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} { + if st, err := os.Stat(filepath.Join(cfg.ImageDir, f)); err != nil || st.Size() == 0 { + img = append(img, f) + } + } + switch { + case len(img) > 0: + d.fail("image", "commissioning image incomplete, missing "+strings.Join(img, " "), "deploy/bootstrap.sh --image-url URL or --build-image") + case *deep: + if bad := verifySums(cfg.ImageDir); bad == "no SHA256SUMS next to the image" { + d.warn("image", bad, "cd "+cfg.ImageDir+" && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS") + } else if bad != "" { + d.fail("image", bad, "download the image again") + } else { + d.ok("image", "checksums match SHA256SUMS") + } + default: + d.ok("image", cfg.ImageDir) + } + + // dnsmasq + if out, _ := exec.Command("systemctl", "is-active", "dnsmasq").Output(); strings.TrimSpace(string(out)) != "active" { + d.fail("dnsmasq", "not running", "journalctl -u dnsmasq -n 20") + } else if _, err := os.Stat(filepath.Join(cfg.DnsmasqDir, "rollcall.conf.rejected")); err == nil { + d.fail("dnsmasq", "the generated configuration was rejected; the last working one is in use", + "journalctl -u rollcall-dnsmasq; a hand-written DHCP/TFTP file in /etc/dnsmasq.d usually conflicts") + } else { + d.ok("dnsmasq", "running") + } + + // DNS: outside, and through dnsmasq as the machines see it + if _, err := net.DefaultResolver.LookupHost(context.Background(), "deb.debian.org"); err != nil { + d.fail("dns (host)", err.Error(), "this host cannot resolve; check /etc/resolv.conf / resolvectl") + } else { + d.ok("dns (host)", "resolves outside names") + } + if srvHost != "" && ports["udp/53"] { + r := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { + return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "udp", net.JoinHostPort(srvHost, "53")) + }} + if _, err := r.LookupHost(context.Background(), "deb.debian.org"); err != nil { + d.fail("dns (machines)", "dnsmasq on "+srvHost+" does not resolve outside names: "+err.Error(), + "the generated dnsmasq configuration needs an upstream (rollcall >= 0.2 writes one); check its # upstream section") + } else { + d.ok("dns (machines)", "dnsmasq on "+srvHost+" forwards outside names") + } + } + + // time + if out, err := exec.Command("timedatectl", "show", "-p", "NTPSynchronized", "--value").Output(); err == nil { + if strings.TrimSpace(string(out)) == "yes" { + d.ok("time", "synchronised") + } else { + d.warn("time", "clock not synchronised", "certificates and tokens depend on it: systemctl enable --now systemd-timesyncd") + } + } + + // other DHCP servers on the provisioning network + if iface != "" && !*noProbe { + servers, err := dhcpProbe(iface, 4*time.Second) + switch { + case err != nil: + d.warn("dhcp probe", err.Error(), "run as root; or --no-dhcp-probe") + default: + var foreign []string + for _, s := range servers { + if ifaceWithAddr(s) == "" { + foreign = append(foreign, s) + } + } + switch { + case len(foreign) > 0: + d.fail("dhcp probe", "another DHCP server answers on "+iface+": "+strings.Join(foreign, ", "), + "machines may PXE-boot from it; switch it off for this VLAN, or use --no-dhcp and let it hand out next-server "+srvHost) + case len(servers) == 0: + d.warn("dhcp probe", "no DHCP server answered on "+iface, "is a VLAN with dhcp assigned to this rack? (rollcall vlan add ... -dhcp)") + default: + d.ok("dhcp probe", "only this host answers on "+iface) + } + } + } + + // running state through the API + if c, err := newClient(); err == nil { + var racks []RackController + if err := c.do("GET", "/api/v1/controllers", nil, &racks); err == nil { + regionVer := version + for _, r := range racks { + if r.Kind == "embedded" && r.Version != "" { + regionVer = r.Version // what the region runs (this binary may differ) + } + } + for _, r := range racks { + switch { + case !r.Online: + d.fail("rack "+r.Name, "offline since "+timeStr(r.LastSeen), "on that host: systemctl status rollcall-rack") + case r.Version != regionVer && r.Version != "": + d.warn("rack "+r.Name, "runs "+r.Version+", region "+regionVer, "it updates itself within minutes (rollcall-rack-update.path); see /var/lib/rollcall/update/last-result there") + default: + d.ok("rack "+r.Name, fmt.Sprintf("online, %d machines", r.Machines)) + } + } + } + var oss []map[string]any + if err := c.do("GET", "/api/v1/os", nil, &oss); err == nil { + if len(oss) == 0 { + d.warn("os images", "none", "runuser -u rollcall -- rollcall os sync debian-13") + } else { + d.ok("os images", fmt.Sprintf("%d in the catalog", len(oss))) + } + } + var ipam map[string]any + if err := c.do("GET", "/api/v1/ipam", nil, &ipam); err == nil && ipam["provider"] != "" && ipam["provider"] != nil { + last, _ := ipam["last_sync"].(map[string]any) + if e, _ := last["error"].(string); e != "" { + d.fail("ipam", e, "check ipam.url/token_file/ca_file; rollcall ipam sync") + } else { + d.ok("ipam", fmt.Sprintf("%v in sync", ipam["provider"])) + } + } + } else { + d.warn("api", err.Error(), "") + } + return d.done() +} + +func (d *doctor) done() error { + fmt.Printf("\n%d failed, %d warnings\n", d.fails, d.warns) + if d.fails > 0 { + os.Exit(1) + } + return nil +} + +func errOrStatus(err error, resp *http.Response) string { + if err != nil { + return err.Error() + } + return resp.Status +} + +func urlHost(u string) (string, error) { + u = strings.TrimPrefix(strings.TrimPrefix(u, "http://"), "https://") + u = strings.SplitN(u, "/", 2)[0] + if h, _, err := net.SplitHostPort(u); err == nil { + return strings.Trim(h, "[]"), nil + } + return strings.Trim(u, "[]"), nil +} + +// ifaceWithAddr: the interface holding ip on this host, "" if none. +func ifaceWithAddr(ip string) string { + want := net.ParseIP(ip) + if want == nil { + return "" + } + ifs, _ := net.Interfaces() + for _, i := range ifs { + addrs, _ := i.Addrs() + for _, a := range addrs { + if n, ok := a.(*net.IPNet); ok && n.IP.Equal(want) { + return i.Name + } + } + } + return "" +} + +func portOf(listen string, def int) int { + if listen == "" { + return 0 + } + _, p, err := net.SplitHostPort(listen) + if err != nil { + return def + } + n, _ := strconv.Atoi(p) + return n +} + +func localPort(listen string) string { + if p := portOf(listen, 0); p != 0 && p != 80 && p != 443 { + return ":" + strconv.Itoa(p) + } + if p := portOf(listen, 0); p == 443 { + return ":443" + } + return "" +} + +// listeningPorts reads /proc/net: "tcp/80", "udp/67" ... +func listeningPorts() map[string]bool { + out := map[string]bool{} + for _, f := range []string{"tcp", "tcp6", "udp", "udp6"} { + fh, err := os.Open("/proc/net/" + f) + if err != nil { + continue + } + sc := bufio.NewScanner(fh) + sc.Scan() + for sc.Scan() { + fl := strings.Fields(sc.Text()) + if len(fl) < 4 { + continue + } + if strings.HasPrefix(f, "tcp") && fl[3] != "0A" { // LISTEN + continue + } + i := strings.LastIndex(fl[1], ":") + p, err := strconv.ParseUint(fl[1][i+1:], 16, 16) + if err == nil { + out[strings.TrimSuffix(f, "6")+"/"+strconv.Itoa(int(p))] = true + } + } + fh.Close() + } + return out +} + +func verifySums(dir string) string { + b, err := os.ReadFile(filepath.Join(dir, "SHA256SUMS")) + if err != nil { + return "no SHA256SUMS next to the image" + } + for _, line := range strings.Split(string(b), "\n") { + f := strings.Fields(line) + if len(f) != 2 { + continue + } + name := strings.TrimPrefix(f[1], "*") + fh, err := os.Open(filepath.Join(dir, name)) + if err != nil { + return name + ": " + err.Error() + } + h := sha256.New() + _, _ = io.Copy(h, fh) + fh.Close() + if hex.EncodeToString(h.Sum(nil)) != f[0] { + return name + ": checksum mismatch" + } + } + return "" +} + +// dhcpProbe broadcasts a DHCPDISCOVER on iface and returns the server +// identifiers that offer an address. Needs root (port 68, SO_BINDTODEVICE). +func dhcpProbe(iface string, wait time.Duration) ([]string, error) { + lc := net.ListenConfig{Control: func(_, _ string, c syscall.RawConn) error { + var serr error + err := c.Control(func(fd uintptr) { + serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1) + if serr == nil { + serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1) + } + if serr == nil { + serr = bindToDevice(int(fd), iface) + } + }) + if err != nil { + return err + } + return serr + }} + pc, err := lc.ListenPacket(context.Background(), "udp4", "0.0.0.0:68") + if err != nil { + return nil, fmt.Errorf("cannot listen on udp/68: %v", err) + } + defer pc.Close() + if _, err := net.InterfaceByName(iface); err != nil { + return nil, err + } + // a locally administered MAC so the probe never takes a real host's lease + mac := make([]byte, 6) + _, _ = rand.Read(mac) + mac[0] = (mac[0] | 0x02) & 0xfe + xid := make([]byte, 4) + _, _ = rand.Read(xid) + p := make([]byte, 240) + p[0], p[1], p[2] = 1, 1, 6 // BOOTREQUEST, ethernet, hlen + copy(p[4:8], xid) + binary.BigEndian.PutUint16(p[10:12], 0x8000) // broadcast reply + copy(p[28:34], mac) + copy(p[236:240], []byte{99, 130, 83, 99}) + p = append(p, 53, 1, 1, 55, 3, 1, 3, 6, 255) // DISCOVER; ask router, DNS + if _, err := pc.WriteTo(p, &net.UDPAddr{IP: net.IPv4bcast, Port: 67}); err != nil { + return nil, err + } + seen := map[string]bool{} + var out []string + _ = pc.SetReadDeadline(time.Now().Add(wait)) + buf := make([]byte, 1500) + for { + n, _, err := pc.ReadFrom(buf) + if err != nil { + break + } + r := buf[:n] + if n < 240 || r[0] != 2 || string(r[4:8]) != string(xid) { + continue + } + srv := "" + for i := 240; i < n; { + o := r[i] + if o == 255 { + break + } + if o == 0 { + i++ + continue + } + if i+1 >= n { + break + } + l := int(r[i+1]) + if o == 54 && l == 4 && i+6 <= n { + srv = net.IP(r[i+2 : i+6]).String() + } + i += 2 + l + } + if srv == "" { + srv = net.IP(r[20:24]).String() // siaddr + } + if !seen[srv] { + seen[srv] = true + out = append(out, srv) + } + } + return out, nil +} diff --git a/cmd/rollcall/doctor_linux.go b/cmd/rollcall/doctor_linux.go new file mode 100644 index 0000000..7e7b021 --- /dev/null +++ b/cmd/rollcall/doctor_linux.go @@ -0,0 +1,9 @@ +//go:build linux + +package main + +import "syscall" + +func bindToDevice(fd int, iface string) error { + return syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, iface) +} diff --git a/cmd/rollcall/doctor_other.go b/cmd/rollcall/doctor_other.go new file mode 100644 index 0000000..4b0ec69 --- /dev/null +++ b/cmd/rollcall/doctor_other.go @@ -0,0 +1,7 @@ +//go:build !linux + +package main + +import "errors" + +func bindToDevice(int, string) error { return errors.New("the DHCP probe needs Linux") } diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index 9a16454..1eacf77 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -27,6 +27,7 @@ func usage() { rollcall in a terminal: the full-screen console rollcall tui [-dump WxH -page P -machine ID -plain] rollcall serve [-config PATH] region: API, database, web UI, embedded rack controller + rollcall doctor [-deep] [-no-dhcp-probe] check this installation and say what to fix rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate) rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host) @@ -107,6 +108,11 @@ func main() { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) } + case "doctor": + if err := doctorCmd(args); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } case "tui", "console": if err := tuiDump(args); err != nil { fmt.Fprintln(os.Stderr, "error:", err) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh new file mode 100755 index 0000000..2386b53 --- /dev/null +++ b/deploy/bootstrap.sh @@ -0,0 +1,240 @@ +#!/bin/bash +# Rollcall region on a fresh Debian 13 host, in one go. Run as root from a +# checkout (deploy/) or an unpacked release (rollcall + deploy/ + agent/). +# +# ./bootstrap.sh --iface eth1 [--name NAME] [--domain D] [--gateway IP | --nat UPLINK] +# [--image-url URL | --build-image] [--os debian-13] [--no-dhcp] [--yes] +# +# --iface interface on the provisioning network; it must already have +# its IPv4 address (this host's address there becomes the server URL) +# --name rack controller name of this host (default: short hostname) +# --domain DNS domain handed to machines (default: rollcall.internal) +# --gateway default router for machines on the provisioning network +# --nat UPLINK make this host the router: forward + masquerade out of UPLINK +# (default when --gateway is not given: the interface of the default route) +# --image-url directory with vmlinuz, initrd.img, filesystem.squashfs and SHA256SUMS +# --build-image build the commissioning image here with live-build (~20 min) +# --os NAME import one OS image now (e.g. debian-13); the nightly sync does the rest +# --no-dhcp model the network but do not serve DHCP (another DHCP server is in charge) +# +# Safe to run again: existing configuration, database and image are kept. +set -euo pipefail +HERE=$(cd "$(dirname "$0")" && pwd) +ROOT=$(cd "$HERE/.." && pwd) +IFACE="" NAME=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0 +while [ $# -gt 0 ]; do + case "$1" in + --iface) IFACE=$2; shift ;; + --name) NAME=$2; shift ;; + --domain) DOMAIN=$2; shift ;; + --gateway) GATEWAY=$2; shift ;; + --nat) NAT=$2; shift ;; + --image-url) IMAGE_URL=${2%/}; shift ;; + --build-image) BUILD=1 ;; + --os) OS=$2; shift ;; + --no-dhcp) DHCP=0 ;; + --yes|-y) YES=1 ;; + -h|--help) sed -n '2,24p' "$0"; exit 0 ;; + *) echo "unknown option $1 (--help)" >&2; exit 2 ;; + esac + shift +done + +say() { printf '\n\033[1m== %s\033[0m\n' "$*"; } +ok() { printf ' \033[32mOK\033[0m %s\n' "$*"; } +die() { printf ' \033[31mERROR\033[0m %s\n' "$*" >&2; exit 1; } +[ "$(id -u)" = 0 ] || die "run as root" +. /etc/os-release +[ "$ID" = debian ] || echo "note: tested on Debian 13; this is $PRETTY_NAME" >&2 + +# ------------------------------------------------------------------ 0. what and where +say "0/9 provisioning network" +if [ -z "$IFACE" ]; then + def=$(ip -4 route show default | awk '{print $5; exit}') + cand=$(ip -4 -o addr show scope global | awk -v d="$def" '$2 != d {print $2}' | sort -u) + [ "$(echo "$cand" | grep -c .)" = 1 ] || die "say which interface faces the machines: --iface IFACE (candidates: $(echo $cand))" + IFACE=$cand +fi +CIDR_ADDR=$(ip -4 -o addr show dev "$IFACE" scope global | awk '{print $4; exit}') +[ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; configure it first (netplan / ifupdown)" +read -r ADDR NET PREFIX FIRST <<<"$(python3 - "$CIDR_ADDR" <<'P' +import ipaddress, sys +i = ipaddress.ip_interface(sys.argv[1]) +print(i.ip, i.network, i.network.prefixlen, i.network.network_address) +P +)" +[ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; create the subnet and ranges by hand" +off() { python3 -c "import ipaddress,sys;print(ipaddress.ip_address('$FIRST')+int(sys.argv[1]))" "$1"; } +UPLINK=$(ip -4 route show default | awk '{print $5; exit}') +if [ -z "$GATEWAY" ]; then + GATEWAY=$ADDR + NAT=${NAT:-$UPLINK} +fi +[ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not the provisioning interface" +SERVER=http://$ADDR +cat </dev/null +apt-get install -yq $PKGS >/dev/null +# until rollcall writes its configuration, dnsmasq must not grab port 53 on every address +install -d /etc/dnsmasq.d +[ -e /etc/dnsmasq.d/00-rollcall-base.conf ] || printf '# rollcall bootstrap: bind only where configured (systemd-resolved keeps 127.0.0.53)\nbind-dynamic\n' > /etc/dnsmasq.d/00-rollcall-base.conf +systemctl restart dnsmasq || true +ok "$(echo $PKGS | wc -w) packages" + +# ------------------------------------------------------------------ 2. database +say "2/9 PostgreSQL" +systemctl enable --now postgresql >/dev/null +CONF=/etc/rollcall/config.json +if [ -f "$CONF" ]; then + ok "config exists, database settings kept" +else + PGPASS=$(head -c 24 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28) + if [ "$(su postgres -c "psql -tAc \"select 1 from pg_roles where rolname='rollcall'\"")" = 1 ]; then + su postgres -c "psql -q -c \"ALTER ROLE rollcall PASSWORD '$PGPASS'\"" + else + su postgres -c "psql -q -c \"CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'\"" + fi + [ "$(su postgres -c "psql -tAc \"select 1 from pg_database where datname='rollcall'\"")" = 1 ] || su postgres -c "psql -q -c 'CREATE DATABASE rollcall OWNER rollcall'" + ok "role and database rollcall" +fi + +# ------------------------------------------------------------------ 3. TFTP +say "3/9 TFTP boot files" +T=/srv/tftp +install -d $T $T/efi64 $T/pxelinux.cfg +cp -u /usr/lib/PXELINUX/pxelinux.0 $T/ +for m in ldlinux.c32 libcom32.c32 libutil.c32 menu.c32; do cp -u /usr/lib/syslinux/modules/bios/$m $T/; done +cp -u /usr/lib/SYSLINUX.EFI/efi64/syslinux.efi $T/efi64/ +for m in ldlinux.e64 libcom32.c32 libutil.c32 menu.c32; do cp -u /usr/lib/syslinux/modules/efi64/$m $T/efi64/; done +ln -sfn ../pxelinux.cfg $T/efi64/pxelinux.cfg +ok "BIOS (pxelinux.0) and UEFI (efi64/syslinux.efi)" + +# ------------------------------------------------------------------ 4. binary +say "4/9 rollcall binary" +BIN="" +for b in "$ROOT/rollcall" "$HERE/rollcall"; do [ -x "$b" ] && BIN=$b && break; done +if [ -z "$BIN" ] && [ -d "$ROOT/cmd/rollcall" ] && command -v go >/dev/null; then + (cd "$ROOT/cmd/rollcall" && CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=$(git describe --tags --always --dirty 2>/dev/null || echo dev)" -o "$HERE/rollcall" .) + BIN=$HERE/rollcall +fi +[ -n "$BIN" ] || die "no rollcall binary next to this script (unpack a release, or build: cd cmd/rollcall && go build)" +[ "$BIN" = "$HERE/rollcall" ] || cp "$BIN" "$HERE/rollcall" +ok "$("$HERE/rollcall" version)" + +# ------------------------------------------------------------------ 5. config +say "5/9 configuration" +getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall +install -d -m 0750 -o root -g rollcall /etc/rollcall +if [ ! -f "$CONF" ]; then + AGENT=$("$HERE/rollcall" keygen) ADMIN=$("$HERE/rollcall" keygen) + FQDN=$(hostname -f 2>/dev/null || hostname) + python3 - "$CONF" <

/dev/null) +for i in $(seq 1 30); do curl -sf -o /dev/null http://127.0.0.1/ca.pem && break; sleep 1; done +curl -sf -o /dev/null http://127.0.0.1/ca.pem || die "rollcall did not come up: journalctl -u rollcall" +ok "rollcall running, CA $(rollcall ca show | awk '/fingerprint/{print $2}')" + +# ------------------------------------------------------------------ 7. commissioning image +say "7/9 commissioning image" +IMG=/srv/rollcall/image +if [ -s $IMG/vmlinuz ] && [ -s $IMG/initrd.img ] && [ -s $IMG/filesystem.squashfs ]; then + ok "present, kept" +elif [ -n "$IMAGE_URL" ]; then + tmp=$(mktemp -d -p /srv/rollcall) + curl -fsSL -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS" || die "$IMAGE_URL/SHA256SUMS missing" + for f in vmlinuz initrd.img filesystem.squashfs; do curl -fL --progress-bar -o "$tmp/$f" "$IMAGE_URL/$f"; done + (cd "$tmp" && grep -E ' (\*)?(vmlinuz|initrd.img|filesystem.squashfs)$' SHA256SUMS | sha256sum -c --quiet -) || die "checksum mismatch" + mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rmdir "$tmp" 2>/dev/null || rm -rf "$tmp" + chmod 0644 $IMG/* + ok "downloaded and verified from $IMAGE_URL" +elif [ $BUILD = 1 ]; then + [ -x "$ROOT/agent/build-image.sh" ] || die "agent/build-image.sh not found next to deploy/" + OUT=$IMG "$ROOT/agent/build-image.sh" + (cd $IMG && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS) + ok "built" +else + echo " no image yet: rerun with --image-url URL or --build-image (machines cannot commission without it)" >&2 +fi + +# ------------------------------------------------------------------ 8. network model +say "8/9 network model" +FAB=provisioning +rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null +if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then + rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null + rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null + rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null + rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null + ok "subnet $NET with ranges" +else + ok "subnet $NET exists, kept" +fi +if [ $DHCP = 1 ]; then + rollcall vlan add "$FAB" 0 -dhcp -rack "$NAME" >/dev/null + ok "DHCP/PXE on $IFACE by rack controller $NAME" +fi +if [ -n "$NAT" ]; then + echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf + sysctl -q -p /etc/sysctl.d/60-rollcall-forward.conf + install -d /etc/nftables.d + printf 'table ip rollcall_nat {\n chain postrouting {\n type nat hook postrouting priority 100;\n oifname "%s" ip saddr %s masquerade\n }\n}\n' "$NAT" "$NET" > /etc/nftables.d/rollcall-nat.nft + grep -q 'include "/etc/nftables.d/\*.nft"' /etc/nftables.conf 2>/dev/null || echo 'include "/etc/nftables.d/*.nft"' >> /etc/nftables.conf + nft -c -f /etc/nftables.conf && systemctl enable --now nftables >/dev/null && systemctl reload nftables + ok "forwarding + masquerade $NET out of $NAT" +fi + +# ------------------------------------------------------------------ 9. OS images +say "9/9 OS images" +if [ -n "$OS" ]; then + runuser -u rollcall -- rollcall os sync "$OS" || echo " os sync $OS failed; the nightly timer retries" >&2 +else + echo " none imported now; rollcall-os-sync.timer fetches the catalog nightly (or: runuser -u rollcall -- rollcall os sync debian-13)" +fi + +cat < /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default +sed -e "s/@AGENT_TOKEN@/$(jq -r .agent_token /etc/rollcall/config.json)/g" -e "s#@SERVER@#$(jq -r .server_url /etc/rollcall/config.json)#g" pxelinux-default > /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default [ -e /srv/tftp/rollcall ] || ln -s /srv/rollcall/image /srv/tftp/rollcall [ -d /srv/tftp/efi64 ] && ln -sfn ../rollcall /srv/tftp/efi64/rollcall # stateless images: //..., reachable by pxelinux as images// diff --git a/deploy/pxelinux-default b/deploy/pxelinux-default index e82d935..4463cd0 100644 --- a/deploy/pxelinux-default +++ b/deploy/pxelinux-default @@ -3,20 +3,20 @@ DEFAULT menu.c32 PROMPT 0 TIMEOUT 80 -MENU TITLE pxe-server (10.213.77.1) - unknown machine: commissioning in 8s +MENU TITLE rollcall (@SERVER@) - unknown machine: commissioning in 8s LABEL commission MENU LABEL Commission this machine (inventory + BMC account) MENU DEFAULT KERNEL rollcall/vmlinuz INITRD rollcall/initrd.img - APPEND boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 noautologin rollcall.token=@AGENT_TOKEN@ console=tty0 console=ttyS0,115200n8 + APPEND boot=live components fetch=@SERVER@/rollcall/filesystem.squashfs rollcall.server=@SERVER@ noautologin rollcall.token=@AGENT_TOKEN@ console=tty0 console=ttyS0,115200n8 LABEL commission-debug MENU LABEL Commission, but stay up afterwards (ssh root@) KERNEL rollcall/vmlinuz INITRD rollcall/initrd.img - APPEND boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 noautologin rollcall.token=@AGENT_TOKEN@ rollcall.nopoweroff console=tty0 console=ttyS0,115200n8 + APPEND boot=live components fetch=@SERVER@/rollcall/filesystem.squashfs rollcall.server=@SERVER@ noautologin rollcall.token=@AGENT_TOKEN@ rollcall.nopoweroff console=tty0 console=ttyS0,115200n8 LABEL netbootxyz MENU LABEL netboot.xyz (needs internet via NAT)