From-scratch install: deploy/bootstrap.sh and rollcall doctor
CI / agent (push) Successful in 3s
CI / site (push) Successful in 2s
CI / go (push) Successful in 36s

bootstrap.sh sets up a region on a fresh Debian 13 host (packages,
PostgreSQL, TFTP tree, config, services, verified commissioning image,
provisioning subnet with ranges/DHCP, optional NAT, first OS image).
rollcall doctor checks an installation and says what to fix, including a
DHCPDISCOVER probe for foreign DHCP servers. pxelinux-default no longer
hard-codes the lab address.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-28 18:08:30 +01:00
1 parent ada6566534
commit 01cc53120c
8 files changed
+756 -4

No files matched your search

+484
View File
@@ -0,0 +1,484 @@
package main
// rollcall doctor: checks an installation from the region host and says what
// to do about what is missing. Local checks read the configuration; checks of
// the running state ask the region's API with its own admin token.
import (
"bufio"
"context"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/binary"
"encoding/hex"
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
)
type doctor struct {
fails, warns int
}
func (d *doctor) ok(name, detail string) {
fmt.Printf(" \033[32mOK \033[0m %-24s %s\n", name, detail)
}
func (d *doctor) warn(name, detail, fix string) {
d.warns++
fmt.Printf(" \033[33mWARN\033[0m %-24s %s\n", name, detail)
if fix != "" {
fmt.Printf(" %-24s -> %s\n", "", fix)
}
}
func (d *doctor) fail(name, detail, fix string) {
d.fails++
fmt.Printf(" \033[31mFAIL\033[0m %-24s %s\n", name, detail)
if fix != "" {
fmt.Printf(" %-24s -> %s\n", "", fix)
}
}
func doctorCmd(args []string) error {
fs := flag.NewFlagSet("doctor", flag.ExitOnError)
cfgPath := fs.String("config", envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"), "config file")
deep := fs.Bool("deep", false, "also verify the commissioning image checksums")
noProbe := fs.Bool("no-dhcp-probe", false, "do not send a DHCPDISCOVER on the provisioning interface")
_ = fs.Parse(args)
d := &doctor{}
fmt.Println("rollcall doctor", version)
cfg, err := loadConfig(*cfgPath)
if err != nil {
d.fail("config", err.Error(), "deploy/bootstrap.sh writes one; or see deploy/config.example.json")
return d.done()
}
d.ok("config", *cfgPath)
srvHost := ""
if u, err := urlHost(cfg.ServerURL); err == nil {
srvHost = u
}
iface := ifaceWithAddr(srvHost)
if iface == "" {
d.warn("server_url", cfg.ServerURL+" is not an address of this host", "machines reach the region there; set server_url to this host's address on the provisioning network")
} else {
d.ok("server_url", fmt.Sprintf("%s on %s", cfg.ServerURL, iface))
}
// database
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
st, err := openStore(ctx, cfg.DatabaseURL)
cancel()
if err != nil {
d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)")
} else {
d.ok("database", "connected, schema current")
st.pool.Close()
}
// listeners
ports := listeningPorts()
for _, p := range []struct {
proto string
port int
what string
}{{"udp", 67, "DHCP (dnsmasq)"}, {"udp", 69, "TFTP (dnsmasq)"}, {"tcp", portOf(cfg.Listen, 80), "HTTP (rollcall)"}, {"tcp", portOf(cfg.TLSListen, 443), "HTTPS (rollcall)"}} {
if p.port == 0 {
continue
}
if ports[fmt.Sprintf("%s/%d", p.proto, p.port)] {
d.ok(fmt.Sprintf("%s/%d", p.proto, p.port), p.what)
} else if p.port == 67 {
d.warn("udp/67", "nothing listens for DHCP", "no VLAN with dhcp on this rack yet (rollcall vlan add FABRIC VID -dhcp -rack "+cfg.RackName+"), or dnsmasq is down")
} else {
d.fail(fmt.Sprintf("%s/%d", p.proto, p.port), "nothing listens for "+p.what, "systemctl status rollcall dnsmasq")
}
}
hc := &http.Client{Timeout: 5 * time.Second}
if resp, err := hc.Get("http://127.0.0.1" + localPort(cfg.Listen) + "/ca.pem"); err != nil || resp.StatusCode != 200 {
d.fail("http", fmt.Sprintf("GET /ca.pem: %v", errOrStatus(err, resp)), "journalctl -u rollcall")
} else {
resp.Body.Close()
d.ok("http", "region answers")
}
// certificates
if cb, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil {
if c, err := parseCertPEM(cb); err == nil {
left := time.Until(c.NotAfter)
switch {
case left < 90*24*time.Hour:
d.warn("CA", "expires "+c.NotAfter.Format("2006-01-02"), "plan rollcall ca rotate")
default:
d.ok("CA", fmt.Sprintf("until %s, delegation %v", c.NotAfter.Format("2006-01-02"), (&CA{Cert: c}).canDelegate()))
}
if cfg.TLSListen != "" {
pool := x509.NewCertPool()
pool.AddCert(c)
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", "127.0.0.1"+localPort(cfg.TLSListen),
&tls.Config{RootCAs: pool, ServerName: srvHost})
if err != nil {
d.fail("https", err.Error(), "the region certificate should cover "+srvHost+" (tls_names); journalctl -u rollcall")
} else {
leaf := conn.ConnectionState().PeerCertificates[0]
conn.Close()
d.ok("https", fmt.Sprintf("certificate for %s until %s", srvHost, leaf.NotAfter.Format("2006-01-02")))
}
}
}
} else {
d.fail("CA", "no CA yet", "it is created when rollcall serve starts")
}
// TFTP + image
missing := []string{}
for _, f := range []string{"pxelinux.0", "ldlinux.c32", "menu.c32", "libcom32.c32", "libutil.c32", "efi64/syslinux.efi", "efi64/ldlinux.e64", "pxelinux.cfg/default"} {
if _, err := os.Stat(filepath.Join(cfg.TFTPRoot, f)); err != nil {
missing = append(missing, f)
}
}
if len(missing) > 0 {
d.fail("tftp", "missing in "+cfg.TFTPRoot+": "+strings.Join(missing, " "), "deploy/bootstrap.sh (step 3) copies them from pxelinux/syslinux-efi")
} else {
d.ok("tftp", cfg.TFTPRoot+" (BIOS + UEFI)")
}
img := []string{}
for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} {
if st, err := os.Stat(filepath.Join(cfg.ImageDir, f)); err != nil || st.Size() == 0 {
img = append(img, f)
}
}
switch {
case len(img) > 0:
d.fail("image", "commissioning image incomplete, missing "+strings.Join(img, " "), "deploy/bootstrap.sh --image-url URL or --build-image")
case *deep:
if bad := verifySums(cfg.ImageDir); bad == "no SHA256SUMS next to the image" {
d.warn("image", bad, "cd "+cfg.ImageDir+" && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS")
} else if bad != "" {
d.fail("image", bad, "download the image again")
} else {
d.ok("image", "checksums match SHA256SUMS")
}
default:
d.ok("image", cfg.ImageDir)
}
// dnsmasq
if out, _ := exec.Command("systemctl", "is-active", "dnsmasq").Output(); strings.TrimSpace(string(out)) != "active" {
d.fail("dnsmasq", "not running", "journalctl -u dnsmasq -n 20")
} else if _, err := os.Stat(filepath.Join(cfg.DnsmasqDir, "rollcall.conf.rejected")); err == nil {
d.fail("dnsmasq", "the generated configuration was rejected; the last working one is in use",
"journalctl -u rollcall-dnsmasq; a hand-written DHCP/TFTP file in /etc/dnsmasq.d usually conflicts")
} else {
d.ok("dnsmasq", "running")
}
// DNS: outside, and through dnsmasq as the machines see it
if _, err := net.DefaultResolver.LookupHost(context.Background(), "deb.debian.org"); err != nil {
d.fail("dns (host)", err.Error(), "this host cannot resolve; check /etc/resolv.conf / resolvectl")
} else {
d.ok("dns (host)", "resolves outside names")
}
if srvHost != "" && ports["udp/53"] {
r := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "udp", net.JoinHostPort(srvHost, "53"))
}}
if _, err := r.LookupHost(context.Background(), "deb.debian.org"); err != nil {
d.fail("dns (machines)", "dnsmasq on "+srvHost+" does not resolve outside names: "+err.Error(),
"the generated dnsmasq configuration needs an upstream (rollcall >= 0.2 writes one); check its # upstream section")
} else {
d.ok("dns (machines)", "dnsmasq on "+srvHost+" forwards outside names")
}
}
// time
if out, err := exec.Command("timedatectl", "show", "-p", "NTPSynchronized", "--value").Output(); err == nil {
if strings.TrimSpace(string(out)) == "yes" {
d.ok("time", "synchronised")
} else {
d.warn("time", "clock not synchronised", "certificates and tokens depend on it: systemctl enable --now systemd-timesyncd")
}
}
// other DHCP servers on the provisioning network
if iface != "" && !*noProbe {
servers, err := dhcpProbe(iface, 4*time.Second)
switch {
case err != nil:
d.warn("dhcp probe", err.Error(), "run as root; or --no-dhcp-probe")
default:
var foreign []string
for _, s := range servers {
if ifaceWithAddr(s) == "" {
foreign = append(foreign, s)
}
}
switch {
case len(foreign) > 0:
d.fail("dhcp probe", "another DHCP server answers on "+iface+": "+strings.Join(foreign, ", "),
"machines may PXE-boot from it; switch it off for this VLAN, or use --no-dhcp and let it hand out next-server "+srvHost)
case len(servers) == 0:
d.warn("dhcp probe", "no DHCP server answered on "+iface, "is a VLAN with dhcp assigned to this rack? (rollcall vlan add ... -dhcp)")
default:
d.ok("dhcp probe", "only this host answers on "+iface)
}
}
}
// running state through the API
if c, err := newClient(); err == nil {
var racks []RackController
if err := c.do("GET", "/api/v1/controllers", nil, &racks); err == nil {
regionVer := version
for _, r := range racks {
if r.Kind == "embedded" && r.Version != "" {
regionVer = r.Version // what the region runs (this binary may differ)
}
}
for _, r := range racks {
switch {
case !r.Online:
d.fail("rack "+r.Name, "offline since "+timeStr(r.LastSeen), "on that host: systemctl status rollcall-rack")
case r.Version != regionVer && r.Version != "":
d.warn("rack "+r.Name, "runs "+r.Version+", region "+regionVer, "it updates itself within minutes (rollcall-rack-update.path); see /var/lib/rollcall/update/last-result there")
default:
d.ok("rack "+r.Name, fmt.Sprintf("online, %d machines", r.Machines))
}
}
}
var oss []map[string]any
if err := c.do("GET", "/api/v1/os", nil, &oss); err == nil {
if len(oss) == 0 {
d.warn("os images", "none", "runuser -u rollcall -- rollcall os sync debian-13")
} else {
d.ok("os images", fmt.Sprintf("%d in the catalog", len(oss)))
}
}
var ipam map[string]any
if err := c.do("GET", "/api/v1/ipam", nil, &ipam); err == nil && ipam["provider"] != "" && ipam["provider"] != nil {
last, _ := ipam["last_sync"].(map[string]any)
if e, _ := last["error"].(string); e != "" {
d.fail("ipam", e, "check ipam.url/token_file/ca_file; rollcall ipam sync")
} else {
d.ok("ipam", fmt.Sprintf("%v in sync", ipam["provider"]))
}
}
} else {
d.warn("api", err.Error(), "")
}
return d.done()
}
func (d *doctor) done() error {
fmt.Printf("\n%d failed, %d warnings\n", d.fails, d.warns)
if d.fails > 0 {
os.Exit(1)
}
return nil
}
func errOrStatus(err error, resp *http.Response) string {
if err != nil {
return err.Error()
}
return resp.Status
}
func urlHost(u string) (string, error) {
u = strings.TrimPrefix(strings.TrimPrefix(u, "http://"), "https://")
u = strings.SplitN(u, "/", 2)[0]
if h, _, err := net.SplitHostPort(u); err == nil {
return strings.Trim(h, "[]"), nil
}
return strings.Trim(u, "[]"), nil
}
// ifaceWithAddr: the interface holding ip on this host, "" if none.
func ifaceWithAddr(ip string) string {
want := net.ParseIP(ip)
if want == nil {
return ""
}
ifs, _ := net.Interfaces()
for _, i := range ifs {
addrs, _ := i.Addrs()
for _, a := range addrs {
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(want) {
return i.Name
}
}
}
return ""
}
func portOf(listen string, def int) int {
if listen == "" {
return 0
}
_, p, err := net.SplitHostPort(listen)
if err != nil {
return def
}
n, _ := strconv.Atoi(p)
return n
}
func localPort(listen string) string {
if p := portOf(listen, 0); p != 0 && p != 80 && p != 443 {
return ":" + strconv.Itoa(p)
}
if p := portOf(listen, 0); p == 443 {
return ":443"
}
return ""
}
// listeningPorts reads /proc/net: "tcp/80", "udp/67" ...
func listeningPorts() map[string]bool {
out := map[string]bool{}
for _, f := range []string{"tcp", "tcp6", "udp", "udp6"} {
fh, err := os.Open("/proc/net/" + f)
if err != nil {
continue
}
sc := bufio.NewScanner(fh)
sc.Scan()
for sc.Scan() {
fl := strings.Fields(sc.Text())
if len(fl) < 4 {
continue
}
if strings.HasPrefix(f, "tcp") && fl[3] != "0A" { // LISTEN
continue
}
i := strings.LastIndex(fl[1], ":")
p, err := strconv.ParseUint(fl[1][i+1:], 16, 16)
if err == nil {
out[strings.TrimSuffix(f, "6")+"/"+strconv.Itoa(int(p))] = true
}
}
fh.Close()
}
return out
}
func verifySums(dir string) string {
b, err := os.ReadFile(filepath.Join(dir, "SHA256SUMS"))
if err != nil {
return "no SHA256SUMS next to the image"
}
for _, line := range strings.Split(string(b), "\n") {
f := strings.Fields(line)
if len(f) != 2 {
continue
}
name := strings.TrimPrefix(f[1], "*")
fh, err := os.Open(filepath.Join(dir, name))
if err != nil {
return name + ": " + err.Error()
}
h := sha256.New()
_, _ = io.Copy(h, fh)
fh.Close()
if hex.EncodeToString(h.Sum(nil)) != f[0] {
return name + ": checksum mismatch"
}
}
return ""
}
// dhcpProbe broadcasts a DHCPDISCOVER on iface and returns the server
// identifiers that offer an address. Needs root (port 68, SO_BINDTODEVICE).
func dhcpProbe(iface string, wait time.Duration) ([]string, error) {
lc := net.ListenConfig{Control: func(_, _ string, c syscall.RawConn) error {
var serr error
err := c.Control(func(fd uintptr) {
serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)
if serr == nil {
serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
}
if serr == nil {
serr = bindToDevice(int(fd), iface)
}
})
if err != nil {
return err
}
return serr
}}
pc, err := lc.ListenPacket(context.Background(), "udp4", "0.0.0.0:68")
if err != nil {
return nil, fmt.Errorf("cannot listen on udp/68: %v", err)
}
defer pc.Close()
if _, err := net.InterfaceByName(iface); err != nil {
return nil, err
}
// a locally administered MAC so the probe never takes a real host's lease
mac := make([]byte, 6)
_, _ = rand.Read(mac)
mac[0] = (mac[0] | 0x02) & 0xfe
xid := make([]byte, 4)
_, _ = rand.Read(xid)
p := make([]byte, 240)
p[0], p[1], p[2] = 1, 1, 6 // BOOTREQUEST, ethernet, hlen
copy(p[4:8], xid)
binary.BigEndian.PutUint16(p[10:12], 0x8000) // broadcast reply
copy(p[28:34], mac)
copy(p[236:240], []byte{99, 130, 83, 99})
p = append(p, 53, 1, 1, 55, 3, 1, 3, 6, 255) // DISCOVER; ask router, DNS
if _, err := pc.WriteTo(p, &net.UDPAddr{IP: net.IPv4bcast, Port: 67}); err != nil {
return nil, err
}
seen := map[string]bool{}
var out []string
_ = pc.SetReadDeadline(time.Now().Add(wait))
buf := make([]byte, 1500)
for {
n, _, err := pc.ReadFrom(buf)
if err != nil {
break
}
r := buf[:n]
if n < 240 || r[0] != 2 || string(r[4:8]) != string(xid) {
continue
}
srv := ""
for i := 240; i < n; {
o := r[i]
if o == 255 {
break
}
if o == 0 {
i++
continue
}
if i+1 >= n {
break
}
l := int(r[i+1])
if o == 54 && l == 4 && i+6 <= n {
srv = net.IP(r[i+2 : i+6]).String()
}
i += 2 + l
}
if srv == "" {
srv = net.IP(r[20:24]).String() // siaddr
}
if !seen[srv] {
seen[srv] = true
out = append(out, srv)
}
}
return out, nil
}
+9
View File
@@ -0,0 +1,9 @@
//go:build linux
package main
import "syscall"
func bindToDevice(fd int, iface string) error {
return syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, iface)
}
+7
View File
@@ -0,0 +1,7 @@
//go:build !linux
package main
import "errors"
func bindToDevice(int, string) error { return errors.New("the DHCP probe needs Linux") }
+6
View File
@@ -27,6 +27,7 @@ func usage() {
rollcall in a terminal: the full-screen console
rollcall tui [-dump WxH -page P -machine ID -plain]
rollcall serve [-config PATH] region: API, database, web UI, embedded rack controller
rollcall doctor [-deep] [-no-dhcp-probe] check this installation and say what to fix
rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines
rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate)
rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host)
@@ -107,6 +108,11 @@ func main() {
fmt.Fprintln(os.Stderr, "error:", err)
os.Exit(1)
}
case "doctor":
if err := doctorCmd(args); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)
os.Exit(1)
}
case "tui", "console":
if err := tuiDump(args); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)