Files
rollcall/cmd/rollcall/doctor.go
T
Ilker ManapandClaude Opus 5.5 05861bd413
CI / go (push) Successful in 51s
CI / agent (push) Successful in 3s
CI / site (push) Successful in 3s
Intermediates renew themselves through the region
The region keeps every intermediate it signs: its name constraints,
certificate, the root that signed it and a renewal token (stored hashed,
printed once). `rollcall ca intermediate` goes through the API now (so it
works from any admin client) and writes NAME.token beside NAME.pem;
`rollcall ca intermediates` lists them.

The host with the intermediate's key runs deploy/rollcall-intermediate-
renew.sh daily (rollcall-intermediate-renew@NAME.timer): it posts a CSR
from the same key with the token to /api/v1/ca/intermediates/NAME/renew.
The region signs again in the last third of the certificate's life, or
at once when another root signs since a rotation's switch, always with
the constraints it was first given; otherwise 204. The script checks the
answer (a CA certificate for its key, from a root it holds), keeps the
old file, keeps the region's roots and the active root current from
/ca-bundle.pem, and reloads (RELOAD).

Attention: an intermediate under 30 days from its end, whose last renewal
failed or whose host has not asked for three days shows in rollcall
doctor, on the web UI's overview, and one from the old root in the
rotation status.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-01 17:33:31 +01:00

534 lines
17 KiB
Go

package main
// rollcall doctor: checks an installation from the region host and says what
// to do about what is missing. Local checks read the configuration; checks of
// the running state ask the region's API with its own admin token.
import (
"bufio"
"context"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/binary"
"encoding/hex"
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
)
type doctor struct {
fails, warns int
}
func (d *doctor) ok(name, detail string) {
fmt.Printf(" \033[32mOK \033[0m %-24s %s\n", name, detail)
}
func (d *doctor) warn(name, detail, fix string) {
d.warns++
fmt.Printf(" \033[33mWARN\033[0m %-24s %s\n", name, detail)
if fix != "" {
fmt.Printf(" %-24s -> %s\n", "", fix)
}
}
func (d *doctor) fail(name, detail, fix string) {
d.fails++
fmt.Printf(" \033[31mFAIL\033[0m %-24s %s\n", name, detail)
if fix != "" {
fmt.Printf(" %-24s -> %s\n", "", fix)
}
}
func doctorCmd(args []string) error {
fs := flag.NewFlagSet("doctor", flag.ExitOnError)
cfgPath := fs.String("config", envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"), "config file")
deep := fs.Bool("deep", false, "also verify the commissioning image checksums")
noProbe := fs.Bool("no-dhcp-probe", false, "do not send a DHCPDISCOVER on the provisioning interface")
_ = fs.Parse(args)
d := &doctor{}
fmt.Println("rollcall doctor", version)
cfg, err := loadConfig(*cfgPath)
if err != nil {
d.fail("config", err.Error(), "deploy/bootstrap.sh writes one; or see deploy/config.example.json")
return d.done()
}
d.ok("config", *cfgPath)
srvHost := ""
if u, err := urlHost(cfg.ServerURL); err == nil {
srvHost = u
}
iface := ifaceWithAddr(srvHost)
if iface == "" {
d.warn("server_url", cfg.ServerURL+" is not an address of this host", "machines reach the region there; set server_url to this host's address on the provisioning network")
} else {
d.ok("server_url", fmt.Sprintf("%s on %s", cfg.ServerURL, iface))
}
// database
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
st, err := openStore(ctx, cfg.DatabaseURL)
cancel()
if err != nil {
d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)")
} else {
d.ok("database", "connected, schema current")
// delegated intermediates the region renews: ending, failing, or a silent host
active := ""
if b, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil {
if c, err := parseCertPEM(b); err == nil {
active = fpHex(c)
}
}
ictx, icancel := context.WithTimeout(context.Background(), 10*time.Second)
if xs, err := st.Intermediates(ictx); err == nil {
for _, x := range xs {
if p := x.problem(active, time.Now()); p != "" {
d.warn("intermediate "+x.Name, p, "on its host: systemctl status rollcall-intermediate-renew; rollcall ca intermediates")
} else {
d.ok("intermediate "+x.Name, "until "+x.NotAfter.Format("2006-01-02")+", renewed by its host")
}
}
}
icancel()
st.pool.Close()
}
// listeners
ports := listeningPorts()
for _, p := range []struct {
proto string
port int
what string
}{{"udp", 67, "DHCP (dnsmasq)"}, {"udp", 69, "TFTP (dnsmasq)"}, {"tcp", portOf(cfg.Listen, 80), "HTTP (rollcall)"}, {"tcp", portOf(cfg.TLSListen, 443), "HTTPS (rollcall)"}} {
if p.port == 0 {
continue
}
if ports[fmt.Sprintf("%s/%d", p.proto, p.port)] {
d.ok(fmt.Sprintf("%s/%d", p.proto, p.port), p.what)
} else if p.port == 69 && !ports["udp/67"] {
d.ok("udp/69", "TFTP starts with the first VLAN served")
} else if p.port == 67 {
d.warn("udp/67", "this host serves no DHCP yet", "no provisioning network decided: rollcall net discover, then deploy/bootstrap.sh --iface IFACE (or a VLAN with dhcp in the web UI)")
} else {
d.fail(fmt.Sprintf("%s/%d", p.proto, p.port), "nothing listens for "+p.what, "systemctl status rollcall dnsmasq")
}
}
hc := &http.Client{Timeout: 5 * time.Second}
if resp, err := hc.Get("http://127.0.0.1" + localPort(cfg.Listen) + "/ca.pem"); err != nil || resp.StatusCode != 200 {
d.fail("http", fmt.Sprintf("GET /ca.pem: %v", errOrStatus(err, resp)), "journalctl -u rollcall")
} else {
resp.Body.Close()
d.ok("http", "region answers")
}
// certificates
if cb, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil {
if c, err := parseCertPEM(cb); err == nil {
left := time.Until(c.NotAfter)
switch {
case left < 90*24*time.Hour:
d.warn("CA", "expires "+c.NotAfter.Format("2006-01-02"), "plan rollcall ca rotate")
default:
d.ok("CA", fmt.Sprintf("until %s, delegation %v", c.NotAfter.Format("2006-01-02"), (&CA{Cert: c}).canDelegate()))
}
if cfg.TLSListen != "" {
pool := x509.NewCertPool()
pool.AddCert(c)
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", "127.0.0.1"+localPort(cfg.TLSListen),
&tls.Config{RootCAs: pool, ServerName: srvHost})
if err != nil {
d.fail("https", err.Error(), "the region certificate should cover "+srvHost+" (tls_names); journalctl -u rollcall")
} else {
leaf := conn.ConnectionState().PeerCertificates[0]
conn.Close()
d.ok("https", fmt.Sprintf("certificate for %s until %s", srvHost, leaf.NotAfter.Format("2006-01-02")))
}
}
}
} else {
d.fail("CA", "no CA yet", "it is created when rollcall serve starts")
}
// TFTP + image
missing := []string{}
for _, f := range []string{"pxelinux.0", "ldlinux.c32", "menu.c32", "libcom32.c32", "libutil.c32", "efi64/syslinux.efi", "efi64/ldlinux.e64", "pxelinux.cfg/default"} {
if _, err := os.Stat(filepath.Join(cfg.TFTPRoot, f)); err != nil {
missing = append(missing, f)
}
}
if len(missing) > 0 {
d.fail("tftp", "missing in "+cfg.TFTPRoot+": "+strings.Join(missing, " "), "deploy/bootstrap.sh (step 3) copies them from pxelinux/syslinux-efi")
} else {
d.ok("tftp", cfg.TFTPRoot+" (BIOS + UEFI)")
}
img := []string{}
for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} {
if st, err := os.Stat(filepath.Join(cfg.ImageDir, f)); err != nil || st.Size() == 0 {
img = append(img, f)
}
}
switch {
case len(img) > 0:
d.fail("image", "commissioning image incomplete, missing "+strings.Join(img, " "), "deploy/bootstrap.sh --image-url URL or --build-image")
case *deep:
if bad := verifySums(cfg.ImageDir); bad == "no SHA256SUMS next to the image" {
d.warn("image", bad, "cd "+cfg.ImageDir+" && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS")
} else if bad != "" {
d.fail("image", bad, "download the image again")
} else {
d.ok("image", "checksums match SHA256SUMS")
}
default:
d.ok("image", cfg.ImageDir)
}
// dnsmasq
if out, _ := exec.Command("systemctl", "is-active", "dnsmasq").Output(); strings.TrimSpace(string(out)) != "active" {
d.fail("dnsmasq", "not running", "journalctl -u dnsmasq -n 20")
} else if _, err := os.Stat("/var/lib/rollcall-dnsmasq/rollcall.conf.rejected"); err == nil {
d.fail("dnsmasq", "the generated configuration was rejected; the last working one is in use",
"journalctl -u rollcall-dnsmasq; a hand-written DHCP/TFTP file in /etc/dnsmasq.d usually conflicts")
} else {
d.ok("dnsmasq", "running")
}
// DNS: outside, and through dnsmasq as the machines see it
if _, err := net.DefaultResolver.LookupHost(context.Background(), "deb.debian.org"); err != nil {
d.fail("dns (host)", err.Error(), "this host cannot resolve; check /etc/resolv.conf / resolvectl")
} else {
d.ok("dns (host)", "resolves outside names")
}
if srvHost != "" && ports["udp/53"] && ports["udp/67"] {
r := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "udp", net.JoinHostPort(srvHost, "53"))
}}
if _, err := r.LookupHost(context.Background(), "deb.debian.org"); err != nil {
d.fail("dns (machines)", "dnsmasq on "+srvHost+" does not resolve outside names: "+err.Error(),
"the generated dnsmasq configuration needs an upstream (rollcall >= 0.2 writes one); check its # upstream section")
} else {
d.ok("dns (machines)", "dnsmasq on "+srvHost+" forwards outside names")
}
}
// time
if out, err := exec.Command("timedatectl", "show", "-p", "NTPSynchronized", "--value").Output(); err == nil {
if strings.TrimSpace(string(out)) == "yes" {
d.ok("time", "synchronised")
} else {
d.warn("time", "clock not synchronised", "certificates and tokens depend on it: systemctl enable --now systemd-timesyncd")
}
}
// other DHCP servers on the provisioning network (expected when this rack proxies)
proxying := false
if c, err := newClient(); err == nil {
var subs []Subnet
if c.do("GET", "/api/v1/subnets", nil, &subs) == nil {
for _, s := range subs {
if s.DHCPMode == "proxy" && (s.PrimaryRack == cfg.RackName || s.SecondaryRack == cfg.RackName) {
proxying = true
}
}
}
}
if iface != "" && !*noProbe && !ports["udp/67"] {
d.ok("dhcp probe", "skipped: this host serves no DHCP yet")
} else if iface != "" && !*noProbe {
servers, err := dhcpProbe(iface, 6*time.Second)
switch {
case err != nil:
d.warn("dhcp probe", err.Error(), "run as root; or --no-dhcp-probe")
default:
var foreign []string
for _, s := range servers {
if ifaceWithAddr(s) == "" {
foreign = append(foreign, s)
}
}
switch {
case proxying && len(foreign) > 0:
d.ok("dhcp probe", "the site's DHCP server "+strings.Join(foreign, ", ")+" hands out addresses; this host answers PXE (proxy)")
case proxying:
d.fail("dhcp probe", "proxy DHCP, but no DHCP server answers on "+iface, "machines would get no address: the site's DHCP server must serve this VLAN (or a relay must reach it)")
case len(foreign) > 0:
d.fail("dhcp probe", "another DHCP server answers on "+iface+": "+strings.Join(foreign, ", "),
"machines may PXE-boot from it: switch it off for this VLAN, or keep it and proxy (bootstrap.sh --iface IFACE --proxy-dhcp, or DHCP mode proxy in the web UI)")
case len(servers) == 0:
d.warn("dhcp probe", "no DHCP server answered on "+iface, "is a VLAN with dhcp assigned to this rack? (rollcall vlan add ... -dhcp)")
default:
d.ok("dhcp probe", "only this host answers on "+iface)
}
}
}
// running state through the API
if c, err := newClient(); err == nil {
var racks []RackController
if err := c.do("GET", "/api/v1/controllers", nil, &racks); err == nil {
regionVer := version
for _, r := range racks {
if r.Kind == "embedded" && r.Version != "" {
regionVer = r.Version // what the region runs (this binary may differ)
}
}
for _, r := range racks {
switch {
case !r.Online:
d.fail("rack "+r.Name, "offline since "+timeStr(r.LastSeen), "on that host: systemctl status rollcall-rack")
case r.Version != regionVer && r.Version != "":
d.warn("rack "+r.Name, "runs "+r.Version+", region "+regionVer, "it updates itself within minutes (rollcall-rack-update.path); see /var/lib/rollcall/update/last-result there")
default:
d.ok("rack "+r.Name, fmt.Sprintf("online, %d machines", r.Machines))
}
}
}
var oss []map[string]any
if err := c.do("GET", "/api/v1/os", nil, &oss); err == nil {
if len(oss) == 0 {
d.warn("os images", "none", "runuser -u rollcall -- rollcall os sync debian-13")
} else {
d.ok("os images", fmt.Sprintf("%d in the catalog", len(oss)))
}
}
var dnsSt map[string]any
if err := c.do("GET", "/api/v1/dns", nil, &dnsSt); err == nil && dnsSt["provider"] != "" && dnsSt["provider"] != nil {
last, _ := dnsSt["last_sync"].(map[string]any)
switch e, _ := last["error"].(string); {
case e != "":
d.fail("dns provider", e, "check dns.* in config.json; rollcall dns sync")
case last["conflicts"] != nil && len(last["conflicts"].([]any)) > 0:
d.warn("dns provider", fmt.Sprintf("%v: %d names held by someone else", dnsSt["provider"], len(last["conflicts"].([]any))), "rollcall dns: those records are left alone")
default:
d.ok("dns provider", fmt.Sprintf("%v, zone %v, in sync", dnsSt["provider"], dnsSt["zone"]))
}
}
var ipam map[string]any
if err := c.do("GET", "/api/v1/ipam", nil, &ipam); err == nil && ipam["provider"] != "" && ipam["provider"] != nil {
last, _ := ipam["last_sync"].(map[string]any)
if e, _ := last["error"].(string); e != "" {
d.fail("ipam", e, "check ipam.url/token_file/ca_file; rollcall ipam sync")
} else {
d.ok("ipam", fmt.Sprintf("%v in sync", ipam["provider"]))
}
}
} else {
d.warn("api", err.Error(), "")
}
return d.done()
}
func (d *doctor) done() error {
fmt.Printf("\n%d failed, %d warnings\n", d.fails, d.warns)
if d.fails > 0 {
os.Exit(1)
}
return nil
}
func errOrStatus(err error, resp *http.Response) string {
if err != nil {
return err.Error()
}
return resp.Status
}
func urlHost(u string) (string, error) {
u = strings.TrimPrefix(strings.TrimPrefix(u, "http://"), "https://")
u = strings.SplitN(u, "/", 2)[0]
if h, _, err := net.SplitHostPort(u); err == nil {
return strings.Trim(h, "[]"), nil
}
return strings.Trim(u, "[]"), nil
}
// ifaceWithAddr: the interface holding ip on this host, "" if none.
func ifaceWithAddr(ip string) string {
want := net.ParseIP(ip)
if want == nil {
return ""
}
ifs, _ := net.Interfaces()
for _, i := range ifs {
addrs, _ := i.Addrs()
for _, a := range addrs {
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(want) {
return i.Name
}
}
}
return ""
}
func portOf(listen string, def int) int {
if listen == "" {
return 0
}
_, p, err := net.SplitHostPort(listen)
if err != nil {
return def
}
n, _ := strconv.Atoi(p)
return n
}
func localPort(listen string) string {
if p := portOf(listen, 0); p != 0 && p != 80 && p != 443 {
return ":" + strconv.Itoa(p)
}
if p := portOf(listen, 0); p == 443 {
return ":443"
}
return ""
}
// listeningPorts reads /proc/net: "tcp/80", "udp/67" ...
func listeningPorts() map[string]bool {
out := map[string]bool{}
for _, f := range []string{"tcp", "tcp6", "udp", "udp6"} {
fh, err := os.Open("/proc/net/" + f)
if err != nil {
continue
}
sc := bufio.NewScanner(fh)
sc.Scan()
for sc.Scan() {
fl := strings.Fields(sc.Text())
if len(fl) < 4 {
continue
}
if strings.HasPrefix(f, "tcp") && fl[3] != "0A" { // LISTEN
continue
}
i := strings.LastIndex(fl[1], ":")
p, err := strconv.ParseUint(fl[1][i+1:], 16, 16)
if err == nil {
out[strings.TrimSuffix(f, "6")+"/"+strconv.Itoa(int(p))] = true
}
}
fh.Close()
}
return out
}
func verifySums(dir string) string {
b, err := os.ReadFile(filepath.Join(dir, "SHA256SUMS"))
if err != nil {
return "no SHA256SUMS next to the image"
}
for _, line := range strings.Split(string(b), "\n") {
f := strings.Fields(line)
if len(f) != 2 {
continue
}
name := strings.TrimPrefix(f[1], "*")
fh, err := os.Open(filepath.Join(dir, name))
if err != nil {
return name + ": " + err.Error()
}
h := sha256.New()
_, _ = io.Copy(h, fh)
fh.Close()
if hex.EncodeToString(h.Sum(nil)) != f[0] {
return name + ": checksum mismatch"
}
}
return ""
}
// dhcpProbe broadcasts a DHCPDISCOVER on iface and returns the server
// identifiers that offer an address. Needs root (port 68, SO_BINDTODEVICE).
func dhcpProbe(iface string, wait time.Duration) ([]string, error) {
lc := net.ListenConfig{Control: func(_, _ string, c syscall.RawConn) error {
var serr error
err := c.Control(func(fd uintptr) {
serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)
if serr == nil {
serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
}
if serr == nil {
serr = bindToDevice(int(fd), iface)
}
})
if err != nil {
return err
}
return serr
}}
pc, err := lc.ListenPacket(context.Background(), "udp4", "0.0.0.0:68")
if err != nil {
return nil, fmt.Errorf("cannot listen on udp/68: %v", err)
}
defer pc.Close()
if _, err := net.InterfaceByName(iface); err != nil {
return nil, err
}
// a locally administered MAC so the probe never takes a real host's lease
mac := make([]byte, 6)
_, _ = rand.Read(mac)
mac[0] = (mac[0] | 0x02) & 0xfe
xid := make([]byte, 4)
_, _ = rand.Read(xid)
p := make([]byte, 240)
p[0], p[1], p[2] = 1, 1, 6 // BOOTREQUEST, ethernet, hlen
copy(p[4:8], xid)
binary.BigEndian.PutUint16(p[10:12], 0x8000) // broadcast reply
copy(p[28:34], mac)
copy(p[236:240], []byte{99, 130, 83, 99})
p = append(p, 53, 1, 1, 55, 3, 1, 3, 6, 255) // DISCOVER; ask router, DNS
if _, err := pc.WriteTo(p, &net.UDPAddr{IP: net.IPv4bcast, Port: 67}); err != nil {
return nil, err
}
seen := map[string]bool{}
var out []string
_ = pc.SetReadDeadline(time.Now().Add(wait))
buf := make([]byte, 1500)
for {
n, _, err := pc.ReadFrom(buf)
if err != nil {
break
}
r := buf[:n]
if n < 240 || r[0] != 2 || string(r[4:8]) != string(xid) {
continue
}
srv := ""
for i := 240; i < n; {
o := r[i]
if o == 255 {
break
}
if o == 0 {
i++
continue
}
if i+1 >= n {
break
}
l := int(r[i+1])
if o == 54 && l == 4 && i+6 <= n {
srv = net.IP(r[i+2 : i+6]).String()
}
i += 2 + l
}
if srv == "" {
srv = net.IP(r[20:24]).String() // siaddr
}
if !seen[srv] {
seen[srv] = true
out = append(out, srv)
}
}
return out, nil
}