Files
rollcall/cmd/rollcall/doctor.go
T
Ilker ManapandClaude Opus 5.5 a549f01ef5
CI / agent (push) Successful in 3s
CI / site (push) Successful in 2s
CI / go (push) Successful in 34s
Security review: fix traversal, identity, injection and takeover paths
Findings from a full review (auth/API, CA and rack channel, config
generation, integrations, UI/agents/scripts), each verified in code:

- rack binary endpoint read any region file via arch=../.. (CA key)
- render endpoint traversal; machine-reported MACs used as file paths
- delegated intermediates could mint admin/rack identities and IPs;
  identities now require root-issued certs, intermediates are
  ServerAuth-only with both name types constrained
- rack certs were ServerAuth for rack-chosen SANs: now client-only;
  racks accept only the region's own cert; join checks the CA
  fingerprint; unbound join codes cannot take an existing name; job
  results bound to the rack that took them
- rack self-update: region signs binaries with the CA key, the root
  unit verifies against the join-time CA in a root-owned dir
- dnsmasq reads a checked, root-owned copy limited to generated options
- commissioning could overwrite deployed machines; MAC takeover of PXE
- injection into dnsmasq/cloud-init/PXE/udev/DNS via names and
  machine-reported strings; stored XSS in the UI; CSP and cross-origin
  protection; deploy agent shell injection and seed traversal
- Redfish requests confined to the BMC's /redfish/ tree
- admin_http defaults to loopback; body limits and timeouts; secrets
  off command lines

Also: IPAM claim/reconcile race, sync stopping on one error, watchdog
race, release wiping the wrong disk, SAN LUNs as auto targets.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 17:58:42 +01:00

516 lines
16 KiB
Go

package main
// rollcall doctor: checks an installation from the region host and says what
// to do about what is missing. Local checks read the configuration; checks of
// the running state ask the region's API with its own admin token.
import (
"bufio"
"context"
"crypto/rand"
"crypto/sha256"
"crypto/tls"
"crypto/x509"
"encoding/binary"
"encoding/hex"
"flag"
"fmt"
"io"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"strconv"
"strings"
"syscall"
"time"
)
type doctor struct {
fails, warns int
}
func (d *doctor) ok(name, detail string) {
fmt.Printf(" \033[32mOK \033[0m %-24s %s\n", name, detail)
}
func (d *doctor) warn(name, detail, fix string) {
d.warns++
fmt.Printf(" \033[33mWARN\033[0m %-24s %s\n", name, detail)
if fix != "" {
fmt.Printf(" %-24s -> %s\n", "", fix)
}
}
func (d *doctor) fail(name, detail, fix string) {
d.fails++
fmt.Printf(" \033[31mFAIL\033[0m %-24s %s\n", name, detail)
if fix != "" {
fmt.Printf(" %-24s -> %s\n", "", fix)
}
}
func doctorCmd(args []string) error {
fs := flag.NewFlagSet("doctor", flag.ExitOnError)
cfgPath := fs.String("config", envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"), "config file")
deep := fs.Bool("deep", false, "also verify the commissioning image checksums")
noProbe := fs.Bool("no-dhcp-probe", false, "do not send a DHCPDISCOVER on the provisioning interface")
_ = fs.Parse(args)
d := &doctor{}
fmt.Println("rollcall doctor", version)
cfg, err := loadConfig(*cfgPath)
if err != nil {
d.fail("config", err.Error(), "deploy/bootstrap.sh writes one; or see deploy/config.example.json")
return d.done()
}
d.ok("config", *cfgPath)
srvHost := ""
if u, err := urlHost(cfg.ServerURL); err == nil {
srvHost = u
}
iface := ifaceWithAddr(srvHost)
if iface == "" {
d.warn("server_url", cfg.ServerURL+" is not an address of this host", "machines reach the region there; set server_url to this host's address on the provisioning network")
} else {
d.ok("server_url", fmt.Sprintf("%s on %s", cfg.ServerURL, iface))
}
// database
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
st, err := openStore(ctx, cfg.DatabaseURL)
cancel()
if err != nil {
d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)")
} else {
d.ok("database", "connected, schema current")
st.pool.Close()
}
// listeners
ports := listeningPorts()
for _, p := range []struct {
proto string
port int
what string
}{{"udp", 67, "DHCP (dnsmasq)"}, {"udp", 69, "TFTP (dnsmasq)"}, {"tcp", portOf(cfg.Listen, 80), "HTTP (rollcall)"}, {"tcp", portOf(cfg.TLSListen, 443), "HTTPS (rollcall)"}} {
if p.port == 0 {
continue
}
if ports[fmt.Sprintf("%s/%d", p.proto, p.port)] {
d.ok(fmt.Sprintf("%s/%d", p.proto, p.port), p.what)
} else if p.port == 69 && !ports["udp/67"] {
d.ok("udp/69", "TFTP starts with the first VLAN served")
} else if p.port == 67 {
d.warn("udp/67", "this host serves no DHCP yet", "no provisioning network decided: rollcall net discover, then deploy/bootstrap.sh --iface IFACE (or a VLAN with dhcp in the web UI)")
} else {
d.fail(fmt.Sprintf("%s/%d", p.proto, p.port), "nothing listens for "+p.what, "systemctl status rollcall dnsmasq")
}
}
hc := &http.Client{Timeout: 5 * time.Second}
if resp, err := hc.Get("http://127.0.0.1" + localPort(cfg.Listen) + "/ca.pem"); err != nil || resp.StatusCode != 200 {
d.fail("http", fmt.Sprintf("GET /ca.pem: %v", errOrStatus(err, resp)), "journalctl -u rollcall")
} else {
resp.Body.Close()
d.ok("http", "region answers")
}
// certificates
if cb, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil {
if c, err := parseCertPEM(cb); err == nil {
left := time.Until(c.NotAfter)
switch {
case left < 90*24*time.Hour:
d.warn("CA", "expires "+c.NotAfter.Format("2006-01-02"), "plan rollcall ca rotate")
default:
d.ok("CA", fmt.Sprintf("until %s, delegation %v", c.NotAfter.Format("2006-01-02"), (&CA{Cert: c}).canDelegate()))
}
if cfg.TLSListen != "" {
pool := x509.NewCertPool()
pool.AddCert(c)
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", "127.0.0.1"+localPort(cfg.TLSListen),
&tls.Config{RootCAs: pool, ServerName: srvHost})
if err != nil {
d.fail("https", err.Error(), "the region certificate should cover "+srvHost+" (tls_names); journalctl -u rollcall")
} else {
leaf := conn.ConnectionState().PeerCertificates[0]
conn.Close()
d.ok("https", fmt.Sprintf("certificate for %s until %s", srvHost, leaf.NotAfter.Format("2006-01-02")))
}
}
}
} else {
d.fail("CA", "no CA yet", "it is created when rollcall serve starts")
}
// TFTP + image
missing := []string{}
for _, f := range []string{"pxelinux.0", "ldlinux.c32", "menu.c32", "libcom32.c32", "libutil.c32", "efi64/syslinux.efi", "efi64/ldlinux.e64", "pxelinux.cfg/default"} {
if _, err := os.Stat(filepath.Join(cfg.TFTPRoot, f)); err != nil {
missing = append(missing, f)
}
}
if len(missing) > 0 {
d.fail("tftp", "missing in "+cfg.TFTPRoot+": "+strings.Join(missing, " "), "deploy/bootstrap.sh (step 3) copies them from pxelinux/syslinux-efi")
} else {
d.ok("tftp", cfg.TFTPRoot+" (BIOS + UEFI)")
}
img := []string{}
for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} {
if st, err := os.Stat(filepath.Join(cfg.ImageDir, f)); err != nil || st.Size() == 0 {
img = append(img, f)
}
}
switch {
case len(img) > 0:
d.fail("image", "commissioning image incomplete, missing "+strings.Join(img, " "), "deploy/bootstrap.sh --image-url URL or --build-image")
case *deep:
if bad := verifySums(cfg.ImageDir); bad == "no SHA256SUMS next to the image" {
d.warn("image", bad, "cd "+cfg.ImageDir+" && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS")
} else if bad != "" {
d.fail("image", bad, "download the image again")
} else {
d.ok("image", "checksums match SHA256SUMS")
}
default:
d.ok("image", cfg.ImageDir)
}
// dnsmasq
if out, _ := exec.Command("systemctl", "is-active", "dnsmasq").Output(); strings.TrimSpace(string(out)) != "active" {
d.fail("dnsmasq", "not running", "journalctl -u dnsmasq -n 20")
} else if _, err := os.Stat("/var/lib/rollcall-dnsmasq/rollcall.conf.rejected"); err == nil {
d.fail("dnsmasq", "the generated configuration was rejected; the last working one is in use",
"journalctl -u rollcall-dnsmasq; a hand-written DHCP/TFTP file in /etc/dnsmasq.d usually conflicts")
} else {
d.ok("dnsmasq", "running")
}
// DNS: outside, and through dnsmasq as the machines see it
if _, err := net.DefaultResolver.LookupHost(context.Background(), "deb.debian.org"); err != nil {
d.fail("dns (host)", err.Error(), "this host cannot resolve; check /etc/resolv.conf / resolvectl")
} else {
d.ok("dns (host)", "resolves outside names")
}
if srvHost != "" && ports["udp/53"] && ports["udp/67"] {
r := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "udp", net.JoinHostPort(srvHost, "53"))
}}
if _, err := r.LookupHost(context.Background(), "deb.debian.org"); err != nil {
d.fail("dns (machines)", "dnsmasq on "+srvHost+" does not resolve outside names: "+err.Error(),
"the generated dnsmasq configuration needs an upstream (rollcall >= 0.2 writes one); check its # upstream section")
} else {
d.ok("dns (machines)", "dnsmasq on "+srvHost+" forwards outside names")
}
}
// time
if out, err := exec.Command("timedatectl", "show", "-p", "NTPSynchronized", "--value").Output(); err == nil {
if strings.TrimSpace(string(out)) == "yes" {
d.ok("time", "synchronised")
} else {
d.warn("time", "clock not synchronised", "certificates and tokens depend on it: systemctl enable --now systemd-timesyncd")
}
}
// other DHCP servers on the provisioning network (expected when this rack proxies)
proxying := false
if c, err := newClient(); err == nil {
var subs []Subnet
if c.do("GET", "/api/v1/subnets", nil, &subs) == nil {
for _, s := range subs {
if s.DHCPMode == "proxy" && (s.PrimaryRack == cfg.RackName || s.SecondaryRack == cfg.RackName) {
proxying = true
}
}
}
}
if iface != "" && !*noProbe && !ports["udp/67"] {
d.ok("dhcp probe", "skipped: this host serves no DHCP yet")
} else if iface != "" && !*noProbe {
servers, err := dhcpProbe(iface, 6*time.Second)
switch {
case err != nil:
d.warn("dhcp probe", err.Error(), "run as root; or --no-dhcp-probe")
default:
var foreign []string
for _, s := range servers {
if ifaceWithAddr(s) == "" {
foreign = append(foreign, s)
}
}
switch {
case proxying && len(foreign) > 0:
d.ok("dhcp probe", "the site's DHCP server "+strings.Join(foreign, ", ")+" hands out addresses; this host answers PXE (proxy)")
case proxying:
d.fail("dhcp probe", "proxy DHCP, but no DHCP server answers on "+iface, "machines would get no address: the site's DHCP server must serve this VLAN (or a relay must reach it)")
case len(foreign) > 0:
d.fail("dhcp probe", "another DHCP server answers on "+iface+": "+strings.Join(foreign, ", "),
"machines may PXE-boot from it: switch it off for this VLAN, or keep it and proxy (bootstrap.sh --iface IFACE --proxy-dhcp, or DHCP mode proxy in the web UI)")
case len(servers) == 0:
d.warn("dhcp probe", "no DHCP server answered on "+iface, "is a VLAN with dhcp assigned to this rack? (rollcall vlan add ... -dhcp)")
default:
d.ok("dhcp probe", "only this host answers on "+iface)
}
}
}
// running state through the API
if c, err := newClient(); err == nil {
var racks []RackController
if err := c.do("GET", "/api/v1/controllers", nil, &racks); err == nil {
regionVer := version
for _, r := range racks {
if r.Kind == "embedded" && r.Version != "" {
regionVer = r.Version // what the region runs (this binary may differ)
}
}
for _, r := range racks {
switch {
case !r.Online:
d.fail("rack "+r.Name, "offline since "+timeStr(r.LastSeen), "on that host: systemctl status rollcall-rack")
case r.Version != regionVer && r.Version != "":
d.warn("rack "+r.Name, "runs "+r.Version+", region "+regionVer, "it updates itself within minutes (rollcall-rack-update.path); see /var/lib/rollcall/update/last-result there")
default:
d.ok("rack "+r.Name, fmt.Sprintf("online, %d machines", r.Machines))
}
}
}
var oss []map[string]any
if err := c.do("GET", "/api/v1/os", nil, &oss); err == nil {
if len(oss) == 0 {
d.warn("os images", "none", "runuser -u rollcall -- rollcall os sync debian-13")
} else {
d.ok("os images", fmt.Sprintf("%d in the catalog", len(oss)))
}
}
var dnsSt map[string]any
if err := c.do("GET", "/api/v1/dns", nil, &dnsSt); err == nil && dnsSt["provider"] != "" && dnsSt["provider"] != nil {
last, _ := dnsSt["last_sync"].(map[string]any)
switch e, _ := last["error"].(string); {
case e != "":
d.fail("dns provider", e, "check dns.* in config.json; rollcall dns sync")
case last["conflicts"] != nil && len(last["conflicts"].([]any)) > 0:
d.warn("dns provider", fmt.Sprintf("%v: %d names held by someone else", dnsSt["provider"], len(last["conflicts"].([]any))), "rollcall dns: those records are left alone")
default:
d.ok("dns provider", fmt.Sprintf("%v, zone %v, in sync", dnsSt["provider"], dnsSt["zone"]))
}
}
var ipam map[string]any
if err := c.do("GET", "/api/v1/ipam", nil, &ipam); err == nil && ipam["provider"] != "" && ipam["provider"] != nil {
last, _ := ipam["last_sync"].(map[string]any)
if e, _ := last["error"].(string); e != "" {
d.fail("ipam", e, "check ipam.url/token_file/ca_file; rollcall ipam sync")
} else {
d.ok("ipam", fmt.Sprintf("%v in sync", ipam["provider"]))
}
}
} else {
d.warn("api", err.Error(), "")
}
return d.done()
}
func (d *doctor) done() error {
fmt.Printf("\n%d failed, %d warnings\n", d.fails, d.warns)
if d.fails > 0 {
os.Exit(1)
}
return nil
}
func errOrStatus(err error, resp *http.Response) string {
if err != nil {
return err.Error()
}
return resp.Status
}
func urlHost(u string) (string, error) {
u = strings.TrimPrefix(strings.TrimPrefix(u, "http://"), "https://")
u = strings.SplitN(u, "/", 2)[0]
if h, _, err := net.SplitHostPort(u); err == nil {
return strings.Trim(h, "[]"), nil
}
return strings.Trim(u, "[]"), nil
}
// ifaceWithAddr: the interface holding ip on this host, "" if none.
func ifaceWithAddr(ip string) string {
want := net.ParseIP(ip)
if want == nil {
return ""
}
ifs, _ := net.Interfaces()
for _, i := range ifs {
addrs, _ := i.Addrs()
for _, a := range addrs {
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(want) {
return i.Name
}
}
}
return ""
}
func portOf(listen string, def int) int {
if listen == "" {
return 0
}
_, p, err := net.SplitHostPort(listen)
if err != nil {
return def
}
n, _ := strconv.Atoi(p)
return n
}
func localPort(listen string) string {
if p := portOf(listen, 0); p != 0 && p != 80 && p != 443 {
return ":" + strconv.Itoa(p)
}
if p := portOf(listen, 0); p == 443 {
return ":443"
}
return ""
}
// listeningPorts reads /proc/net: "tcp/80", "udp/67" ...
func listeningPorts() map[string]bool {
out := map[string]bool{}
for _, f := range []string{"tcp", "tcp6", "udp", "udp6"} {
fh, err := os.Open("/proc/net/" + f)
if err != nil {
continue
}
sc := bufio.NewScanner(fh)
sc.Scan()
for sc.Scan() {
fl := strings.Fields(sc.Text())
if len(fl) < 4 {
continue
}
if strings.HasPrefix(f, "tcp") && fl[3] != "0A" { // LISTEN
continue
}
i := strings.LastIndex(fl[1], ":")
p, err := strconv.ParseUint(fl[1][i+1:], 16, 16)
if err == nil {
out[strings.TrimSuffix(f, "6")+"/"+strconv.Itoa(int(p))] = true
}
}
fh.Close()
}
return out
}
func verifySums(dir string) string {
b, err := os.ReadFile(filepath.Join(dir, "SHA256SUMS"))
if err != nil {
return "no SHA256SUMS next to the image"
}
for _, line := range strings.Split(string(b), "\n") {
f := strings.Fields(line)
if len(f) != 2 {
continue
}
name := strings.TrimPrefix(f[1], "*")
fh, err := os.Open(filepath.Join(dir, name))
if err != nil {
return name + ": " + err.Error()
}
h := sha256.New()
_, _ = io.Copy(h, fh)
fh.Close()
if hex.EncodeToString(h.Sum(nil)) != f[0] {
return name + ": checksum mismatch"
}
}
return ""
}
// dhcpProbe broadcasts a DHCPDISCOVER on iface and returns the server
// identifiers that offer an address. Needs root (port 68, SO_BINDTODEVICE).
func dhcpProbe(iface string, wait time.Duration) ([]string, error) {
lc := net.ListenConfig{Control: func(_, _ string, c syscall.RawConn) error {
var serr error
err := c.Control(func(fd uintptr) {
serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)
if serr == nil {
serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
}
if serr == nil {
serr = bindToDevice(int(fd), iface)
}
})
if err != nil {
return err
}
return serr
}}
pc, err := lc.ListenPacket(context.Background(), "udp4", "0.0.0.0:68")
if err != nil {
return nil, fmt.Errorf("cannot listen on udp/68: %v", err)
}
defer pc.Close()
if _, err := net.InterfaceByName(iface); err != nil {
return nil, err
}
// a locally administered MAC so the probe never takes a real host's lease
mac := make([]byte, 6)
_, _ = rand.Read(mac)
mac[0] = (mac[0] | 0x02) & 0xfe
xid := make([]byte, 4)
_, _ = rand.Read(xid)
p := make([]byte, 240)
p[0], p[1], p[2] = 1, 1, 6 // BOOTREQUEST, ethernet, hlen
copy(p[4:8], xid)
binary.BigEndian.PutUint16(p[10:12], 0x8000) // broadcast reply
copy(p[28:34], mac)
copy(p[236:240], []byte{99, 130, 83, 99})
p = append(p, 53, 1, 1, 55, 3, 1, 3, 6, 255) // DISCOVER; ask router, DNS
if _, err := pc.WriteTo(p, &net.UDPAddr{IP: net.IPv4bcast, Port: 67}); err != nil {
return nil, err
}
seen := map[string]bool{}
var out []string
_ = pc.SetReadDeadline(time.Now().Add(wait))
buf := make([]byte, 1500)
for {
n, _, err := pc.ReadFrom(buf)
if err != nil {
break
}
r := buf[:n]
if n < 240 || r[0] != 2 || string(r[4:8]) != string(xid) {
continue
}
srv := ""
for i := 240; i < n; {
o := r[i]
if o == 255 {
break
}
if o == 0 {
i++
continue
}
if i+1 >= n {
break
}
l := int(r[i+1])
if o == 54 && l == 4 && i+6 <= n {
srv = net.IP(r[i+2 : i+6]).String()
}
i += 2 + l
}
if srv == "" {
srv = net.IP(r[20:24]).String() // siaddr
}
if !seen[srv] {
seen[srv] = true
out = append(out, srv)
}
}
return out, nil
}