Files
rollcall/cmd/rollcall/rackctl.go
T
Ilker ManapandClaude Opus 5.5 a9ca326dd2
CI / agent (push) Successful in 3s
CI / site (push) Successful in 2s
CI / go (push) Failing after 20s
Agents over HTTPS: rack server certificates, fingerprinted CA
Rack controllers ask the region for a server certificate for their own
URL's host (POST /api/v1/rack/server-cert; CN rack-server:NAME,
ServerAuth only, never one of the region's names or addresses) and serve
the agents' endpoints and images over HTTPS too (tls_listen, default
<host>:443), reporting the HTTPS URL when they register (accepted only
on the rack's own host). The pxelinux command lines name it with the CA
fingerprint (rollcall.tls=, rollcall.ca=); the embedded rack uses the
region's listener. Agents fetch /ca.pem, accept it only by that
fingerprint and then use HTTPS with it, including the image download; a
job fetched over HTTPS gets an HTTPS image URL. No fallback to HTTP once
TLS was asked for; old agents ignore the new words. The CLI refuses a
rack controller's certificate as the region.

PXE itself (TFTP, kernel, initrd) stays unauthenticated: this protects
what agents send and receive from listeners and from an HTTP server
posing as the rack, not from an attacker rewriting boot files.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 22:51:17 +01:00

972 lines
29 KiB
Go

package main
// Region side of the region/rack split, after MAAS. The region (rollcall
// serve) owns the database and computes everything; rack controllers sit next
// to the machines and do what has to happen on the machines' networks: DHCP
// and TFTP (dnsmasq), the per-MAC PXE files, BMC commands on a BMC network the
// region may not reach, and serving boot and OS images from a local cache.
//
// A rack controller pulls: it long-polls its desired state (PXE files,
// dnsmasq configuration, host files) and a queue of commands (ipmitool), so
// only rack → region connectivity is needed. The region host itself runs an
// embedded rack controller that applies the same state in-process, which is
// what a single-server installation uses.
import (
"context"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
"path/filepath"
"regexp"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"
)
type RackIface struct {
Name string `json:"name"`
MAC string `json:"mac"`
Addrs []string `json:"addrs"`
}
type RackController struct {
Name string `json:"name"`
URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1
TLSURL string `json:"tls_url,omitempty"` // the same host over HTTPS, when it serves it (agents)
Kind string `json:"kind"` // embedded | remote
Version string `json:"version"`
Arch string `json:"arch,omitempty"` // GOARCH, for self-update
Interfaces []RackIface `json:"interfaces"`
DHCPInterfaces []string `json:"dhcp_interfaces"`
TFTPRoot string `json:"tftp_root"`
LastSeen *time.Time `json:"last_seen"`
Registered time.Time `json:"registered"`
Online bool `json:"online"`
Machines int `json:"machines"`
}
// rackState is everything a rack controller writes to disk.
type rackState struct {
Gen int64 `json:"gen"`
DefaultPXE string `json:"default_pxe"`
PXE map[string]string `json:"pxe"` // pxelinux.cfg file name (01-aa-bb-…) → content
DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here
HostsDHCP string `json:"hosts_dhcp"`
Hosts string `json:"hosts"`
BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP
Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run
}
type rackJob struct {
ID string `json:"id"`
Kind string `json:"kind"` // power
Driver string `json:"driver"`
Address string `json:"address"`
Port int `json:"port"`
User string `json:"user"`
Password string `json:"password"`
Extra map[string]string `json:"extra"`
Op string `json:"op"`
Arg string `json:"arg"`
Timeout int `json:"timeout,omitempty"` // seconds the region waits (default 90)
done chan rackResult
rack string // who took it: only that rack may answer
}
type rackResult struct {
ID string `json:"id"`
Output string `json:"output"`
Error string `json:"error"`
}
// ---------------------------------------------------------------- store
func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
if err != nil {
return nil, err
}
defer rows.Close()
out := []RackController{}
for rows.Next() {
var r RackController
var ifs, dif []byte
if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
return nil, err
}
_ = json.Unmarshal(ifs, &r.Interfaces)
_ = json.Unmarshal(dif, &r.DHCPInterfaces)
out = append(out, r)
}
return out, rows.Err()
}
func (s *Store) PutRackController(ctx context.Context, r RackController) error {
if r.Interfaces == nil {
r.Interfaces = []RackIface{}
}
if r.DHCPInterfaces == nil {
r.DHCPInterfaces = []string{}
}
ifs, _ := json.Marshal(r.Interfaces)
dif, _ := json.Marshal(r.DHCPInterfaces)
_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch, tls_url) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''),nullif($9,''))
ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,''), tls_url=nullif($9,'')`,
r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch, r.TLSURL)
return err
}
func (s *Store) TouchRackController(ctx context.Context, name string) error {
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET last_seen=now() WHERE name=$1`, name)
return err
}
func (s *Store) DeleteRackController(ctx context.Context, name string) error {
_, err := s.pool.Exec(ctx, `DELETE FROM rack_controllers WHERE name=$1`, name)
return err
}
func (s *Store) SetRackController(ctx context.Context, id, rack string) error {
_, err := s.pool.Exec(ctx, `UPDATE machines SET rack_controller=nullif($2,'') WHERE id=$1`, id, rack)
return err
}
func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error {
// a MAC another machine owns is not taken over: a box claiming a deployed
// machine's MAC would otherwise replace that machine's boot file
tag, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4)
ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now() WHERE pxe_files.machine_id=$2`, mac, machineID, rack, content)
if err == nil && tag.RowsAffected() == 0 {
return fmt.Errorf("MAC %s belongs to another machine; delete that machine first", mac)
}
return err
}
func (s *Store) DeletePXE(ctx context.Context, mac string) error {
_, err := s.pool.Exec(ctx, `DELETE FROM pxe_files WHERE mac=$1`, mac)
return err
}
func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, error) {
rows, err := s.pool.Query(ctx, `SELECT mac, content FROM pxe_files WHERE rack=$1`, rack)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]string{}
for rows.Next() {
var mac, content string
if err := rows.Scan(&mac, &content); err != nil {
return nil, err
}
out[pxeFileName(mac)] = content
}
return out, rows.Err()
}
var pxeNameRe = regexp.MustCompile(`^01(-[0-9a-f]{2}){6,20}$`)
func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") }
// ---------------------------------------------------------------- hub: state generations and the command queue
type rackHub struct {
mu sync.Mutex
gen int64
changed chan struct{}
work map[string][]*rackJob
wake map[string]chan struct{}
flight map[string]*rackJob
kick chan struct{} // the embedded rack applies on this
}
func newRackHub() *rackHub {
return &rackHub{gen: time.Now().UnixNano(), changed: make(chan struct{}), work: map[string][]*rackJob{},
wake: map[string]chan struct{}{}, flight: map[string]*rackJob{}, kick: make(chan struct{}, 1)}
}
func (h *rackHub) current() int64 {
h.mu.Lock()
defer h.mu.Unlock()
return h.gen
}
// bump says the desired state of some rack changed.
func (h *rackHub) bump() {
h.mu.Lock()
h.gen++
close(h.changed)
h.changed = make(chan struct{})
h.mu.Unlock()
select {
case h.kick <- struct{}{}:
default:
}
}
// waitChange blocks until the generation moves past since, or the wait is over.
func (h *rackHub) waitChange(ctx context.Context, since int64, wait time.Duration) int64 {
t := time.NewTimer(wait)
defer t.Stop()
for {
h.mu.Lock()
g, ch := h.gen, h.changed
h.mu.Unlock()
if g != since {
return g
}
select {
case <-ch:
case <-t.C:
return g
case <-ctx.Done():
return g
}
}
}
func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string, error) {
j.ID = randomToken(8)
j.done = make(chan rackResult, 1)
h.mu.Lock()
h.work[rack] = append(h.work[rack], j)
if w, ok := h.wake[rack]; ok {
close(w)
delete(h.wake, rack)
}
h.mu.Unlock()
wait := 90 * time.Second
if j.Timeout > 0 {
wait = time.Duration(j.Timeout) * time.Second
}
t := time.NewTimer(wait)
defer t.Stop()
select {
case r := <-j.done:
if r.Error != "" {
return r.Output, errors.New(r.Error)
}
return r.Output, nil
case <-t.C:
case <-ctx.Done():
}
h.mu.Lock()
delete(h.flight, j.ID)
q := h.work[rack][:0]
for _, x := range h.work[rack] {
if x != j {
q = append(q, x)
}
}
h.work[rack] = q
h.mu.Unlock()
return "", fmt.Errorf("rack controller %s did not answer in time", rack)
}
func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*rackJob {
deadline := time.Now().Add(wait)
for {
h.mu.Lock()
if q := h.work[rack]; len(q) > 0 {
h.work[rack] = nil
for _, j := range q {
j.rack = rack
h.flight[j.ID] = j
}
h.mu.Unlock()
return q
}
w, ok := h.wake[rack]
if !ok {
w = make(chan struct{})
h.wake[rack] = w
}
h.mu.Unlock()
left := time.Until(deadline)
if left <= 0 {
return nil
}
t := time.NewTimer(left)
select {
case <-w:
case <-t.C:
case <-ctx.Done():
t.Stop()
return nil
}
t.Stop()
}
}
func (h *rackHub) finish(rack string, r rackResult) bool {
h.mu.Lock()
j, ok := h.flight[r.ID]
if ok && j.rack != rack {
ok = false
} else {
delete(h.flight, r.ID)
}
h.mu.Unlock()
if ok {
j.done <- r
}
return ok
}
// ---------------------------------------------------------------- routing machines to racks
func (a *App) racksSnapshot() map[string]RackController {
a.netMu.RLock()
defer a.netMu.RUnlock()
out := make(map[string]RackController, len(a.racks))
for k, v := range a.racks {
out[k] = v
}
return out
}
func (a *App) subnetsSnapshot() []Subnet {
a.netMu.RLock()
defer a.netMu.RUnlock()
return a.subnets
}
// reloadNet refreshes the cached subnets and rack controllers after a change.
func (a *App) reloadNet(ctx context.Context) {
if sn, err := a.store.Subnets(ctx); err == nil {
a.netMu.Lock()
a.subnets = sn
a.netMu.Unlock()
}
a.reloadRacks(ctx)
}
func (a *App) reloadRacks(ctx context.Context) {
rs, err := a.store.RackControllers(ctx)
if err != nil {
log.Printf("racks: %v", err)
return
}
m := map[string]RackController{}
for _, r := range rs {
m[r.Name] = r
}
a.netMu.Lock()
a.racks = m
a.netMu.Unlock()
}
func rackOnline(r RackController, local string) bool {
return r.Name == local || (r.LastSeen != nil && time.Since(*r.LastSeen) < 90*time.Second)
}
// rackOf names the rack controller serving a machine: the one it last came
// through, else the one serving the VLAN of its management or boot address,
// else the embedded one.
func (a *App) rackOf(m *Machine) string {
racks := a.racksSnapshot()
if m.RackController != "" {
if _, ok := racks[m.RackController]; ok {
return m.RackController
}
}
if subnets := a.subnetsSnapshot(); len(subnets) > 0 {
ips := []string{deref(m.MgmtIP)}
if b := bootNIC(machineNICs(m), m.MACs); b != nil {
ips = append(ips, b.IPv4...)
}
for _, ip := range ips {
if sn := subnetFor(subnets, ip); sn != nil && sn.PrimaryRack != "" {
if _, ok := racks[sn.PrimaryRack]; ok {
return sn.PrimaryRack
}
}
}
}
return a.cfg.RackName
}
func (a *App) rackURL(name string) string {
if name == "" || name == a.cfg.RackName {
return a.cfg.ServerURL
}
if r, ok := a.racksSnapshot()[name]; ok && r.URL != "" {
return strings.TrimRight(r.URL, "/")
}
return a.cfg.ServerURL
}
// urlFor is the base URL a machine uses for images and the node API.
func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) }
// appendFor is commission_append with the server URL of the machine's rack.
func (a *App) appendFor(base string) string {
s := a.cfg.CommissionAppend
if base != a.cfg.ServerURL {
s = strings.ReplaceAll(s, a.cfg.ServerURL, base)
}
if t := a.agentTLSArgs(base); len(t) > 0 && !strings.Contains(s, "rollcall.tls=") {
s += " " + strings.Join(t, " ")
}
return s
}
// rackAddresses are the rack controllers' own addresses, never handed to machines.
func (a *App) rackAddresses() []string {
var out []string
for _, r := range a.racksSnapshot() {
for _, i := range r.Interfaces {
out = append(out, i.Addrs...)
}
}
return out
}
// ---------------------------------------------------------------- BMC through the machine's rack
// ---------------------------------------------------------------- desired state
const defaultPXETmpl = `# generated by rollcall for rack controller %s
# unknown machines land here; known ones get a per-MAC file (01-xx-xx-…)
DEFAULT menu.c32
PROMPT 0
TIMEOUT 80
MENU TITLE rollcall · %s · unknown machine: commissioning in 8s
LABEL commission
MENU LABEL Commission this machine (inventory + BMC account)
MENU DEFAULT
KERNEL rollcall/vmlinuz
INITRD rollcall/initrd.img
APPEND %s
LABEL commission-debug
MENU LABEL Commission, but stay up afterwards (ssh root@<ip>)
KERNEL rollcall/vmlinuz
INITRD rollcall/initrd.img
APPEND %s rollcall.nopoweroff
LABEL local
MENU LABEL Boot from local disk
LOCALBOOT 0
`
func (a *App) rackState(ctx context.Context, name string) (*rackState, error) {
gen := a.hub.current()
st := &rackState{Gen: gen}
if rc, ok := a.racksSnapshot()[name]; ok {
st.Update = a.rackUpdateFor(rc)
}
base := a.rackURL(name)
app := a.appendFor(base)
st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app)
var err error
if st.PXE, err = a.store.PXEFiles(ctx, name); err != nil {
return nil, err
}
subnets, err := a.store.Subnets(ctx)
if err != nil {
return nil, err
}
rc := a.racksSnapshot()[name]
rc.Name = name
dnsDir := a.cfg.DnsmasqDir
if dnsDir == "" {
dnsDir = "/var/lib/rollcall/dnsmasq"
}
if conf, ok := dnsmasqConfig(rc, subnets, dnsDir); ok {
st.DnsmasqConf = conf
}
if st.HostsDHCP, st.Hosts, err = a.hostsFiles(ctx, name, subnets); err != nil {
return nil, err
}
st.BootFiles = []string{"rollcall/vmlinuz", "rollcall/initrd.img"}
if ents, err := os.ReadDir(a.cfg.ImagesDir); err == nil {
for _, e := range ents {
if e.IsDir() {
st.BootFiles = append(st.BootFiles, "images/"+e.Name()+"/vmlinuz", "images/"+e.Name()+"/initrd.img")
}
}
}
return st, nil
}
// hostsFiles: static DHCP leases for the machines this rack serves, and names
// for every machine: the addresses on the interface carrying the management
// address get the plain name (A and AAAA), others name-<interface>, IB name-ib.
var nonHostChars = regexp.MustCompile(`[^a-z0-9-]+`)
func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) {
ms, err := a.store.List(ctx)
if err != nil {
return "", "", err
}
all, _ := a.store.IPs(ctx, "", "")
byMachine := map[string][]IPAddr{}
for _, x := range all {
if x.MachineID != "" {
byMachine[x.MachineID] = append(byMachine[x.MachineID], x)
}
}
var dhcp, hosts strings.Builder
dhcp.WriteString("# generated by rollcall\n")
hosts.WriteString("# generated by rollcall\n")
domainOf := func(ip string) string {
if sn := subnetFor(subnets, ip); sn != nil && sn.Domain != "" {
return sn.Domain
}
return str(a.cfg.ClusterVars["domain"])
}
line := func(ip, name string) {
if d := domainOf(ip); d != "" {
fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name)
} else {
fmt.Fprintf(&hosts, "%s %s\n", ip, name)
}
}
// proxy subnets: the site's DHCP server hands out the addresses, so no dhcp-host
// lines for them (they would do nothing, and the "known" tag they give a client
// helps dnsmasq 2.91 build a cyclic tag list in its PXE proxy code: busy loop)
proxied := func(ip string) bool {
sn := subnetFor(subnets, ip)
return sn != nil && sn.DHCPMode == "proxy"
}
sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) })
for _, m := range ms {
if m.NodeName == nil || len(m.MACs) == 0 {
continue
}
name, mgmt, ibip := *m.NodeName, deref(m.MgmtIP), deref(m.IBIP)
mine := a.rackOf(m) == rack
mgmtIface := ""
for _, x := range byMachine[m.ID] {
if x.IP == mgmt {
mgmtIface = x.Iface
}
}
if mgmt != "" {
line(mgmt, name)
if mine && familyOf(mgmt+"/32") == 4 && !proxied(mgmt) {
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], mgmt, name)
}
}
for _, x := range byMachine[m.ID] {
if x.IP == mgmt || x.IP == ibip {
continue
}
v6 := strings.Contains(x.IP, ":")
switch {
case x.Iface == mgmtIface && mgmtIface != "":
line(x.IP, name)
if v6 && mine && !proxied(x.IP) {
fmt.Fprintf(&dhcp, "%s,[%s],%s\n", m.MACs[0], x.IP, name)
}
case x.Iface != "":
if sfx := strings.Trim(nonHostChars.ReplaceAllString(strings.ToLower(x.Iface), "-"), "-"); sfx != "" {
line(x.IP, name+"-"+sfx)
}
}
}
if ibip != "" {
fmt.Fprintf(&hosts, "%s %s-ib\n", ibip, name)
}
}
return dhcp.String(), hosts.String(), nil
}
// writeIfChanged replaces a file only when its content differs (path units watch these).
func writeIfChanged(p, content string, mode os.FileMode) (bool, error) {
if old, err := os.ReadFile(p); err == nil && string(old) == content {
return false, nil
}
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
return false, err
}
if err := os.WriteFile(p+".tmp", []byte(content), mode); err != nil {
return false, err
}
return true, os.Rename(p+".tmp", p)
}
// applyRackState writes a rack's state: PXE files under tftp/pxelinux.cfg
// (stale per-MAC files removed), the dnsmasq configuration and host files.
func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err error) {
dir := filepath.Join(tftp, "pxelinux.cfg")
note := func(p string, ch bool, e error) {
if e != nil && err == nil {
err = e
}
if ch {
changed = append(changed, p)
}
}
if st.DefaultPXE != "" {
ch, e := writeIfChanged(filepath.Join(dir, "default"), st.DefaultPXE, 0o644)
note("default", ch, e)
}
for name, body := range st.PXE {
if !pxeNameRe.MatchString(name) {
note(name, false, fmt.Errorf("not a pxelinux file name"))
continue
}
ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644)
note(name, ch, e)
}
if olds, e := filepath.Glob(filepath.Join(dir, "01-*")); e == nil {
for _, p := range olds {
if _, keep := st.PXE[filepath.Base(p)]; !keep && !strings.HasSuffix(p, ".tmp") {
note("-"+filepath.Base(p), true, os.Remove(p))
}
}
}
if dnsDir != "" {
ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644)
note("hosts.dhcp", ch, e)
ch, e = writeIfChanged(filepath.Join(dnsDir, "hosts"), st.Hosts, 0o644)
note("hosts", ch, e)
conf := filepath.Join(dnsDir, "conf.d", "rollcall.conf")
if st.DnsmasqConf != "" {
ch, e = writeIfChanged(conf, st.DnsmasqConf+dnsmasqUpstream("/run/systemd/resolve/stub-resolv.conf", "/etc/resolv.conf", localAddrs()), 0o644)
note("dnsmasq.conf", ch, e)
} else if _, e := os.Stat(conf); e == nil {
note("-dnsmasq.conf", true, os.Remove(conf))
}
}
return changed, err
}
// localRackLoop is the embedded rack controller: it applies this host's state on every change.
func (a *App) localRackLoop() {
apply := func() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
st, err := a.rackState(ctx, a.cfg.RackName)
if err != nil {
log.Printf("embedded rack: %v", err)
return
}
if ch, err := applyRackState(st, a.cfg.TFTPRoot, a.cfg.DnsmasqDir); err != nil {
log.Printf("embedded rack: %v", err)
} else if len(ch) > 0 {
log.Printf("embedded rack %s: wrote %s", a.cfg.RackName, strings.Join(ch, " "))
}
_ = a.store.TouchRackController(ctx, a.cfg.RackName)
}
apply()
tick := time.NewTicker(60 * time.Second)
for {
select {
case <-a.hub.kick:
time.Sleep(150 * time.Millisecond) // let a burst of changes settle
select {
case <-a.hub.kick:
default:
}
case <-tick.C:
}
apply()
}
}
func localInterfaces() []RackIface {
var out []RackIface
ifs, _ := net.Interfaces()
for _, i := range ifs {
if i.Flags&net.FlagLoopback != 0 {
continue
}
ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()}
addrs, _ := i.Addrs()
for _, ad := range addrs {
if ipn, ok := ad.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() {
ri.Addrs = append(ri.Addrs, ipn.String())
}
}
if len(ri.Addrs) > 0 {
out = append(out, ri)
}
}
return out
}
// startRacks registers the embedded rack controller, carries over what an
// older version kept on disk or in config, and starts the local apply loop.
func (a *App) startRacks(ctx context.Context) error {
if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, Arch: runtime.GOARCH,
Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil {
return err
}
a.reloadNet(ctx)
// per-MAC files written before pxe_files existed
var n int
_ = a.store.pool.QueryRow(ctx, `SELECT count(*) FROM pxe_files`).Scan(&n)
if n == 0 {
files, _ := filepath.Glob(filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-*"))
ms, _ := a.store.List(ctx)
for _, f := range files {
b, err := os.ReadFile(f)
if err != nil {
continue
}
mac := strings.ReplaceAll(strings.TrimPrefix(filepath.Base(f), "01-"), "-", ":")
owner := ""
for _, m := range ms {
if hasTag(m.MACs, mac) {
owner = m.ID
}
}
_ = a.store.PutPXE(ctx, mac, owner, a.cfg.RackName, string(b))
}
if len(files) > 0 {
log.Printf("imported %d existing PXE files", len(files))
}
}
a.adoptIPs(ctx)
go a.localRackLoop()
return nil
}
// adoptIPs records addresses machines already have (node pools, before a
// subnet was modeled) so IPAM never hands them out twice.
func (a *App) adoptIPs(ctx context.Context) {
subnets, err := a.store.Subnets(ctx)
if err != nil || len(subnets) == 0 {
return
}
ms, _ := a.store.List(ctx)
for _, m := range ms {
for _, ip := range []string{deref(m.MgmtIP), deref(m.IBIP)} {
if sn := subnetFor(subnets, ip); sn != nil {
iface := ""
if b := bootNIC(machineNICs(m), m.MACs); b != nil && ip == deref(m.MgmtIP) {
iface = b.Name
}
_, _ = a.claimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: m.ID, Iface: iface, Kind: "auto", Hostname: deref(m.NodeName)})
}
}
}
}
// ---------------------------------------------------------------- PXE files through the rack
// writePXEBody records a machine's PXE file for its rack controller.
func (a *App) writePXEBody(m *Machine, body string) error {
rack := a.rackOf(m)
for _, mac := range m.MACs {
if err := a.store.PutPXE(context.Background(), normMAC(mac), m.ID, rack, body); err != nil {
return err
}
}
a.hub.bump()
return nil
}
func (a *App) removePXE(m *Machine) {
for _, mac := range m.MACs {
_ = a.store.DeletePXE(context.Background(), normMAC(mac))
}
a.hub.bump()
}
// exportDnsmasq: host files and dnsmasq configuration follow from the state now.
func (a *App) exportDnsmasq(ctx context.Context) error {
a.hub.bump()
return nil
}
// ---------------------------------------------------------------- HTTP: rack controllers
func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if name := a.peerRack(r); name != "" {
next(w, r, name)
return
}
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" {
fail(w, 401, errors.New("rack controller certificate (or legacy secret) required"))
return
}
next(w, r, name)
}
}
// fromRack names the rack controller a node request came through (it proxies them).
func (a *App) fromRack(r *http.Request) string {
if name := a.peerRack(r); name != "" {
return name
}
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 {
return name
}
return a.cfg.RackName
}
func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack string) {
var rc RackController
if err := json.NewDecoder(r.Body).Decode(&rc); err != nil {
fail(w, 400, err)
return
}
rc.Name, rc.Kind = rack, "remote"
if rack == a.cfg.RackName {
fail(w, 409, fmt.Errorf("%s is the embedded rack controller's name", rack))
return
}
if rc.URL == "" {
fail(w, 400, errors.New("url is required"))
return
}
rc.TLSURL = validTLSURL(rc) // its own host only
if err := a.store.PutRackController(r.Context(), rc); err != nil {
fail(w, 500, err)
return
}
a.reloadRacks(r.Context())
a.hub.bump()
log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces))
writeJSON(w, 200, obj{"ok": true, "gen": a.hub.current()})
}
func (a *App) handleRackState(w http.ResponseWriter, r *http.Request, rack string) {
if _, ok := a.racksSnapshot()[rack]; !ok {
fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack))
return
}
since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
_ = a.store.TouchRackController(r.Context(), rack)
a.hub.waitChange(r.Context(), since, 25*time.Second)
_ = a.store.TouchRackController(r.Context(), rack)
go a.reloadRacks(context.Background()) // keep last_seen fresh for online checks
st, err := a.rackState(r.Context(), rack)
if err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, st)
}
func (a *App) handleRackWork(w http.ResponseWriter, r *http.Request, rack string) {
jobs := a.hub.take(r.Context(), rack, 25*time.Second)
if jobs == nil {
jobs = []*rackJob{}
}
writeJSON(w, 200, jobs)
}
func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack string) {
var res rackResult
if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
fail(w, 400, err)
return
}
writeJSON(w, 200, obj{"accepted": a.hub.finish(rack, res)})
}
func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) {
rs, err := a.store.RackControllers(r.Context())
if err != nil {
fail(w, 500, err)
return
}
ms, _ := a.store.List(r.Context())
for i := range rs {
rs[i].Online = rackOnline(rs[i], a.cfg.RackName)
for _, m := range ms {
if a.rackOf(m) == rs[i].Name {
rs[i].Machines++
}
}
}
writeJSON(w, 200, rs)
}
func (a *App) handleControllerDelete(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("name")
if name == a.cfg.RackName {
fail(w, 409, errors.New("the embedded rack controller cannot be removed"))
return
}
if err := a.store.DeleteRackController(r.Context(), name); err != nil {
fail(w, 500, err)
return
}
a.reloadRacks(r.Context())
a.hub.bump()
writeJSON(w, 200, obj{"deleted": name})
}
// handleRackDHCP shows a rack's generated dnsmasq configuration (admin).
func (a *App) handleRackDHCP(w http.ResponseWriter, r *http.Request) {
st, err := a.rackState(r.Context(), r.PathValue("name"))
if err != nil {
fail(w, 500, err)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
if st.DnsmasqConf == "" {
fmt.Fprintf(w, "# rack controller %s serves no VLAN with dhcp on: rollcall leaves dnsmasq's configuration alone\n", r.PathValue("name"))
}
fmt.Fprint(w, st.DnsmasqConf, "\n# --- hosts.dhcp\n", st.HostsDHCP)
}
// dnsmasqUpstream tells dnsmasq where to forward names it does not answer
// itself. Without it Debian's dnsmasq looks for a resolvconf-made file that
// usually does not exist and refuses every outside name, so nodes could not
// resolve mirrors or site services. With systemd-resolved the stub is used, so
// its per-link routing (split DNS such as ~int.example.com) keeps working;
// otherwise the host's resolv.conf servers, never ourselves.
func dnsmasqUpstream(stub, resolv string, own []string) string {
if _, err := os.Stat(stub); err == nil {
return "# upstream: systemd-resolved on this host (keeps its split DNS)\nno-resolv\nserver=127.0.0.53\n"
}
b, err := os.ReadFile(resolv)
if err != nil {
return ""
}
mine := map[string]bool{}
for _, a := range own {
mine[stripPrefix(a)] = true
}
var servers []string
for _, line := range strings.Split(string(b), "\n") {
f := strings.Fields(line)
if len(f) < 2 || f[0] != "nameserver" {
continue
}
ip, ok := parseAddr(f[1])
if !ok || mine[ip.String()] || (ip.IsLoopback() && ip.String() != "127.0.0.53") {
continue
}
servers = append(servers, ip.String())
}
if len(servers) == 0 {
return ""
}
out := "# upstream: this host's resolv.conf\nno-resolv\n"
for _, sv := range servers {
out += "server=" + sv + "\n"
}
return out
}
// localAddrs: this host's interface addresses (CIDR form).
func localAddrs() []string {
var out []string
addrs, _ := net.InterfaceAddrs()
for _, a := range addrs {
out = append(out, a.String())
}
return out
}