Files
rollcall/cmd/rollcall/rackctl.go
T
Ilker ManapandClaude Opus 5.5 7a007ef048
CI / agent (push) Successful in 3s
CI / site (push) Successful in 3s
CI / go (push) Successful in 36s
Proxy DHCP for networks whose DHCP server stays in charge
A VLAN in proxy mode leaves addresses to the site's DHCP server; the rack
only answers PXE (dnsmasq proxy range + pxe-service), machines keep DHCP
unless a static range is set aside. CLI -proxy-dhcp, web UI mode select,
bootstrap --proxy-dhcp, doctor/discover aware.

dnsmasq 2.91 (and 2.93) loops forever in its proxy path when the proxy
range carries a set: tag: option_filter appends a request's tags to the
context's persistent list, so the next request's stack interface tag
points at itself (found with gdb on the lab rack). Proxy ranges are
untagged and PXE services limited by interface; dhcp-boot/dhcp-match only
for served subnets; no dhcp-host lines for proxied ones.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-29 09:04:10 +01:00

939 lines
28 KiB
Go

package main
// Region side of the region/rack split, after MAAS. The region (rollcall
// serve) owns the database and computes everything; rack controllers sit next
// to the machines and do what has to happen on the machines' networks: DHCP
// and TFTP (dnsmasq), the per-MAC PXE files, BMC commands on a BMC network the
// region may not reach, and serving boot and OS images from a local cache.
//
// A rack controller pulls: it long-polls its desired state (PXE files,
// dnsmasq configuration, host files) and a queue of commands (ipmitool), so
// only rack → region connectivity is needed. The region host itself runs an
// embedded rack controller that applies the same state in-process, which is
// what a single-server installation uses.
import (
"context"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"sync"
"time"
)
type RackIface struct {
Name string `json:"name"`
MAC string `json:"mac"`
Addrs []string `json:"addrs"`
}
type RackController struct {
Name string `json:"name"`
URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1
Kind string `json:"kind"` // embedded | remote
Version string `json:"version"`
Arch string `json:"arch,omitempty"` // GOARCH, for self-update
Interfaces []RackIface `json:"interfaces"`
DHCPInterfaces []string `json:"dhcp_interfaces"`
TFTPRoot string `json:"tftp_root"`
LastSeen *time.Time `json:"last_seen"`
Registered time.Time `json:"registered"`
Online bool `json:"online"`
Machines int `json:"machines"`
}
// rackState is everything a rack controller writes to disk.
type rackState struct {
Gen int64 `json:"gen"`
DefaultPXE string `json:"default_pxe"`
PXE map[string]string `json:"pxe"` // pxelinux.cfg file name (01-aa-bb-…) → content
DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here
HostsDHCP string `json:"hosts_dhcp"`
Hosts string `json:"hosts"`
BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP
Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run
}
type rackJob struct {
ID string `json:"id"`
Kind string `json:"kind"` // power
Driver string `json:"driver"`
Address string `json:"address"`
Port int `json:"port"`
User string `json:"user"`
Password string `json:"password"`
Extra map[string]string `json:"extra"`
Op string `json:"op"`
Arg string `json:"arg"`
done chan rackResult
}
type rackResult struct {
ID string `json:"id"`
Output string `json:"output"`
Error string `json:"error"`
}
// ---------------------------------------------------------------- store
func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
if err != nil {
return nil, err
}
defer rows.Close()
out := []RackController{}
for rows.Next() {
var r RackController
var ifs, dif []byte
if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
return nil, err
}
_ = json.Unmarshal(ifs, &r.Interfaces)
_ = json.Unmarshal(dif, &r.DHCPInterfaces)
out = append(out, r)
}
return out, rows.Err()
}
func (s *Store) PutRackController(ctx context.Context, r RackController) error {
if r.Interfaces == nil {
r.Interfaces = []RackIface{}
}
if r.DHCPInterfaces == nil {
r.DHCPInterfaces = []string{}
}
ifs, _ := json.Marshal(r.Interfaces)
dif, _ := json.Marshal(r.DHCPInterfaces)
_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''))
ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,'')`,
r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch)
return err
}
func (s *Store) TouchRackController(ctx context.Context, name string) error {
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET last_seen=now() WHERE name=$1`, name)
return err
}
func (s *Store) DeleteRackController(ctx context.Context, name string) error {
_, err := s.pool.Exec(ctx, `DELETE FROM rack_controllers WHERE name=$1`, name)
return err
}
func (s *Store) SetRackController(ctx context.Context, id, rack string) error {
_, err := s.pool.Exec(ctx, `UPDATE machines SET rack_controller=nullif($2,'') WHERE id=$1`, id, rack)
return err
}
func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error {
_, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4)
ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now()`, mac, machineID, rack, content)
return err
}
func (s *Store) DeletePXE(ctx context.Context, mac string) error {
_, err := s.pool.Exec(ctx, `DELETE FROM pxe_files WHERE mac=$1`, mac)
return err
}
func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, error) {
rows, err := s.pool.Query(ctx, `SELECT mac, content FROM pxe_files WHERE rack=$1`, rack)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]string{}
for rows.Next() {
var mac, content string
if err := rows.Scan(&mac, &content); err != nil {
return nil, err
}
out[pxeFileName(mac)] = content
}
return out, rows.Err()
}
func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") }
// ---------------------------------------------------------------- hub: state generations and the command queue
type rackHub struct {
mu sync.Mutex
gen int64
changed chan struct{}
work map[string][]*rackJob
wake map[string]chan struct{}
flight map[string]*rackJob
kick chan struct{} // the embedded rack applies on this
}
func newRackHub() *rackHub {
return &rackHub{gen: time.Now().UnixNano(), changed: make(chan struct{}), work: map[string][]*rackJob{},
wake: map[string]chan struct{}{}, flight: map[string]*rackJob{}, kick: make(chan struct{}, 1)}
}
func (h *rackHub) current() int64 {
h.mu.Lock()
defer h.mu.Unlock()
return h.gen
}
// bump says the desired state of some rack changed.
func (h *rackHub) bump() {
h.mu.Lock()
h.gen++
close(h.changed)
h.changed = make(chan struct{})
h.mu.Unlock()
select {
case h.kick <- struct{}{}:
default:
}
}
// waitChange blocks until the generation moves past since, or the wait is over.
func (h *rackHub) waitChange(ctx context.Context, since int64, wait time.Duration) int64 {
t := time.NewTimer(wait)
defer t.Stop()
for {
h.mu.Lock()
g, ch := h.gen, h.changed
h.mu.Unlock()
if g != since {
return g
}
select {
case <-ch:
case <-t.C:
return g
case <-ctx.Done():
return g
}
}
}
func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string, error) {
j.ID = randomToken(8)
j.done = make(chan rackResult, 1)
h.mu.Lock()
h.work[rack] = append(h.work[rack], j)
if w, ok := h.wake[rack]; ok {
close(w)
delete(h.wake, rack)
}
h.mu.Unlock()
t := time.NewTimer(90 * time.Second)
defer t.Stop()
select {
case r := <-j.done:
if r.Error != "" {
return r.Output, errors.New(r.Error)
}
return r.Output, nil
case <-t.C:
case <-ctx.Done():
}
h.mu.Lock()
delete(h.flight, j.ID)
q := h.work[rack][:0]
for _, x := range h.work[rack] {
if x != j {
q = append(q, x)
}
}
h.work[rack] = q
h.mu.Unlock()
return "", fmt.Errorf("rack controller %s did not answer in time", rack)
}
func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*rackJob {
deadline := time.Now().Add(wait)
for {
h.mu.Lock()
if q := h.work[rack]; len(q) > 0 {
h.work[rack] = nil
for _, j := range q {
h.flight[j.ID] = j
}
h.mu.Unlock()
return q
}
w, ok := h.wake[rack]
if !ok {
w = make(chan struct{})
h.wake[rack] = w
}
h.mu.Unlock()
left := time.Until(deadline)
if left <= 0 {
return nil
}
t := time.NewTimer(left)
select {
case <-w:
case <-t.C:
case <-ctx.Done():
t.Stop()
return nil
}
t.Stop()
}
}
func (h *rackHub) finish(r rackResult) bool {
h.mu.Lock()
j, ok := h.flight[r.ID]
delete(h.flight, r.ID)
h.mu.Unlock()
if ok {
j.done <- r
}
return ok
}
// ---------------------------------------------------------------- routing machines to racks
func (a *App) racksSnapshot() map[string]RackController {
a.netMu.RLock()
defer a.netMu.RUnlock()
out := make(map[string]RackController, len(a.racks))
for k, v := range a.racks {
out[k] = v
}
return out
}
func (a *App) subnetsSnapshot() []Subnet {
a.netMu.RLock()
defer a.netMu.RUnlock()
return a.subnets
}
// reloadNet refreshes the cached subnets and rack controllers after a change.
func (a *App) reloadNet(ctx context.Context) {
if sn, err := a.store.Subnets(ctx); err == nil {
a.netMu.Lock()
a.subnets = sn
a.netMu.Unlock()
}
a.reloadRacks(ctx)
}
func (a *App) reloadRacks(ctx context.Context) {
rs, err := a.store.RackControllers(ctx)
if err != nil {
log.Printf("racks: %v", err)
return
}
m := map[string]RackController{}
for _, r := range rs {
m[r.Name] = r
}
a.netMu.Lock()
a.racks = m
a.netMu.Unlock()
}
func rackOnline(r RackController, local string) bool {
return r.Name == local || (r.LastSeen != nil && time.Since(*r.LastSeen) < 90*time.Second)
}
// rackOf names the rack controller serving a machine: the one it last came
// through, else the one serving the VLAN of its management or boot address,
// else the embedded one.
func (a *App) rackOf(m *Machine) string {
racks := a.racksSnapshot()
if m.RackController != "" {
if _, ok := racks[m.RackController]; ok {
return m.RackController
}
}
if subnets := a.subnetsSnapshot(); len(subnets) > 0 {
ips := []string{deref(m.MgmtIP)}
if b := bootNIC(machineNICs(m), m.MACs); b != nil {
ips = append(ips, b.IPv4...)
}
for _, ip := range ips {
if sn := subnetFor(subnets, ip); sn != nil && sn.PrimaryRack != "" {
if _, ok := racks[sn.PrimaryRack]; ok {
return sn.PrimaryRack
}
}
}
}
return a.cfg.RackName
}
func (a *App) rackURL(name string) string {
if name == "" || name == a.cfg.RackName {
return a.cfg.ServerURL
}
if r, ok := a.racksSnapshot()[name]; ok && r.URL != "" {
return strings.TrimRight(r.URL, "/")
}
return a.cfg.ServerURL
}
// urlFor is the base URL a machine uses for images and the node API.
func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) }
// appendFor is commission_append with the server URL of the machine's rack.
func (a *App) appendFor(base string) string {
if base == a.cfg.ServerURL {
return a.cfg.CommissionAppend
}
return strings.ReplaceAll(a.cfg.CommissionAppend, a.cfg.ServerURL, base)
}
// rackAddresses are the rack controllers' own addresses, never handed to machines.
func (a *App) rackAddresses() []string {
var out []string
for _, r := range a.racksSnapshot() {
for _, i := range r.Interfaces {
out = append(out, i.Addrs...)
}
}
return out
}
// ---------------------------------------------------------------- BMC through the machine's rack
// ---------------------------------------------------------------- desired state
const defaultPXETmpl = `# generated by rollcall for rack controller %s
# unknown machines land here; known ones get a per-MAC file (01-xx-xx-…)
DEFAULT menu.c32
PROMPT 0
TIMEOUT 80
MENU TITLE rollcall · %s · unknown machine: commissioning in 8s
LABEL commission
MENU LABEL Commission this machine (inventory + BMC account)
MENU DEFAULT
KERNEL rollcall/vmlinuz
INITRD rollcall/initrd.img
APPEND %s
LABEL commission-debug
MENU LABEL Commission, but stay up afterwards (ssh root@<ip>)
KERNEL rollcall/vmlinuz
INITRD rollcall/initrd.img
APPEND %s rollcall.nopoweroff
LABEL local
MENU LABEL Boot from local disk
LOCALBOOT 0
`
func (a *App) rackState(ctx context.Context, name string) (*rackState, error) {
gen := a.hub.current()
st := &rackState{Gen: gen}
if rc, ok := a.racksSnapshot()[name]; ok {
st.Update = a.rackUpdateFor(rc)
}
base := a.rackURL(name)
app := a.appendFor(base)
st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app)
var err error
if st.PXE, err = a.store.PXEFiles(ctx, name); err != nil {
return nil, err
}
subnets, err := a.store.Subnets(ctx)
if err != nil {
return nil, err
}
rc := a.racksSnapshot()[name]
rc.Name = name
dnsDir := a.cfg.DnsmasqDir
if dnsDir == "" {
dnsDir = "/var/lib/rollcall/dnsmasq"
}
if conf, ok := dnsmasqConfig(rc, subnets, dnsDir); ok {
st.DnsmasqConf = conf
}
if st.HostsDHCP, st.Hosts, err = a.hostsFiles(ctx, name, subnets); err != nil {
return nil, err
}
st.BootFiles = []string{"rollcall/vmlinuz", "rollcall/initrd.img"}
if ents, err := os.ReadDir(a.cfg.ImagesDir); err == nil {
for _, e := range ents {
if e.IsDir() {
st.BootFiles = append(st.BootFiles, "images/"+e.Name()+"/vmlinuz", "images/"+e.Name()+"/initrd.img")
}
}
}
return st, nil
}
// hostsFiles: static DHCP leases for the machines this rack serves, and names
// for every machine: the addresses on the interface carrying the management
// address get the plain name (A and AAAA), others name-<interface>, IB name-ib.
func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) {
ms, err := a.store.List(ctx)
if err != nil {
return "", "", err
}
all, _ := a.store.IPs(ctx, "", "")
byMachine := map[string][]IPAddr{}
for _, x := range all {
if x.MachineID != "" {
byMachine[x.MachineID] = append(byMachine[x.MachineID], x)
}
}
var dhcp, hosts strings.Builder
dhcp.WriteString("# generated by rollcall\n")
hosts.WriteString("# generated by rollcall\n")
domainOf := func(ip string) string {
if sn := subnetFor(subnets, ip); sn != nil && sn.Domain != "" {
return sn.Domain
}
return str(a.cfg.ClusterVars["domain"])
}
line := func(ip, name string) {
if d := domainOf(ip); d != "" {
fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name)
} else {
fmt.Fprintf(&hosts, "%s %s\n", ip, name)
}
}
// proxy subnets: the site's DHCP server hands out the addresses, so no dhcp-host
// lines for them (they would do nothing, and the "known" tag they give a client
// helps dnsmasq 2.91 build a cyclic tag list in its PXE proxy code: busy loop)
proxied := func(ip string) bool {
sn := subnetFor(subnets, ip)
return sn != nil && sn.DHCPMode == "proxy"
}
sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) })
for _, m := range ms {
if m.NodeName == nil || len(m.MACs) == 0 {
continue
}
name, mgmt, ibip := *m.NodeName, deref(m.MgmtIP), deref(m.IBIP)
mine := a.rackOf(m) == rack
mgmtIface := ""
for _, x := range byMachine[m.ID] {
if x.IP == mgmt {
mgmtIface = x.Iface
}
}
if mgmt != "" {
line(mgmt, name)
if mine && familyOf(mgmt+"/32") == 4 && !proxied(mgmt) {
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], mgmt, name)
}
}
for _, x := range byMachine[m.ID] {
if x.IP == mgmt || x.IP == ibip {
continue
}
v6 := strings.Contains(x.IP, ":")
switch {
case x.Iface == mgmtIface && mgmtIface != "":
line(x.IP, name)
if v6 && mine && !proxied(x.IP) {
fmt.Fprintf(&dhcp, "%s,[%s],%s\n", m.MACs[0], x.IP, name)
}
case x.Iface != "":
line(x.IP, name+"-"+strings.NewReplacer(".", "-", ":", "-").Replace(x.Iface))
}
}
if ibip != "" {
fmt.Fprintf(&hosts, "%s %s-ib\n", ibip, name)
}
}
return dhcp.String(), hosts.String(), nil
}
// writeIfChanged replaces a file only when its content differs (path units watch these).
func writeIfChanged(p, content string, mode os.FileMode) (bool, error) {
if old, err := os.ReadFile(p); err == nil && string(old) == content {
return false, nil
}
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
return false, err
}
if err := os.WriteFile(p+".tmp", []byte(content), mode); err != nil {
return false, err
}
return true, os.Rename(p+".tmp", p)
}
// applyRackState writes a rack's state: PXE files under tftp/pxelinux.cfg
// (stale per-MAC files removed), the dnsmasq configuration and host files.
func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err error) {
dir := filepath.Join(tftp, "pxelinux.cfg")
note := func(p string, ch bool, e error) {
if e != nil && err == nil {
err = e
}
if ch {
changed = append(changed, p)
}
}
if st.DefaultPXE != "" {
ch, e := writeIfChanged(filepath.Join(dir, "default"), st.DefaultPXE, 0o644)
note("default", ch, e)
}
for name, body := range st.PXE {
ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644)
note(name, ch, e)
}
if olds, e := filepath.Glob(filepath.Join(dir, "01-*")); e == nil {
for _, p := range olds {
if _, keep := st.PXE[filepath.Base(p)]; !keep && !strings.HasSuffix(p, ".tmp") {
note("-"+filepath.Base(p), true, os.Remove(p))
}
}
}
if dnsDir != "" {
ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644)
note("hosts.dhcp", ch, e)
ch, e = writeIfChanged(filepath.Join(dnsDir, "hosts"), st.Hosts, 0o644)
note("hosts", ch, e)
conf := filepath.Join(dnsDir, "conf.d", "rollcall.conf")
if st.DnsmasqConf != "" {
ch, e = writeIfChanged(conf, st.DnsmasqConf+dnsmasqUpstream("/run/systemd/resolve/stub-resolv.conf", "/etc/resolv.conf", localAddrs()), 0o644)
note("dnsmasq.conf", ch, e)
} else if _, e := os.Stat(conf); e == nil {
note("-dnsmasq.conf", true, os.Remove(conf))
}
}
return changed, err
}
// localRackLoop is the embedded rack controller: it applies this host's state on every change.
func (a *App) localRackLoop() {
apply := func() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
st, err := a.rackState(ctx, a.cfg.RackName)
if err != nil {
log.Printf("embedded rack: %v", err)
return
}
if ch, err := applyRackState(st, a.cfg.TFTPRoot, a.cfg.DnsmasqDir); err != nil {
log.Printf("embedded rack: %v", err)
} else if len(ch) > 0 {
log.Printf("embedded rack %s: wrote %s", a.cfg.RackName, strings.Join(ch, " "))
}
_ = a.store.TouchRackController(ctx, a.cfg.RackName)
}
apply()
tick := time.NewTicker(60 * time.Second)
for {
select {
case <-a.hub.kick:
time.Sleep(150 * time.Millisecond) // let a burst of changes settle
select {
case <-a.hub.kick:
default:
}
case <-tick.C:
}
apply()
}
}
func localInterfaces() []RackIface {
var out []RackIface
ifs, _ := net.Interfaces()
for _, i := range ifs {
if i.Flags&net.FlagLoopback != 0 {
continue
}
ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()}
addrs, _ := i.Addrs()
for _, ad := range addrs {
if ipn, ok := ad.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() {
ri.Addrs = append(ri.Addrs, ipn.String())
}
}
if len(ri.Addrs) > 0 {
out = append(out, ri)
}
}
return out
}
// startRacks registers the embedded rack controller, carries over what an
// older version kept on disk or in config, and starts the local apply loop.
func (a *App) startRacks(ctx context.Context) error {
if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, Arch: runtime.GOARCH,
Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil {
return err
}
a.reloadNet(ctx)
// per-MAC files written before pxe_files existed
var n int
_ = a.store.pool.QueryRow(ctx, `SELECT count(*) FROM pxe_files`).Scan(&n)
if n == 0 {
files, _ := filepath.Glob(filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-*"))
ms, _ := a.store.List(ctx)
for _, f := range files {
b, err := os.ReadFile(f)
if err != nil {
continue
}
mac := strings.ReplaceAll(strings.TrimPrefix(filepath.Base(f), "01-"), "-", ":")
owner := ""
for _, m := range ms {
if hasTag(m.MACs, mac) {
owner = m.ID
}
}
_ = a.store.PutPXE(ctx, mac, owner, a.cfg.RackName, string(b))
}
if len(files) > 0 {
log.Printf("imported %d existing PXE files", len(files))
}
}
a.adoptIPs(ctx)
go a.localRackLoop()
return nil
}
// adoptIPs records addresses machines already have (node pools, before a
// subnet was modeled) so IPAM never hands them out twice.
func (a *App) adoptIPs(ctx context.Context) {
subnets, err := a.store.Subnets(ctx)
if err != nil || len(subnets) == 0 {
return
}
ms, _ := a.store.List(ctx)
for _, m := range ms {
for _, ip := range []string{deref(m.MgmtIP), deref(m.IBIP)} {
if sn := subnetFor(subnets, ip); sn != nil {
iface := ""
if b := bootNIC(machineNICs(m), m.MACs); b != nil && ip == deref(m.MgmtIP) {
iface = b.Name
}
_, _ = a.claimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: m.ID, Iface: iface, Kind: "auto", Hostname: deref(m.NodeName)})
}
}
}
}
// ---------------------------------------------------------------- PXE files through the rack
// writePXEBody records a machine's PXE file for its rack controller.
func (a *App) writePXEBody(m *Machine, body string) error {
rack := a.rackOf(m)
for _, mac := range m.MACs {
if err := a.store.PutPXE(context.Background(), normMAC(mac), m.ID, rack, body); err != nil {
return err
}
}
a.hub.bump()
return nil
}
func (a *App) removePXE(m *Machine) {
for _, mac := range m.MACs {
_ = a.store.DeletePXE(context.Background(), normMAC(mac))
}
a.hub.bump()
}
// exportDnsmasq: host files and dnsmasq configuration follow from the state now.
func (a *App) exportDnsmasq(ctx context.Context) error {
a.hub.bump()
return nil
}
// ---------------------------------------------------------------- HTTP: rack controllers
func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
if name := a.peerRack(r); name != "" {
next(w, r, name)
return
}
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" {
fail(w, 401, errors.New("rack controller certificate (or legacy secret) required"))
return
}
next(w, r, name)
}
}
// fromRack names the rack controller a node request came through (it proxies them).
func (a *App) fromRack(r *http.Request) string {
if name := a.peerRack(r); name != "" {
return name
}
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 {
return name
}
return a.cfg.RackName
}
func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack string) {
var rc RackController
if err := json.NewDecoder(r.Body).Decode(&rc); err != nil {
fail(w, 400, err)
return
}
rc.Name, rc.Kind = rack, "remote"
if rack == a.cfg.RackName {
fail(w, 409, fmt.Errorf("%s is the embedded rack controller's name", rack))
return
}
if rc.URL == "" {
fail(w, 400, errors.New("url is required"))
return
}
if err := a.store.PutRackController(r.Context(), rc); err != nil {
fail(w, 500, err)
return
}
a.reloadRacks(r.Context())
a.hub.bump()
log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces))
writeJSON(w, 200, obj{"ok": true, "gen": a.hub.current()})
}
func (a *App) handleRackState(w http.ResponseWriter, r *http.Request, rack string) {
if _, ok := a.racksSnapshot()[rack]; !ok {
fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack))
return
}
since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
_ = a.store.TouchRackController(r.Context(), rack)
a.hub.waitChange(r.Context(), since, 25*time.Second)
_ = a.store.TouchRackController(r.Context(), rack)
go a.reloadRacks(context.Background()) // keep last_seen fresh for online checks
st, err := a.rackState(r.Context(), rack)
if err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, st)
}
func (a *App) handleRackWork(w http.ResponseWriter, r *http.Request, rack string) {
jobs := a.hub.take(r.Context(), rack, 25*time.Second)
if jobs == nil {
jobs = []*rackJob{}
}
writeJSON(w, 200, jobs)
}
func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack string) {
var res rackResult
if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
fail(w, 400, err)
return
}
writeJSON(w, 200, obj{"accepted": a.hub.finish(res)})
}
func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) {
rs, err := a.store.RackControllers(r.Context())
if err != nil {
fail(w, 500, err)
return
}
ms, _ := a.store.List(r.Context())
for i := range rs {
rs[i].Online = rackOnline(rs[i], a.cfg.RackName)
for _, m := range ms {
if a.rackOf(m) == rs[i].Name {
rs[i].Machines++
}
}
}
writeJSON(w, 200, rs)
}
func (a *App) handleControllerDelete(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("name")
if name == a.cfg.RackName {
fail(w, 409, errors.New("the embedded rack controller cannot be removed"))
return
}
if err := a.store.DeleteRackController(r.Context(), name); err != nil {
fail(w, 500, err)
return
}
a.reloadRacks(r.Context())
a.hub.bump()
writeJSON(w, 200, obj{"deleted": name})
}
// handleRackDHCP shows a rack's generated dnsmasq configuration (admin).
func (a *App) handleRackDHCP(w http.ResponseWriter, r *http.Request) {
st, err := a.rackState(r.Context(), r.PathValue("name"))
if err != nil {
fail(w, 500, err)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
if st.DnsmasqConf == "" {
fmt.Fprintf(w, "# rack controller %s serves no VLAN with dhcp on: rollcall leaves dnsmasq's configuration alone\n", r.PathValue("name"))
}
fmt.Fprint(w, st.DnsmasqConf, "\n# --- hosts.dhcp\n", st.HostsDHCP)
}
// dnsmasqUpstream tells dnsmasq where to forward names it does not answer
// itself. Without it Debian's dnsmasq looks for a resolvconf-made file that
// usually does not exist and refuses every outside name, so nodes could not
// resolve mirrors or site services. With systemd-resolved the stub is used, so
// its per-link routing (split DNS such as ~int.example.com) keeps working;
// otherwise the host's resolv.conf servers, never ourselves.
func dnsmasqUpstream(stub, resolv string, own []string) string {
if _, err := os.Stat(stub); err == nil {
return "# upstream: systemd-resolved on this host (keeps its split DNS)\nno-resolv\nserver=127.0.0.53\n"
}
b, err := os.ReadFile(resolv)
if err != nil {
return ""
}
mine := map[string]bool{}
for _, a := range own {
mine[stripPrefix(a)] = true
}
var servers []string
for _, line := range strings.Split(string(b), "\n") {
f := strings.Fields(line)
if len(f) < 2 || f[0] != "nameserver" {
continue
}
ip, ok := parseAddr(f[1])
if !ok || mine[ip.String()] || (ip.IsLoopback() && ip.String() != "127.0.0.53") {
continue
}
servers = append(servers, ip.String())
}
if len(servers) == 0 {
return ""
}
out := "# upstream: this host's resolv.conf\nno-resolv\n"
for _, sv := range servers {
out += "server=" + sv + "\n"
}
return out
}
// localAddrs: this host's interface addresses (CIDR form).
func localAddrs() []string {
var out []string
addrs, _ := net.InterfaceAddrs()
for _, a := range addrs {
out = append(out, a.String())
}
return out
}