A VLAN in proxy mode leaves addresses to the site's DHCP server; the rack only answers PXE (dnsmasq proxy range + pxe-service), machines keep DHCP unless a static range is set aside. CLI -proxy-dhcp, web UI mode select, bootstrap --proxy-dhcp, doctor/discover aware. dnsmasq 2.91 (and 2.93) loops forever in its proxy path when the proxy range carries a set: tag: option_filter appends a request's tags to the context's persistent list, so the next request's stack interface tag points at itself (found with gdb on the lab rack). Proxy ranges are untagged and PXE services limited by interface; dhcp-boot/dhcp-match only for served subnets; no dhcp-host lines for proxied ones. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
939 lines
28 KiB
Go
939 lines
28 KiB
Go
package main
|
|
|
|
// Region side of the region/rack split, after MAAS. The region (rollcall
|
|
// serve) owns the database and computes everything; rack controllers sit next
|
|
// to the machines and do what has to happen on the machines' networks: DHCP
|
|
// and TFTP (dnsmasq), the per-MAC PXE files, BMC commands on a BMC network the
|
|
// region may not reach, and serving boot and OS images from a local cache.
|
|
//
|
|
// A rack controller pulls: it long-polls its desired state (PXE files,
|
|
// dnsmasq configuration, host files) and a queue of commands (ipmitool), so
|
|
// only rack → region connectivity is needed. The region host itself runs an
|
|
// embedded rack controller that applies the same state in-process, which is
|
|
// what a single-server installation uses.
|
|
|
|
import (
|
|
"context"
|
|
"crypto/subtle"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type RackIface struct {
|
|
Name string `json:"name"`
|
|
MAC string `json:"mac"`
|
|
Addrs []string `json:"addrs"`
|
|
}
|
|
|
|
type RackController struct {
|
|
Name string `json:"name"`
|
|
URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1
|
|
Kind string `json:"kind"` // embedded | remote
|
|
Version string `json:"version"`
|
|
Arch string `json:"arch,omitempty"` // GOARCH, for self-update
|
|
Interfaces []RackIface `json:"interfaces"`
|
|
DHCPInterfaces []string `json:"dhcp_interfaces"`
|
|
TFTPRoot string `json:"tftp_root"`
|
|
LastSeen *time.Time `json:"last_seen"`
|
|
Registered time.Time `json:"registered"`
|
|
Online bool `json:"online"`
|
|
Machines int `json:"machines"`
|
|
}
|
|
|
|
// rackState is everything a rack controller writes to disk.
|
|
type rackState struct {
|
|
Gen int64 `json:"gen"`
|
|
DefaultPXE string `json:"default_pxe"`
|
|
PXE map[string]string `json:"pxe"` // pxelinux.cfg file name (01-aa-bb-…) → content
|
|
DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here
|
|
HostsDHCP string `json:"hosts_dhcp"`
|
|
Hosts string `json:"hosts"`
|
|
BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP
|
|
Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run
|
|
}
|
|
|
|
type rackJob struct {
|
|
ID string `json:"id"`
|
|
Kind string `json:"kind"` // power
|
|
Driver string `json:"driver"`
|
|
Address string `json:"address"`
|
|
Port int `json:"port"`
|
|
User string `json:"user"`
|
|
Password string `json:"password"`
|
|
Extra map[string]string `json:"extra"`
|
|
Op string `json:"op"`
|
|
Arg string `json:"arg"`
|
|
done chan rackResult
|
|
}
|
|
|
|
type rackResult struct {
|
|
ID string `json:"id"`
|
|
Output string `json:"output"`
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
// ---------------------------------------------------------------- store
|
|
|
|
func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
|
|
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
out := []RackController{}
|
|
for rows.Next() {
|
|
var r RackController
|
|
var ifs, dif []byte
|
|
if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
|
|
return nil, err
|
|
}
|
|
_ = json.Unmarshal(ifs, &r.Interfaces)
|
|
_ = json.Unmarshal(dif, &r.DHCPInterfaces)
|
|
out = append(out, r)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
func (s *Store) PutRackController(ctx context.Context, r RackController) error {
|
|
if r.Interfaces == nil {
|
|
r.Interfaces = []RackIface{}
|
|
}
|
|
if r.DHCPInterfaces == nil {
|
|
r.DHCPInterfaces = []string{}
|
|
}
|
|
ifs, _ := json.Marshal(r.Interfaces)
|
|
dif, _ := json.Marshal(r.DHCPInterfaces)
|
|
_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''))
|
|
ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,'')`,
|
|
r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) TouchRackController(ctx context.Context, name string) error {
|
|
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET last_seen=now() WHERE name=$1`, name)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) DeleteRackController(ctx context.Context, name string) error {
|
|
_, err := s.pool.Exec(ctx, `DELETE FROM rack_controllers WHERE name=$1`, name)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) SetRackController(ctx context.Context, id, rack string) error {
|
|
_, err := s.pool.Exec(ctx, `UPDATE machines SET rack_controller=nullif($2,'') WHERE id=$1`, id, rack)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error {
|
|
_, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4)
|
|
ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now()`, mac, machineID, rack, content)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) DeletePXE(ctx context.Context, mac string) error {
|
|
_, err := s.pool.Exec(ctx, `DELETE FROM pxe_files WHERE mac=$1`, mac)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, error) {
|
|
rows, err := s.pool.Query(ctx, `SELECT mac, content FROM pxe_files WHERE rack=$1`, rack)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
out := map[string]string{}
|
|
for rows.Next() {
|
|
var mac, content string
|
|
if err := rows.Scan(&mac, &content); err != nil {
|
|
return nil, err
|
|
}
|
|
out[pxeFileName(mac)] = content
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") }
|
|
|
|
// ---------------------------------------------------------------- hub: state generations and the command queue
|
|
|
|
type rackHub struct {
|
|
mu sync.Mutex
|
|
gen int64
|
|
changed chan struct{}
|
|
work map[string][]*rackJob
|
|
wake map[string]chan struct{}
|
|
flight map[string]*rackJob
|
|
kick chan struct{} // the embedded rack applies on this
|
|
}
|
|
|
|
func newRackHub() *rackHub {
|
|
return &rackHub{gen: time.Now().UnixNano(), changed: make(chan struct{}), work: map[string][]*rackJob{},
|
|
wake: map[string]chan struct{}{}, flight: map[string]*rackJob{}, kick: make(chan struct{}, 1)}
|
|
}
|
|
|
|
func (h *rackHub) current() int64 {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
return h.gen
|
|
}
|
|
|
|
// bump says the desired state of some rack changed.
|
|
func (h *rackHub) bump() {
|
|
h.mu.Lock()
|
|
h.gen++
|
|
close(h.changed)
|
|
h.changed = make(chan struct{})
|
|
h.mu.Unlock()
|
|
select {
|
|
case h.kick <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// waitChange blocks until the generation moves past since, or the wait is over.
|
|
func (h *rackHub) waitChange(ctx context.Context, since int64, wait time.Duration) int64 {
|
|
t := time.NewTimer(wait)
|
|
defer t.Stop()
|
|
for {
|
|
h.mu.Lock()
|
|
g, ch := h.gen, h.changed
|
|
h.mu.Unlock()
|
|
if g != since {
|
|
return g
|
|
}
|
|
select {
|
|
case <-ch:
|
|
case <-t.C:
|
|
return g
|
|
case <-ctx.Done():
|
|
return g
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string, error) {
|
|
j.ID = randomToken(8)
|
|
j.done = make(chan rackResult, 1)
|
|
h.mu.Lock()
|
|
h.work[rack] = append(h.work[rack], j)
|
|
if w, ok := h.wake[rack]; ok {
|
|
close(w)
|
|
delete(h.wake, rack)
|
|
}
|
|
h.mu.Unlock()
|
|
t := time.NewTimer(90 * time.Second)
|
|
defer t.Stop()
|
|
select {
|
|
case r := <-j.done:
|
|
if r.Error != "" {
|
|
return r.Output, errors.New(r.Error)
|
|
}
|
|
return r.Output, nil
|
|
case <-t.C:
|
|
case <-ctx.Done():
|
|
}
|
|
h.mu.Lock()
|
|
delete(h.flight, j.ID)
|
|
q := h.work[rack][:0]
|
|
for _, x := range h.work[rack] {
|
|
if x != j {
|
|
q = append(q, x)
|
|
}
|
|
}
|
|
h.work[rack] = q
|
|
h.mu.Unlock()
|
|
return "", fmt.Errorf("rack controller %s did not answer in time", rack)
|
|
}
|
|
|
|
func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*rackJob {
|
|
deadline := time.Now().Add(wait)
|
|
for {
|
|
h.mu.Lock()
|
|
if q := h.work[rack]; len(q) > 0 {
|
|
h.work[rack] = nil
|
|
for _, j := range q {
|
|
h.flight[j.ID] = j
|
|
}
|
|
h.mu.Unlock()
|
|
return q
|
|
}
|
|
w, ok := h.wake[rack]
|
|
if !ok {
|
|
w = make(chan struct{})
|
|
h.wake[rack] = w
|
|
}
|
|
h.mu.Unlock()
|
|
left := time.Until(deadline)
|
|
if left <= 0 {
|
|
return nil
|
|
}
|
|
t := time.NewTimer(left)
|
|
select {
|
|
case <-w:
|
|
case <-t.C:
|
|
case <-ctx.Done():
|
|
t.Stop()
|
|
return nil
|
|
}
|
|
t.Stop()
|
|
}
|
|
}
|
|
|
|
func (h *rackHub) finish(r rackResult) bool {
|
|
h.mu.Lock()
|
|
j, ok := h.flight[r.ID]
|
|
delete(h.flight, r.ID)
|
|
h.mu.Unlock()
|
|
if ok {
|
|
j.done <- r
|
|
}
|
|
return ok
|
|
}
|
|
|
|
// ---------------------------------------------------------------- routing machines to racks
|
|
|
|
func (a *App) racksSnapshot() map[string]RackController {
|
|
a.netMu.RLock()
|
|
defer a.netMu.RUnlock()
|
|
out := make(map[string]RackController, len(a.racks))
|
|
for k, v := range a.racks {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (a *App) subnetsSnapshot() []Subnet {
|
|
a.netMu.RLock()
|
|
defer a.netMu.RUnlock()
|
|
return a.subnets
|
|
}
|
|
|
|
// reloadNet refreshes the cached subnets and rack controllers after a change.
|
|
func (a *App) reloadNet(ctx context.Context) {
|
|
if sn, err := a.store.Subnets(ctx); err == nil {
|
|
a.netMu.Lock()
|
|
a.subnets = sn
|
|
a.netMu.Unlock()
|
|
}
|
|
a.reloadRacks(ctx)
|
|
}
|
|
|
|
func (a *App) reloadRacks(ctx context.Context) {
|
|
rs, err := a.store.RackControllers(ctx)
|
|
if err != nil {
|
|
log.Printf("racks: %v", err)
|
|
return
|
|
}
|
|
m := map[string]RackController{}
|
|
for _, r := range rs {
|
|
m[r.Name] = r
|
|
}
|
|
a.netMu.Lock()
|
|
a.racks = m
|
|
a.netMu.Unlock()
|
|
}
|
|
|
|
func rackOnline(r RackController, local string) bool {
|
|
return r.Name == local || (r.LastSeen != nil && time.Since(*r.LastSeen) < 90*time.Second)
|
|
}
|
|
|
|
// rackOf names the rack controller serving a machine: the one it last came
|
|
// through, else the one serving the VLAN of its management or boot address,
|
|
// else the embedded one.
|
|
func (a *App) rackOf(m *Machine) string {
|
|
racks := a.racksSnapshot()
|
|
if m.RackController != "" {
|
|
if _, ok := racks[m.RackController]; ok {
|
|
return m.RackController
|
|
}
|
|
}
|
|
if subnets := a.subnetsSnapshot(); len(subnets) > 0 {
|
|
ips := []string{deref(m.MgmtIP)}
|
|
if b := bootNIC(machineNICs(m), m.MACs); b != nil {
|
|
ips = append(ips, b.IPv4...)
|
|
}
|
|
for _, ip := range ips {
|
|
if sn := subnetFor(subnets, ip); sn != nil && sn.PrimaryRack != "" {
|
|
if _, ok := racks[sn.PrimaryRack]; ok {
|
|
return sn.PrimaryRack
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return a.cfg.RackName
|
|
}
|
|
|
|
func (a *App) rackURL(name string) string {
|
|
if name == "" || name == a.cfg.RackName {
|
|
return a.cfg.ServerURL
|
|
}
|
|
if r, ok := a.racksSnapshot()[name]; ok && r.URL != "" {
|
|
return strings.TrimRight(r.URL, "/")
|
|
}
|
|
return a.cfg.ServerURL
|
|
}
|
|
|
|
// urlFor is the base URL a machine uses for images and the node API.
|
|
func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) }
|
|
|
|
// appendFor is commission_append with the server URL of the machine's rack.
|
|
func (a *App) appendFor(base string) string {
|
|
if base == a.cfg.ServerURL {
|
|
return a.cfg.CommissionAppend
|
|
}
|
|
return strings.ReplaceAll(a.cfg.CommissionAppend, a.cfg.ServerURL, base)
|
|
}
|
|
|
|
// rackAddresses are the rack controllers' own addresses, never handed to machines.
|
|
func (a *App) rackAddresses() []string {
|
|
var out []string
|
|
for _, r := range a.racksSnapshot() {
|
|
for _, i := range r.Interfaces {
|
|
out = append(out, i.Addrs...)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ---------------------------------------------------------------- BMC through the machine's rack
|
|
|
|
// ---------------------------------------------------------------- desired state
|
|
|
|
const defaultPXETmpl = `# generated by rollcall for rack controller %s
|
|
# unknown machines land here; known ones get a per-MAC file (01-xx-xx-…)
|
|
DEFAULT menu.c32
|
|
PROMPT 0
|
|
TIMEOUT 80
|
|
MENU TITLE rollcall · %s · unknown machine: commissioning in 8s
|
|
|
|
LABEL commission
|
|
MENU LABEL Commission this machine (inventory + BMC account)
|
|
MENU DEFAULT
|
|
KERNEL rollcall/vmlinuz
|
|
INITRD rollcall/initrd.img
|
|
APPEND %s
|
|
|
|
LABEL commission-debug
|
|
MENU LABEL Commission, but stay up afterwards (ssh root@<ip>)
|
|
KERNEL rollcall/vmlinuz
|
|
INITRD rollcall/initrd.img
|
|
APPEND %s rollcall.nopoweroff
|
|
|
|
LABEL local
|
|
MENU LABEL Boot from local disk
|
|
LOCALBOOT 0
|
|
`
|
|
|
|
func (a *App) rackState(ctx context.Context, name string) (*rackState, error) {
|
|
gen := a.hub.current()
|
|
st := &rackState{Gen: gen}
|
|
if rc, ok := a.racksSnapshot()[name]; ok {
|
|
st.Update = a.rackUpdateFor(rc)
|
|
}
|
|
base := a.rackURL(name)
|
|
app := a.appendFor(base)
|
|
st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app)
|
|
var err error
|
|
if st.PXE, err = a.store.PXEFiles(ctx, name); err != nil {
|
|
return nil, err
|
|
}
|
|
subnets, err := a.store.Subnets(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rc := a.racksSnapshot()[name]
|
|
rc.Name = name
|
|
dnsDir := a.cfg.DnsmasqDir
|
|
if dnsDir == "" {
|
|
dnsDir = "/var/lib/rollcall/dnsmasq"
|
|
}
|
|
if conf, ok := dnsmasqConfig(rc, subnets, dnsDir); ok {
|
|
st.DnsmasqConf = conf
|
|
}
|
|
if st.HostsDHCP, st.Hosts, err = a.hostsFiles(ctx, name, subnets); err != nil {
|
|
return nil, err
|
|
}
|
|
st.BootFiles = []string{"rollcall/vmlinuz", "rollcall/initrd.img"}
|
|
if ents, err := os.ReadDir(a.cfg.ImagesDir); err == nil {
|
|
for _, e := range ents {
|
|
if e.IsDir() {
|
|
st.BootFiles = append(st.BootFiles, "images/"+e.Name()+"/vmlinuz", "images/"+e.Name()+"/initrd.img")
|
|
}
|
|
}
|
|
}
|
|
return st, nil
|
|
}
|
|
|
|
// hostsFiles: static DHCP leases for the machines this rack serves, and names
|
|
// for every machine: the addresses on the interface carrying the management
|
|
// address get the plain name (A and AAAA), others name-<interface>, IB name-ib.
|
|
func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) {
|
|
ms, err := a.store.List(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
all, _ := a.store.IPs(ctx, "", "")
|
|
byMachine := map[string][]IPAddr{}
|
|
for _, x := range all {
|
|
if x.MachineID != "" {
|
|
byMachine[x.MachineID] = append(byMachine[x.MachineID], x)
|
|
}
|
|
}
|
|
var dhcp, hosts strings.Builder
|
|
dhcp.WriteString("# generated by rollcall\n")
|
|
hosts.WriteString("# generated by rollcall\n")
|
|
domainOf := func(ip string) string {
|
|
if sn := subnetFor(subnets, ip); sn != nil && sn.Domain != "" {
|
|
return sn.Domain
|
|
}
|
|
return str(a.cfg.ClusterVars["domain"])
|
|
}
|
|
line := func(ip, name string) {
|
|
if d := domainOf(ip); d != "" {
|
|
fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name)
|
|
} else {
|
|
fmt.Fprintf(&hosts, "%s %s\n", ip, name)
|
|
}
|
|
}
|
|
// proxy subnets: the site's DHCP server hands out the addresses, so no dhcp-host
|
|
// lines for them (they would do nothing, and the "known" tag they give a client
|
|
// helps dnsmasq 2.91 build a cyclic tag list in its PXE proxy code: busy loop)
|
|
proxied := func(ip string) bool {
|
|
sn := subnetFor(subnets, ip)
|
|
return sn != nil && sn.DHCPMode == "proxy"
|
|
}
|
|
sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) })
|
|
for _, m := range ms {
|
|
if m.NodeName == nil || len(m.MACs) == 0 {
|
|
continue
|
|
}
|
|
name, mgmt, ibip := *m.NodeName, deref(m.MgmtIP), deref(m.IBIP)
|
|
mine := a.rackOf(m) == rack
|
|
mgmtIface := ""
|
|
for _, x := range byMachine[m.ID] {
|
|
if x.IP == mgmt {
|
|
mgmtIface = x.Iface
|
|
}
|
|
}
|
|
if mgmt != "" {
|
|
line(mgmt, name)
|
|
if mine && familyOf(mgmt+"/32") == 4 && !proxied(mgmt) {
|
|
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], mgmt, name)
|
|
}
|
|
}
|
|
for _, x := range byMachine[m.ID] {
|
|
if x.IP == mgmt || x.IP == ibip {
|
|
continue
|
|
}
|
|
v6 := strings.Contains(x.IP, ":")
|
|
switch {
|
|
case x.Iface == mgmtIface && mgmtIface != "":
|
|
line(x.IP, name)
|
|
if v6 && mine && !proxied(x.IP) {
|
|
fmt.Fprintf(&dhcp, "%s,[%s],%s\n", m.MACs[0], x.IP, name)
|
|
}
|
|
case x.Iface != "":
|
|
line(x.IP, name+"-"+strings.NewReplacer(".", "-", ":", "-").Replace(x.Iface))
|
|
}
|
|
}
|
|
if ibip != "" {
|
|
fmt.Fprintf(&hosts, "%s %s-ib\n", ibip, name)
|
|
}
|
|
}
|
|
return dhcp.String(), hosts.String(), nil
|
|
}
|
|
|
|
// writeIfChanged replaces a file only when its content differs (path units watch these).
|
|
func writeIfChanged(p, content string, mode os.FileMode) (bool, error) {
|
|
if old, err := os.ReadFile(p); err == nil && string(old) == content {
|
|
return false, nil
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
return false, err
|
|
}
|
|
if err := os.WriteFile(p+".tmp", []byte(content), mode); err != nil {
|
|
return false, err
|
|
}
|
|
return true, os.Rename(p+".tmp", p)
|
|
}
|
|
|
|
// applyRackState writes a rack's state: PXE files under tftp/pxelinux.cfg
|
|
// (stale per-MAC files removed), the dnsmasq configuration and host files.
|
|
func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err error) {
|
|
dir := filepath.Join(tftp, "pxelinux.cfg")
|
|
note := func(p string, ch bool, e error) {
|
|
if e != nil && err == nil {
|
|
err = e
|
|
}
|
|
if ch {
|
|
changed = append(changed, p)
|
|
}
|
|
}
|
|
if st.DefaultPXE != "" {
|
|
ch, e := writeIfChanged(filepath.Join(dir, "default"), st.DefaultPXE, 0o644)
|
|
note("default", ch, e)
|
|
}
|
|
for name, body := range st.PXE {
|
|
ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644)
|
|
note(name, ch, e)
|
|
}
|
|
if olds, e := filepath.Glob(filepath.Join(dir, "01-*")); e == nil {
|
|
for _, p := range olds {
|
|
if _, keep := st.PXE[filepath.Base(p)]; !keep && !strings.HasSuffix(p, ".tmp") {
|
|
note("-"+filepath.Base(p), true, os.Remove(p))
|
|
}
|
|
}
|
|
}
|
|
if dnsDir != "" {
|
|
ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644)
|
|
note("hosts.dhcp", ch, e)
|
|
ch, e = writeIfChanged(filepath.Join(dnsDir, "hosts"), st.Hosts, 0o644)
|
|
note("hosts", ch, e)
|
|
conf := filepath.Join(dnsDir, "conf.d", "rollcall.conf")
|
|
if st.DnsmasqConf != "" {
|
|
ch, e = writeIfChanged(conf, st.DnsmasqConf+dnsmasqUpstream("/run/systemd/resolve/stub-resolv.conf", "/etc/resolv.conf", localAddrs()), 0o644)
|
|
note("dnsmasq.conf", ch, e)
|
|
} else if _, e := os.Stat(conf); e == nil {
|
|
note("-dnsmasq.conf", true, os.Remove(conf))
|
|
}
|
|
}
|
|
return changed, err
|
|
}
|
|
|
|
// localRackLoop is the embedded rack controller: it applies this host's state on every change.
|
|
func (a *App) localRackLoop() {
|
|
apply := func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
st, err := a.rackState(ctx, a.cfg.RackName)
|
|
if err != nil {
|
|
log.Printf("embedded rack: %v", err)
|
|
return
|
|
}
|
|
if ch, err := applyRackState(st, a.cfg.TFTPRoot, a.cfg.DnsmasqDir); err != nil {
|
|
log.Printf("embedded rack: %v", err)
|
|
} else if len(ch) > 0 {
|
|
log.Printf("embedded rack %s: wrote %s", a.cfg.RackName, strings.Join(ch, " "))
|
|
}
|
|
_ = a.store.TouchRackController(ctx, a.cfg.RackName)
|
|
}
|
|
apply()
|
|
tick := time.NewTicker(60 * time.Second)
|
|
for {
|
|
select {
|
|
case <-a.hub.kick:
|
|
time.Sleep(150 * time.Millisecond) // let a burst of changes settle
|
|
select {
|
|
case <-a.hub.kick:
|
|
default:
|
|
}
|
|
case <-tick.C:
|
|
}
|
|
apply()
|
|
}
|
|
}
|
|
|
|
func localInterfaces() []RackIface {
|
|
var out []RackIface
|
|
ifs, _ := net.Interfaces()
|
|
for _, i := range ifs {
|
|
if i.Flags&net.FlagLoopback != 0 {
|
|
continue
|
|
}
|
|
ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()}
|
|
addrs, _ := i.Addrs()
|
|
for _, ad := range addrs {
|
|
if ipn, ok := ad.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() {
|
|
ri.Addrs = append(ri.Addrs, ipn.String())
|
|
}
|
|
}
|
|
if len(ri.Addrs) > 0 {
|
|
out = append(out, ri)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// startRacks registers the embedded rack controller, carries over what an
|
|
// older version kept on disk or in config, and starts the local apply loop.
|
|
func (a *App) startRacks(ctx context.Context) error {
|
|
if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, Arch: runtime.GOARCH,
|
|
Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil {
|
|
return err
|
|
}
|
|
a.reloadNet(ctx)
|
|
// per-MAC files written before pxe_files existed
|
|
var n int
|
|
_ = a.store.pool.QueryRow(ctx, `SELECT count(*) FROM pxe_files`).Scan(&n)
|
|
if n == 0 {
|
|
files, _ := filepath.Glob(filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-*"))
|
|
ms, _ := a.store.List(ctx)
|
|
for _, f := range files {
|
|
b, err := os.ReadFile(f)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
mac := strings.ReplaceAll(strings.TrimPrefix(filepath.Base(f), "01-"), "-", ":")
|
|
owner := ""
|
|
for _, m := range ms {
|
|
if hasTag(m.MACs, mac) {
|
|
owner = m.ID
|
|
}
|
|
}
|
|
_ = a.store.PutPXE(ctx, mac, owner, a.cfg.RackName, string(b))
|
|
}
|
|
if len(files) > 0 {
|
|
log.Printf("imported %d existing PXE files", len(files))
|
|
}
|
|
}
|
|
a.adoptIPs(ctx)
|
|
go a.localRackLoop()
|
|
return nil
|
|
}
|
|
|
|
// adoptIPs records addresses machines already have (node pools, before a
|
|
// subnet was modeled) so IPAM never hands them out twice.
|
|
func (a *App) adoptIPs(ctx context.Context) {
|
|
subnets, err := a.store.Subnets(ctx)
|
|
if err != nil || len(subnets) == 0 {
|
|
return
|
|
}
|
|
ms, _ := a.store.List(ctx)
|
|
for _, m := range ms {
|
|
for _, ip := range []string{deref(m.MgmtIP), deref(m.IBIP)} {
|
|
if sn := subnetFor(subnets, ip); sn != nil {
|
|
iface := ""
|
|
if b := bootNIC(machineNICs(m), m.MACs); b != nil && ip == deref(m.MgmtIP) {
|
|
iface = b.Name
|
|
}
|
|
_, _ = a.claimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: m.ID, Iface: iface, Kind: "auto", Hostname: deref(m.NodeName)})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- PXE files through the rack
|
|
|
|
// writePXEBody records a machine's PXE file for its rack controller.
|
|
func (a *App) writePXEBody(m *Machine, body string) error {
|
|
rack := a.rackOf(m)
|
|
for _, mac := range m.MACs {
|
|
if err := a.store.PutPXE(context.Background(), normMAC(mac), m.ID, rack, body); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
a.hub.bump()
|
|
return nil
|
|
}
|
|
|
|
func (a *App) removePXE(m *Machine) {
|
|
for _, mac := range m.MACs {
|
|
_ = a.store.DeletePXE(context.Background(), normMAC(mac))
|
|
}
|
|
a.hub.bump()
|
|
}
|
|
|
|
// exportDnsmasq: host files and dnsmasq configuration follow from the state now.
|
|
func (a *App) exportDnsmasq(ctx context.Context) error {
|
|
a.hub.bump()
|
|
return nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------- HTTP: rack controllers
|
|
|
|
func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
if name := a.peerRack(r); name != "" {
|
|
next(w, r, name)
|
|
return
|
|
}
|
|
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
|
|
if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" {
|
|
fail(w, 401, errors.New("rack controller certificate (or legacy secret) required"))
|
|
return
|
|
}
|
|
next(w, r, name)
|
|
}
|
|
}
|
|
|
|
// fromRack names the rack controller a node request came through (it proxies them).
|
|
func (a *App) fromRack(r *http.Request) string {
|
|
if name := a.peerRack(r); name != "" {
|
|
return name
|
|
}
|
|
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
|
|
if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 {
|
|
return name
|
|
}
|
|
return a.cfg.RackName
|
|
}
|
|
|
|
func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack string) {
|
|
var rc RackController
|
|
if err := json.NewDecoder(r.Body).Decode(&rc); err != nil {
|
|
fail(w, 400, err)
|
|
return
|
|
}
|
|
rc.Name, rc.Kind = rack, "remote"
|
|
if rack == a.cfg.RackName {
|
|
fail(w, 409, fmt.Errorf("%s is the embedded rack controller's name", rack))
|
|
return
|
|
}
|
|
if rc.URL == "" {
|
|
fail(w, 400, errors.New("url is required"))
|
|
return
|
|
}
|
|
if err := a.store.PutRackController(r.Context(), rc); err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
a.reloadRacks(r.Context())
|
|
a.hub.bump()
|
|
log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces))
|
|
writeJSON(w, 200, obj{"ok": true, "gen": a.hub.current()})
|
|
}
|
|
|
|
func (a *App) handleRackState(w http.ResponseWriter, r *http.Request, rack string) {
|
|
if _, ok := a.racksSnapshot()[rack]; !ok {
|
|
fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack))
|
|
return
|
|
}
|
|
since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
|
|
_ = a.store.TouchRackController(r.Context(), rack)
|
|
a.hub.waitChange(r.Context(), since, 25*time.Second)
|
|
_ = a.store.TouchRackController(r.Context(), rack)
|
|
go a.reloadRacks(context.Background()) // keep last_seen fresh for online checks
|
|
st, err := a.rackState(r.Context(), rack)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, st)
|
|
}
|
|
|
|
func (a *App) handleRackWork(w http.ResponseWriter, r *http.Request, rack string) {
|
|
jobs := a.hub.take(r.Context(), rack, 25*time.Second)
|
|
if jobs == nil {
|
|
jobs = []*rackJob{}
|
|
}
|
|
writeJSON(w, 200, jobs)
|
|
}
|
|
|
|
func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack string) {
|
|
var res rackResult
|
|
if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
|
|
fail(w, 400, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, obj{"accepted": a.hub.finish(res)})
|
|
}
|
|
|
|
func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) {
|
|
rs, err := a.store.RackControllers(r.Context())
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
ms, _ := a.store.List(r.Context())
|
|
for i := range rs {
|
|
rs[i].Online = rackOnline(rs[i], a.cfg.RackName)
|
|
for _, m := range ms {
|
|
if a.rackOf(m) == rs[i].Name {
|
|
rs[i].Machines++
|
|
}
|
|
}
|
|
}
|
|
writeJSON(w, 200, rs)
|
|
}
|
|
|
|
func (a *App) handleControllerDelete(w http.ResponseWriter, r *http.Request) {
|
|
name := r.PathValue("name")
|
|
if name == a.cfg.RackName {
|
|
fail(w, 409, errors.New("the embedded rack controller cannot be removed"))
|
|
return
|
|
}
|
|
if err := a.store.DeleteRackController(r.Context(), name); err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
a.reloadRacks(r.Context())
|
|
a.hub.bump()
|
|
writeJSON(w, 200, obj{"deleted": name})
|
|
}
|
|
|
|
// handleRackDHCP shows a rack's generated dnsmasq configuration (admin).
|
|
func (a *App) handleRackDHCP(w http.ResponseWriter, r *http.Request) {
|
|
st, err := a.rackState(r.Context(), r.PathValue("name"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
if st.DnsmasqConf == "" {
|
|
fmt.Fprintf(w, "# rack controller %s serves no VLAN with dhcp on: rollcall leaves dnsmasq's configuration alone\n", r.PathValue("name"))
|
|
}
|
|
fmt.Fprint(w, st.DnsmasqConf, "\n# --- hosts.dhcp\n", st.HostsDHCP)
|
|
}
|
|
|
|
// dnsmasqUpstream tells dnsmasq where to forward names it does not answer
|
|
// itself. Without it Debian's dnsmasq looks for a resolvconf-made file that
|
|
// usually does not exist and refuses every outside name, so nodes could not
|
|
// resolve mirrors or site services. With systemd-resolved the stub is used, so
|
|
// its per-link routing (split DNS such as ~int.example.com) keeps working;
|
|
// otherwise the host's resolv.conf servers, never ourselves.
|
|
func dnsmasqUpstream(stub, resolv string, own []string) string {
|
|
if _, err := os.Stat(stub); err == nil {
|
|
return "# upstream: systemd-resolved on this host (keeps its split DNS)\nno-resolv\nserver=127.0.0.53\n"
|
|
}
|
|
b, err := os.ReadFile(resolv)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
mine := map[string]bool{}
|
|
for _, a := range own {
|
|
mine[stripPrefix(a)] = true
|
|
}
|
|
var servers []string
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) < 2 || f[0] != "nameserver" {
|
|
continue
|
|
}
|
|
ip, ok := parseAddr(f[1])
|
|
if !ok || mine[ip.String()] || (ip.IsLoopback() && ip.String() != "127.0.0.53") {
|
|
continue
|
|
}
|
|
servers = append(servers, ip.String())
|
|
}
|
|
if len(servers) == 0 {
|
|
return ""
|
|
}
|
|
out := "# upstream: this host's resolv.conf\nno-resolv\n"
|
|
for _, sv := range servers {
|
|
out += "server=" + sv + "\n"
|
|
}
|
|
return out
|
|
}
|
|
|
|
// localAddrs: this host's interface addresses (CIDR form).
|
|
func localAddrs() []string {
|
|
var out []string
|
|
addrs, _ := net.InterfaceAddrs()
|
|
for _, a := range addrs {
|
|
out = append(out, a.String())
|
|
}
|
|
return out
|
|
}
|