Rack controllers ask the region for a server certificate for their own URL's host (POST /api/v1/rack/server-cert; CN rack-server:NAME, ServerAuth only, never one of the region's names or addresses) and serve the agents' endpoints and images over HTTPS too (tls_listen, default <host>:443), reporting the HTTPS URL when they register (accepted only on the rack's own host). The pxelinux command lines name it with the CA fingerprint (rollcall.tls=, rollcall.ca=); the embedded rack uses the region's listener. Agents fetch /ca.pem, accept it only by that fingerprint and then use HTTPS with it, including the image download; a job fetched over HTTPS gets an HTTPS image URL. No fallback to HTTP once TLS was asked for; old agents ignore the new words. The CLI refuses a rack controller's certificate as the region. PXE itself (TFTP, kernel, initrd) stays unauthenticated: this protects what agents send and receive from listeners and from an HTTP server posing as the rack, not from an attacker rewriting boot files. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
972 lines
29 KiB
Go
972 lines
29 KiB
Go
package main
|
|
|
|
// Region side of the region/rack split, after MAAS. The region (rollcall
|
|
// serve) owns the database and computes everything; rack controllers sit next
|
|
// to the machines and do what has to happen on the machines' networks: DHCP
|
|
// and TFTP (dnsmasq), the per-MAC PXE files, BMC commands on a BMC network the
|
|
// region may not reach, and serving boot and OS images from a local cache.
|
|
//
|
|
// A rack controller pulls: it long-polls its desired state (PXE files,
|
|
// dnsmasq configuration, host files) and a queue of commands (ipmitool), so
|
|
// only rack → region connectivity is needed. The region host itself runs an
|
|
// embedded rack controller that applies the same state in-process, which is
|
|
// what a single-server installation uses.
|
|
|
|
import (
|
|
"context"
|
|
"crypto/subtle"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"runtime"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
type RackIface struct {
|
|
Name string `json:"name"`
|
|
MAC string `json:"mac"`
|
|
Addrs []string `json:"addrs"`
|
|
}
|
|
|
|
type RackController struct {
|
|
Name string `json:"name"`
|
|
URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1
|
|
TLSURL string `json:"tls_url,omitempty"` // the same host over HTTPS, when it serves it (agents)
|
|
Kind string `json:"kind"` // embedded | remote
|
|
Version string `json:"version"`
|
|
Arch string `json:"arch,omitempty"` // GOARCH, for self-update
|
|
Interfaces []RackIface `json:"interfaces"`
|
|
DHCPInterfaces []string `json:"dhcp_interfaces"`
|
|
TFTPRoot string `json:"tftp_root"`
|
|
LastSeen *time.Time `json:"last_seen"`
|
|
Registered time.Time `json:"registered"`
|
|
Online bool `json:"online"`
|
|
Machines int `json:"machines"`
|
|
}
|
|
|
|
// rackState is everything a rack controller writes to disk.
|
|
type rackState struct {
|
|
Gen int64 `json:"gen"`
|
|
DefaultPXE string `json:"default_pxe"`
|
|
PXE map[string]string `json:"pxe"` // pxelinux.cfg file name (01-aa-bb-…) → content
|
|
DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here
|
|
HostsDHCP string `json:"hosts_dhcp"`
|
|
Hosts string `json:"hosts"`
|
|
BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP
|
|
Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run
|
|
}
|
|
|
|
type rackJob struct {
|
|
ID string `json:"id"`
|
|
Kind string `json:"kind"` // power
|
|
Driver string `json:"driver"`
|
|
Address string `json:"address"`
|
|
Port int `json:"port"`
|
|
User string `json:"user"`
|
|
Password string `json:"password"`
|
|
Extra map[string]string `json:"extra"`
|
|
Op string `json:"op"`
|
|
Arg string `json:"arg"`
|
|
Timeout int `json:"timeout,omitempty"` // seconds the region waits (default 90)
|
|
done chan rackResult
|
|
rack string // who took it: only that rack may answer
|
|
}
|
|
|
|
type rackResult struct {
|
|
ID string `json:"id"`
|
|
Output string `json:"output"`
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
// ---------------------------------------------------------------- store
|
|
|
|
func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
|
|
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
out := []RackController{}
|
|
for rows.Next() {
|
|
var r RackController
|
|
var ifs, dif []byte
|
|
if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
|
|
return nil, err
|
|
}
|
|
_ = json.Unmarshal(ifs, &r.Interfaces)
|
|
_ = json.Unmarshal(dif, &r.DHCPInterfaces)
|
|
out = append(out, r)
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
func (s *Store) PutRackController(ctx context.Context, r RackController) error {
|
|
if r.Interfaces == nil {
|
|
r.Interfaces = []RackIface{}
|
|
}
|
|
if r.DHCPInterfaces == nil {
|
|
r.DHCPInterfaces = []string{}
|
|
}
|
|
ifs, _ := json.Marshal(r.Interfaces)
|
|
dif, _ := json.Marshal(r.DHCPInterfaces)
|
|
_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch, tls_url) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''),nullif($9,''))
|
|
ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,''), tls_url=nullif($9,'')`,
|
|
r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch, r.TLSURL)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) TouchRackController(ctx context.Context, name string) error {
|
|
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET last_seen=now() WHERE name=$1`, name)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) DeleteRackController(ctx context.Context, name string) error {
|
|
_, err := s.pool.Exec(ctx, `DELETE FROM rack_controllers WHERE name=$1`, name)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) SetRackController(ctx context.Context, id, rack string) error {
|
|
_, err := s.pool.Exec(ctx, `UPDATE machines SET rack_controller=nullif($2,'') WHERE id=$1`, id, rack)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error {
|
|
// a MAC another machine owns is not taken over: a box claiming a deployed
|
|
// machine's MAC would otherwise replace that machine's boot file
|
|
tag, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4)
|
|
ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now() WHERE pxe_files.machine_id=$2`, mac, machineID, rack, content)
|
|
if err == nil && tag.RowsAffected() == 0 {
|
|
return fmt.Errorf("MAC %s belongs to another machine; delete that machine first", mac)
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (s *Store) DeletePXE(ctx context.Context, mac string) error {
|
|
_, err := s.pool.Exec(ctx, `DELETE FROM pxe_files WHERE mac=$1`, mac)
|
|
return err
|
|
}
|
|
|
|
func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, error) {
|
|
rows, err := s.pool.Query(ctx, `SELECT mac, content FROM pxe_files WHERE rack=$1`, rack)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
out := map[string]string{}
|
|
for rows.Next() {
|
|
var mac, content string
|
|
if err := rows.Scan(&mac, &content); err != nil {
|
|
return nil, err
|
|
}
|
|
out[pxeFileName(mac)] = content
|
|
}
|
|
return out, rows.Err()
|
|
}
|
|
|
|
var pxeNameRe = regexp.MustCompile(`^01(-[0-9a-f]{2}){6,20}$`)
|
|
|
|
func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") }
|
|
|
|
// ---------------------------------------------------------------- hub: state generations and the command queue
|
|
|
|
type rackHub struct {
|
|
mu sync.Mutex
|
|
gen int64
|
|
changed chan struct{}
|
|
work map[string][]*rackJob
|
|
wake map[string]chan struct{}
|
|
flight map[string]*rackJob
|
|
kick chan struct{} // the embedded rack applies on this
|
|
}
|
|
|
|
func newRackHub() *rackHub {
|
|
return &rackHub{gen: time.Now().UnixNano(), changed: make(chan struct{}), work: map[string][]*rackJob{},
|
|
wake: map[string]chan struct{}{}, flight: map[string]*rackJob{}, kick: make(chan struct{}, 1)}
|
|
}
|
|
|
|
func (h *rackHub) current() int64 {
|
|
h.mu.Lock()
|
|
defer h.mu.Unlock()
|
|
return h.gen
|
|
}
|
|
|
|
// bump says the desired state of some rack changed.
|
|
func (h *rackHub) bump() {
|
|
h.mu.Lock()
|
|
h.gen++
|
|
close(h.changed)
|
|
h.changed = make(chan struct{})
|
|
h.mu.Unlock()
|
|
select {
|
|
case h.kick <- struct{}{}:
|
|
default:
|
|
}
|
|
}
|
|
|
|
// waitChange blocks until the generation moves past since, or the wait is over.
|
|
func (h *rackHub) waitChange(ctx context.Context, since int64, wait time.Duration) int64 {
|
|
t := time.NewTimer(wait)
|
|
defer t.Stop()
|
|
for {
|
|
h.mu.Lock()
|
|
g, ch := h.gen, h.changed
|
|
h.mu.Unlock()
|
|
if g != since {
|
|
return g
|
|
}
|
|
select {
|
|
case <-ch:
|
|
case <-t.C:
|
|
return g
|
|
case <-ctx.Done():
|
|
return g
|
|
}
|
|
}
|
|
}
|
|
|
|
func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string, error) {
|
|
j.ID = randomToken(8)
|
|
j.done = make(chan rackResult, 1)
|
|
h.mu.Lock()
|
|
h.work[rack] = append(h.work[rack], j)
|
|
if w, ok := h.wake[rack]; ok {
|
|
close(w)
|
|
delete(h.wake, rack)
|
|
}
|
|
h.mu.Unlock()
|
|
wait := 90 * time.Second
|
|
if j.Timeout > 0 {
|
|
wait = time.Duration(j.Timeout) * time.Second
|
|
}
|
|
t := time.NewTimer(wait)
|
|
defer t.Stop()
|
|
select {
|
|
case r := <-j.done:
|
|
if r.Error != "" {
|
|
return r.Output, errors.New(r.Error)
|
|
}
|
|
return r.Output, nil
|
|
case <-t.C:
|
|
case <-ctx.Done():
|
|
}
|
|
h.mu.Lock()
|
|
delete(h.flight, j.ID)
|
|
q := h.work[rack][:0]
|
|
for _, x := range h.work[rack] {
|
|
if x != j {
|
|
q = append(q, x)
|
|
}
|
|
}
|
|
h.work[rack] = q
|
|
h.mu.Unlock()
|
|
return "", fmt.Errorf("rack controller %s did not answer in time", rack)
|
|
}
|
|
|
|
func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*rackJob {
|
|
deadline := time.Now().Add(wait)
|
|
for {
|
|
h.mu.Lock()
|
|
if q := h.work[rack]; len(q) > 0 {
|
|
h.work[rack] = nil
|
|
for _, j := range q {
|
|
j.rack = rack
|
|
h.flight[j.ID] = j
|
|
}
|
|
h.mu.Unlock()
|
|
return q
|
|
}
|
|
w, ok := h.wake[rack]
|
|
if !ok {
|
|
w = make(chan struct{})
|
|
h.wake[rack] = w
|
|
}
|
|
h.mu.Unlock()
|
|
left := time.Until(deadline)
|
|
if left <= 0 {
|
|
return nil
|
|
}
|
|
t := time.NewTimer(left)
|
|
select {
|
|
case <-w:
|
|
case <-t.C:
|
|
case <-ctx.Done():
|
|
t.Stop()
|
|
return nil
|
|
}
|
|
t.Stop()
|
|
}
|
|
}
|
|
|
|
func (h *rackHub) finish(rack string, r rackResult) bool {
|
|
h.mu.Lock()
|
|
j, ok := h.flight[r.ID]
|
|
if ok && j.rack != rack {
|
|
ok = false
|
|
} else {
|
|
delete(h.flight, r.ID)
|
|
}
|
|
h.mu.Unlock()
|
|
if ok {
|
|
j.done <- r
|
|
}
|
|
return ok
|
|
}
|
|
|
|
// ---------------------------------------------------------------- routing machines to racks
|
|
|
|
func (a *App) racksSnapshot() map[string]RackController {
|
|
a.netMu.RLock()
|
|
defer a.netMu.RUnlock()
|
|
out := make(map[string]RackController, len(a.racks))
|
|
for k, v := range a.racks {
|
|
out[k] = v
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (a *App) subnetsSnapshot() []Subnet {
|
|
a.netMu.RLock()
|
|
defer a.netMu.RUnlock()
|
|
return a.subnets
|
|
}
|
|
|
|
// reloadNet refreshes the cached subnets and rack controllers after a change.
|
|
func (a *App) reloadNet(ctx context.Context) {
|
|
if sn, err := a.store.Subnets(ctx); err == nil {
|
|
a.netMu.Lock()
|
|
a.subnets = sn
|
|
a.netMu.Unlock()
|
|
}
|
|
a.reloadRacks(ctx)
|
|
}
|
|
|
|
func (a *App) reloadRacks(ctx context.Context) {
|
|
rs, err := a.store.RackControllers(ctx)
|
|
if err != nil {
|
|
log.Printf("racks: %v", err)
|
|
return
|
|
}
|
|
m := map[string]RackController{}
|
|
for _, r := range rs {
|
|
m[r.Name] = r
|
|
}
|
|
a.netMu.Lock()
|
|
a.racks = m
|
|
a.netMu.Unlock()
|
|
}
|
|
|
|
func rackOnline(r RackController, local string) bool {
|
|
return r.Name == local || (r.LastSeen != nil && time.Since(*r.LastSeen) < 90*time.Second)
|
|
}
|
|
|
|
// rackOf names the rack controller serving a machine: the one it last came
|
|
// through, else the one serving the VLAN of its management or boot address,
|
|
// else the embedded one.
|
|
func (a *App) rackOf(m *Machine) string {
|
|
racks := a.racksSnapshot()
|
|
if m.RackController != "" {
|
|
if _, ok := racks[m.RackController]; ok {
|
|
return m.RackController
|
|
}
|
|
}
|
|
if subnets := a.subnetsSnapshot(); len(subnets) > 0 {
|
|
ips := []string{deref(m.MgmtIP)}
|
|
if b := bootNIC(machineNICs(m), m.MACs); b != nil {
|
|
ips = append(ips, b.IPv4...)
|
|
}
|
|
for _, ip := range ips {
|
|
if sn := subnetFor(subnets, ip); sn != nil && sn.PrimaryRack != "" {
|
|
if _, ok := racks[sn.PrimaryRack]; ok {
|
|
return sn.PrimaryRack
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return a.cfg.RackName
|
|
}
|
|
|
|
func (a *App) rackURL(name string) string {
|
|
if name == "" || name == a.cfg.RackName {
|
|
return a.cfg.ServerURL
|
|
}
|
|
if r, ok := a.racksSnapshot()[name]; ok && r.URL != "" {
|
|
return strings.TrimRight(r.URL, "/")
|
|
}
|
|
return a.cfg.ServerURL
|
|
}
|
|
|
|
// urlFor is the base URL a machine uses for images and the node API.
|
|
func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) }
|
|
|
|
// appendFor is commission_append with the server URL of the machine's rack.
|
|
func (a *App) appendFor(base string) string {
|
|
s := a.cfg.CommissionAppend
|
|
if base != a.cfg.ServerURL {
|
|
s = strings.ReplaceAll(s, a.cfg.ServerURL, base)
|
|
}
|
|
if t := a.agentTLSArgs(base); len(t) > 0 && !strings.Contains(s, "rollcall.tls=") {
|
|
s += " " + strings.Join(t, " ")
|
|
}
|
|
return s
|
|
}
|
|
|
|
// rackAddresses are the rack controllers' own addresses, never handed to machines.
|
|
func (a *App) rackAddresses() []string {
|
|
var out []string
|
|
for _, r := range a.racksSnapshot() {
|
|
for _, i := range r.Interfaces {
|
|
out = append(out, i.Addrs...)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ---------------------------------------------------------------- BMC through the machine's rack
|
|
|
|
// ---------------------------------------------------------------- desired state
|
|
|
|
const defaultPXETmpl = `# generated by rollcall for rack controller %s
|
|
# unknown machines land here; known ones get a per-MAC file (01-xx-xx-…)
|
|
DEFAULT menu.c32
|
|
PROMPT 0
|
|
TIMEOUT 80
|
|
MENU TITLE rollcall · %s · unknown machine: commissioning in 8s
|
|
|
|
LABEL commission
|
|
MENU LABEL Commission this machine (inventory + BMC account)
|
|
MENU DEFAULT
|
|
KERNEL rollcall/vmlinuz
|
|
INITRD rollcall/initrd.img
|
|
APPEND %s
|
|
|
|
LABEL commission-debug
|
|
MENU LABEL Commission, but stay up afterwards (ssh root@<ip>)
|
|
KERNEL rollcall/vmlinuz
|
|
INITRD rollcall/initrd.img
|
|
APPEND %s rollcall.nopoweroff
|
|
|
|
LABEL local
|
|
MENU LABEL Boot from local disk
|
|
LOCALBOOT 0
|
|
`
|
|
|
|
func (a *App) rackState(ctx context.Context, name string) (*rackState, error) {
|
|
gen := a.hub.current()
|
|
st := &rackState{Gen: gen}
|
|
if rc, ok := a.racksSnapshot()[name]; ok {
|
|
st.Update = a.rackUpdateFor(rc)
|
|
}
|
|
base := a.rackURL(name)
|
|
app := a.appendFor(base)
|
|
st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app)
|
|
var err error
|
|
if st.PXE, err = a.store.PXEFiles(ctx, name); err != nil {
|
|
return nil, err
|
|
}
|
|
subnets, err := a.store.Subnets(ctx)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
rc := a.racksSnapshot()[name]
|
|
rc.Name = name
|
|
dnsDir := a.cfg.DnsmasqDir
|
|
if dnsDir == "" {
|
|
dnsDir = "/var/lib/rollcall/dnsmasq"
|
|
}
|
|
if conf, ok := dnsmasqConfig(rc, subnets, dnsDir); ok {
|
|
st.DnsmasqConf = conf
|
|
}
|
|
if st.HostsDHCP, st.Hosts, err = a.hostsFiles(ctx, name, subnets); err != nil {
|
|
return nil, err
|
|
}
|
|
st.BootFiles = []string{"rollcall/vmlinuz", "rollcall/initrd.img"}
|
|
if ents, err := os.ReadDir(a.cfg.ImagesDir); err == nil {
|
|
for _, e := range ents {
|
|
if e.IsDir() {
|
|
st.BootFiles = append(st.BootFiles, "images/"+e.Name()+"/vmlinuz", "images/"+e.Name()+"/initrd.img")
|
|
}
|
|
}
|
|
}
|
|
return st, nil
|
|
}
|
|
|
|
// hostsFiles: static DHCP leases for the machines this rack serves, and names
|
|
// for every machine: the addresses on the interface carrying the management
|
|
// address get the plain name (A and AAAA), others name-<interface>, IB name-ib.
|
|
var nonHostChars = regexp.MustCompile(`[^a-z0-9-]+`)
|
|
|
|
func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) {
|
|
ms, err := a.store.List(ctx)
|
|
if err != nil {
|
|
return "", "", err
|
|
}
|
|
all, _ := a.store.IPs(ctx, "", "")
|
|
byMachine := map[string][]IPAddr{}
|
|
for _, x := range all {
|
|
if x.MachineID != "" {
|
|
byMachine[x.MachineID] = append(byMachine[x.MachineID], x)
|
|
}
|
|
}
|
|
var dhcp, hosts strings.Builder
|
|
dhcp.WriteString("# generated by rollcall\n")
|
|
hosts.WriteString("# generated by rollcall\n")
|
|
domainOf := func(ip string) string {
|
|
if sn := subnetFor(subnets, ip); sn != nil && sn.Domain != "" {
|
|
return sn.Domain
|
|
}
|
|
return str(a.cfg.ClusterVars["domain"])
|
|
}
|
|
line := func(ip, name string) {
|
|
if d := domainOf(ip); d != "" {
|
|
fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name)
|
|
} else {
|
|
fmt.Fprintf(&hosts, "%s %s\n", ip, name)
|
|
}
|
|
}
|
|
// proxy subnets: the site's DHCP server hands out the addresses, so no dhcp-host
|
|
// lines for them (they would do nothing, and the "known" tag they give a client
|
|
// helps dnsmasq 2.91 build a cyclic tag list in its PXE proxy code: busy loop)
|
|
proxied := func(ip string) bool {
|
|
sn := subnetFor(subnets, ip)
|
|
return sn != nil && sn.DHCPMode == "proxy"
|
|
}
|
|
sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) })
|
|
for _, m := range ms {
|
|
if m.NodeName == nil || len(m.MACs) == 0 {
|
|
continue
|
|
}
|
|
name, mgmt, ibip := *m.NodeName, deref(m.MgmtIP), deref(m.IBIP)
|
|
mine := a.rackOf(m) == rack
|
|
mgmtIface := ""
|
|
for _, x := range byMachine[m.ID] {
|
|
if x.IP == mgmt {
|
|
mgmtIface = x.Iface
|
|
}
|
|
}
|
|
if mgmt != "" {
|
|
line(mgmt, name)
|
|
if mine && familyOf(mgmt+"/32") == 4 && !proxied(mgmt) {
|
|
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], mgmt, name)
|
|
}
|
|
}
|
|
for _, x := range byMachine[m.ID] {
|
|
if x.IP == mgmt || x.IP == ibip {
|
|
continue
|
|
}
|
|
v6 := strings.Contains(x.IP, ":")
|
|
switch {
|
|
case x.Iface == mgmtIface && mgmtIface != "":
|
|
line(x.IP, name)
|
|
if v6 && mine && !proxied(x.IP) {
|
|
fmt.Fprintf(&dhcp, "%s,[%s],%s\n", m.MACs[0], x.IP, name)
|
|
}
|
|
case x.Iface != "":
|
|
if sfx := strings.Trim(nonHostChars.ReplaceAllString(strings.ToLower(x.Iface), "-"), "-"); sfx != "" {
|
|
line(x.IP, name+"-"+sfx)
|
|
}
|
|
}
|
|
}
|
|
if ibip != "" {
|
|
fmt.Fprintf(&hosts, "%s %s-ib\n", ibip, name)
|
|
}
|
|
}
|
|
return dhcp.String(), hosts.String(), nil
|
|
}
|
|
|
|
// writeIfChanged replaces a file only when its content differs (path units watch these).
|
|
func writeIfChanged(p, content string, mode os.FileMode) (bool, error) {
|
|
if old, err := os.ReadFile(p); err == nil && string(old) == content {
|
|
return false, nil
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
return false, err
|
|
}
|
|
if err := os.WriteFile(p+".tmp", []byte(content), mode); err != nil {
|
|
return false, err
|
|
}
|
|
return true, os.Rename(p+".tmp", p)
|
|
}
|
|
|
|
// applyRackState writes a rack's state: PXE files under tftp/pxelinux.cfg
|
|
// (stale per-MAC files removed), the dnsmasq configuration and host files.
|
|
func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err error) {
|
|
dir := filepath.Join(tftp, "pxelinux.cfg")
|
|
note := func(p string, ch bool, e error) {
|
|
if e != nil && err == nil {
|
|
err = e
|
|
}
|
|
if ch {
|
|
changed = append(changed, p)
|
|
}
|
|
}
|
|
if st.DefaultPXE != "" {
|
|
ch, e := writeIfChanged(filepath.Join(dir, "default"), st.DefaultPXE, 0o644)
|
|
note("default", ch, e)
|
|
}
|
|
for name, body := range st.PXE {
|
|
if !pxeNameRe.MatchString(name) {
|
|
note(name, false, fmt.Errorf("not a pxelinux file name"))
|
|
continue
|
|
}
|
|
ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644)
|
|
note(name, ch, e)
|
|
}
|
|
if olds, e := filepath.Glob(filepath.Join(dir, "01-*")); e == nil {
|
|
for _, p := range olds {
|
|
if _, keep := st.PXE[filepath.Base(p)]; !keep && !strings.HasSuffix(p, ".tmp") {
|
|
note("-"+filepath.Base(p), true, os.Remove(p))
|
|
}
|
|
}
|
|
}
|
|
if dnsDir != "" {
|
|
ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644)
|
|
note("hosts.dhcp", ch, e)
|
|
ch, e = writeIfChanged(filepath.Join(dnsDir, "hosts"), st.Hosts, 0o644)
|
|
note("hosts", ch, e)
|
|
conf := filepath.Join(dnsDir, "conf.d", "rollcall.conf")
|
|
if st.DnsmasqConf != "" {
|
|
ch, e = writeIfChanged(conf, st.DnsmasqConf+dnsmasqUpstream("/run/systemd/resolve/stub-resolv.conf", "/etc/resolv.conf", localAddrs()), 0o644)
|
|
note("dnsmasq.conf", ch, e)
|
|
} else if _, e := os.Stat(conf); e == nil {
|
|
note("-dnsmasq.conf", true, os.Remove(conf))
|
|
}
|
|
}
|
|
return changed, err
|
|
}
|
|
|
|
// localRackLoop is the embedded rack controller: it applies this host's state on every change.
|
|
func (a *App) localRackLoop() {
|
|
apply := func() {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
defer cancel()
|
|
st, err := a.rackState(ctx, a.cfg.RackName)
|
|
if err != nil {
|
|
log.Printf("embedded rack: %v", err)
|
|
return
|
|
}
|
|
if ch, err := applyRackState(st, a.cfg.TFTPRoot, a.cfg.DnsmasqDir); err != nil {
|
|
log.Printf("embedded rack: %v", err)
|
|
} else if len(ch) > 0 {
|
|
log.Printf("embedded rack %s: wrote %s", a.cfg.RackName, strings.Join(ch, " "))
|
|
}
|
|
_ = a.store.TouchRackController(ctx, a.cfg.RackName)
|
|
}
|
|
apply()
|
|
tick := time.NewTicker(60 * time.Second)
|
|
for {
|
|
select {
|
|
case <-a.hub.kick:
|
|
time.Sleep(150 * time.Millisecond) // let a burst of changes settle
|
|
select {
|
|
case <-a.hub.kick:
|
|
default:
|
|
}
|
|
case <-tick.C:
|
|
}
|
|
apply()
|
|
}
|
|
}
|
|
|
|
func localInterfaces() []RackIface {
|
|
var out []RackIface
|
|
ifs, _ := net.Interfaces()
|
|
for _, i := range ifs {
|
|
if i.Flags&net.FlagLoopback != 0 {
|
|
continue
|
|
}
|
|
ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()}
|
|
addrs, _ := i.Addrs()
|
|
for _, ad := range addrs {
|
|
if ipn, ok := ad.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() {
|
|
ri.Addrs = append(ri.Addrs, ipn.String())
|
|
}
|
|
}
|
|
if len(ri.Addrs) > 0 {
|
|
out = append(out, ri)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// startRacks registers the embedded rack controller, carries over what an
|
|
// older version kept on disk or in config, and starts the local apply loop.
|
|
func (a *App) startRacks(ctx context.Context) error {
|
|
if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, Arch: runtime.GOARCH,
|
|
Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil {
|
|
return err
|
|
}
|
|
a.reloadNet(ctx)
|
|
// per-MAC files written before pxe_files existed
|
|
var n int
|
|
_ = a.store.pool.QueryRow(ctx, `SELECT count(*) FROM pxe_files`).Scan(&n)
|
|
if n == 0 {
|
|
files, _ := filepath.Glob(filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-*"))
|
|
ms, _ := a.store.List(ctx)
|
|
for _, f := range files {
|
|
b, err := os.ReadFile(f)
|
|
if err != nil {
|
|
continue
|
|
}
|
|
mac := strings.ReplaceAll(strings.TrimPrefix(filepath.Base(f), "01-"), "-", ":")
|
|
owner := ""
|
|
for _, m := range ms {
|
|
if hasTag(m.MACs, mac) {
|
|
owner = m.ID
|
|
}
|
|
}
|
|
_ = a.store.PutPXE(ctx, mac, owner, a.cfg.RackName, string(b))
|
|
}
|
|
if len(files) > 0 {
|
|
log.Printf("imported %d existing PXE files", len(files))
|
|
}
|
|
}
|
|
a.adoptIPs(ctx)
|
|
go a.localRackLoop()
|
|
return nil
|
|
}
|
|
|
|
// adoptIPs records addresses machines already have (node pools, before a
|
|
// subnet was modeled) so IPAM never hands them out twice.
|
|
func (a *App) adoptIPs(ctx context.Context) {
|
|
subnets, err := a.store.Subnets(ctx)
|
|
if err != nil || len(subnets) == 0 {
|
|
return
|
|
}
|
|
ms, _ := a.store.List(ctx)
|
|
for _, m := range ms {
|
|
for _, ip := range []string{deref(m.MgmtIP), deref(m.IBIP)} {
|
|
if sn := subnetFor(subnets, ip); sn != nil {
|
|
iface := ""
|
|
if b := bootNIC(machineNICs(m), m.MACs); b != nil && ip == deref(m.MgmtIP) {
|
|
iface = b.Name
|
|
}
|
|
_, _ = a.claimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: m.ID, Iface: iface, Kind: "auto", Hostname: deref(m.NodeName)})
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------- PXE files through the rack
|
|
|
|
// writePXEBody records a machine's PXE file for its rack controller.
|
|
func (a *App) writePXEBody(m *Machine, body string) error {
|
|
rack := a.rackOf(m)
|
|
for _, mac := range m.MACs {
|
|
if err := a.store.PutPXE(context.Background(), normMAC(mac), m.ID, rack, body); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
a.hub.bump()
|
|
return nil
|
|
}
|
|
|
|
func (a *App) removePXE(m *Machine) {
|
|
for _, mac := range m.MACs {
|
|
_ = a.store.DeletePXE(context.Background(), normMAC(mac))
|
|
}
|
|
a.hub.bump()
|
|
}
|
|
|
|
// exportDnsmasq: host files and dnsmasq configuration follow from the state now.
|
|
func (a *App) exportDnsmasq(ctx context.Context) error {
|
|
a.hub.bump()
|
|
return nil
|
|
}
|
|
|
|
// ---------------------------------------------------------------- HTTP: rack controllers
|
|
|
|
func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
if name := a.peerRack(r); name != "" {
|
|
next(w, r, name)
|
|
return
|
|
}
|
|
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
|
|
if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" {
|
|
fail(w, 401, errors.New("rack controller certificate (or legacy secret) required"))
|
|
return
|
|
}
|
|
next(w, r, name)
|
|
}
|
|
}
|
|
|
|
// fromRack names the rack controller a node request came through (it proxies them).
|
|
func (a *App) fromRack(r *http.Request) string {
|
|
if name := a.peerRack(r); name != "" {
|
|
return name
|
|
}
|
|
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
|
|
if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 {
|
|
return name
|
|
}
|
|
return a.cfg.RackName
|
|
}
|
|
|
|
func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack string) {
|
|
var rc RackController
|
|
if err := json.NewDecoder(r.Body).Decode(&rc); err != nil {
|
|
fail(w, 400, err)
|
|
return
|
|
}
|
|
rc.Name, rc.Kind = rack, "remote"
|
|
if rack == a.cfg.RackName {
|
|
fail(w, 409, fmt.Errorf("%s is the embedded rack controller's name", rack))
|
|
return
|
|
}
|
|
if rc.URL == "" {
|
|
fail(w, 400, errors.New("url is required"))
|
|
return
|
|
}
|
|
rc.TLSURL = validTLSURL(rc) // its own host only
|
|
if err := a.store.PutRackController(r.Context(), rc); err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
a.reloadRacks(r.Context())
|
|
a.hub.bump()
|
|
log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces))
|
|
writeJSON(w, 200, obj{"ok": true, "gen": a.hub.current()})
|
|
}
|
|
|
|
func (a *App) handleRackState(w http.ResponseWriter, r *http.Request, rack string) {
|
|
if _, ok := a.racksSnapshot()[rack]; !ok {
|
|
fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack))
|
|
return
|
|
}
|
|
since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
|
|
_ = a.store.TouchRackController(r.Context(), rack)
|
|
a.hub.waitChange(r.Context(), since, 25*time.Second)
|
|
_ = a.store.TouchRackController(r.Context(), rack)
|
|
go a.reloadRacks(context.Background()) // keep last_seen fresh for online checks
|
|
st, err := a.rackState(r.Context(), rack)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, st)
|
|
}
|
|
|
|
func (a *App) handleRackWork(w http.ResponseWriter, r *http.Request, rack string) {
|
|
jobs := a.hub.take(r.Context(), rack, 25*time.Second)
|
|
if jobs == nil {
|
|
jobs = []*rackJob{}
|
|
}
|
|
writeJSON(w, 200, jobs)
|
|
}
|
|
|
|
func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack string) {
|
|
var res rackResult
|
|
if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
|
|
fail(w, 400, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, obj{"accepted": a.hub.finish(rack, res)})
|
|
}
|
|
|
|
func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) {
|
|
rs, err := a.store.RackControllers(r.Context())
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
ms, _ := a.store.List(r.Context())
|
|
for i := range rs {
|
|
rs[i].Online = rackOnline(rs[i], a.cfg.RackName)
|
|
for _, m := range ms {
|
|
if a.rackOf(m) == rs[i].Name {
|
|
rs[i].Machines++
|
|
}
|
|
}
|
|
}
|
|
writeJSON(w, 200, rs)
|
|
}
|
|
|
|
func (a *App) handleControllerDelete(w http.ResponseWriter, r *http.Request) {
|
|
name := r.PathValue("name")
|
|
if name == a.cfg.RackName {
|
|
fail(w, 409, errors.New("the embedded rack controller cannot be removed"))
|
|
return
|
|
}
|
|
if err := a.store.DeleteRackController(r.Context(), name); err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
a.reloadRacks(r.Context())
|
|
a.hub.bump()
|
|
writeJSON(w, 200, obj{"deleted": name})
|
|
}
|
|
|
|
// handleRackDHCP shows a rack's generated dnsmasq configuration (admin).
|
|
func (a *App) handleRackDHCP(w http.ResponseWriter, r *http.Request) {
|
|
st, err := a.rackState(r.Context(), r.PathValue("name"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
|
if st.DnsmasqConf == "" {
|
|
fmt.Fprintf(w, "# rack controller %s serves no VLAN with dhcp on: rollcall leaves dnsmasq's configuration alone\n", r.PathValue("name"))
|
|
}
|
|
fmt.Fprint(w, st.DnsmasqConf, "\n# --- hosts.dhcp\n", st.HostsDHCP)
|
|
}
|
|
|
|
// dnsmasqUpstream tells dnsmasq where to forward names it does not answer
|
|
// itself. Without it Debian's dnsmasq looks for a resolvconf-made file that
|
|
// usually does not exist and refuses every outside name, so nodes could not
|
|
// resolve mirrors or site services. With systemd-resolved the stub is used, so
|
|
// its per-link routing (split DNS such as ~int.example.com) keeps working;
|
|
// otherwise the host's resolv.conf servers, never ourselves.
|
|
func dnsmasqUpstream(stub, resolv string, own []string) string {
|
|
if _, err := os.Stat(stub); err == nil {
|
|
return "# upstream: systemd-resolved on this host (keeps its split DNS)\nno-resolv\nserver=127.0.0.53\n"
|
|
}
|
|
b, err := os.ReadFile(resolv)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
mine := map[string]bool{}
|
|
for _, a := range own {
|
|
mine[stripPrefix(a)] = true
|
|
}
|
|
var servers []string
|
|
for _, line := range strings.Split(string(b), "\n") {
|
|
f := strings.Fields(line)
|
|
if len(f) < 2 || f[0] != "nameserver" {
|
|
continue
|
|
}
|
|
ip, ok := parseAddr(f[1])
|
|
if !ok || mine[ip.String()] || (ip.IsLoopback() && ip.String() != "127.0.0.53") {
|
|
continue
|
|
}
|
|
servers = append(servers, ip.String())
|
|
}
|
|
if len(servers) == 0 {
|
|
return ""
|
|
}
|
|
out := "# upstream: this host's resolv.conf\nno-resolv\n"
|
|
for _, sv := range servers {
|
|
out += "server=" + sv + "\n"
|
|
}
|
|
return out
|
|
}
|
|
|
|
// localAddrs: this host's interface addresses (CIDR form).
|
|
func localAddrs() []string {
|
|
var out []string
|
|
addrs, _ := net.InterfaceAddrs()
|
|
for _, a := range addrs {
|
|
out = append(out, a.String())
|
|
}
|
|
return out
|
|
}
|