package main // Region side of the region/rack split, after MAAS. The region (rollcall // serve) owns the database and computes everything; rack controllers sit next // to the machines and do what has to happen on the machines' networks: DHCP // and TFTP (dnsmasq), the per-MAC PXE files, BMC commands on a BMC network the // region may not reach, and serving boot and OS images from a local cache. // // A rack controller pulls: it long-polls its desired state (PXE files, // dnsmasq configuration, host files) and a queue of commands (ipmitool), so // only rack → region connectivity is needed. The region host itself runs an // embedded rack controller that applies the same state in-process, which is // what a single-server installation uses. import ( "context" "crypto/subtle" "encoding/json" "errors" "fmt" "log" "net" "net/http" "os" "path/filepath" "regexp" "runtime" "sort" "strconv" "strings" "sync" "time" ) type RackIface struct { Name string `json:"name"` MAC string `json:"mac"` Addrs []string `json:"addrs"` } type RackController struct { Name string `json:"name"` URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1 TLSURL string `json:"tls_url,omitempty"` // the same host over HTTPS, when it serves it (agents) Kind string `json:"kind"` // embedded | remote Version string `json:"version"` Arch string `json:"arch,omitempty"` // GOARCH, for self-update Interfaces []RackIface `json:"interfaces"` DHCPInterfaces []string `json:"dhcp_interfaces"` TFTPRoot string `json:"tftp_root"` LastSeen *time.Time `json:"last_seen"` Registered time.Time `json:"registered"` Online bool `json:"online"` Machines int `json:"machines"` } // rackState is everything a rack controller writes to disk. type rackState struct { Gen int64 `json:"gen"` DefaultPXE string `json:"default_pxe"` PXE map[string]string `json:"pxe"` // pxelinux.cfg file name (01-aa-bb-…) → content DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here HostsDHCP string `json:"hosts_dhcp"` Hosts string `json:"hosts"` BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run } type rackJob struct { ID string `json:"id"` Kind string `json:"kind"` // power Driver string `json:"driver"` Address string `json:"address"` Port int `json:"port"` User string `json:"user"` Password string `json:"password"` Extra map[string]string `json:"extra"` Op string `json:"op"` Arg string `json:"arg"` Timeout int `json:"timeout,omitempty"` // seconds the region waits (default 90) done chan rackResult rack string // who took it: only that rack may answer } type rackResult struct { ID string `json:"id"` Output string `json:"output"` Error string `json:"error"` } // ---------------------------------------------------------------- store func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`) if err != nil { return nil, err } defer rows.Close() out := []RackController{} for rows.Next() { var r RackController var ifs, dif []byte if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil { return nil, err } _ = json.Unmarshal(ifs, &r.Interfaces) _ = json.Unmarshal(dif, &r.DHCPInterfaces) out = append(out, r) } return out, rows.Err() } func (s *Store) PutRackController(ctx context.Context, r RackController) error { if r.Interfaces == nil { r.Interfaces = []RackIface{} } if r.DHCPInterfaces == nil { r.DHCPInterfaces = []string{} } ifs, _ := json.Marshal(r.Interfaces) dif, _ := json.Marshal(r.DHCPInterfaces) _, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch, tls_url) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''),nullif($9,'')) ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,''), tls_url=nullif($9,'')`, r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch, r.TLSURL) return err } func (s *Store) TouchRackController(ctx context.Context, name string) error { _, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET last_seen=now() WHERE name=$1`, name) return err } func (s *Store) DeleteRackController(ctx context.Context, name string) error { _, err := s.pool.Exec(ctx, `DELETE FROM rack_controllers WHERE name=$1`, name) return err } func (s *Store) SetRackController(ctx context.Context, id, rack string) error { _, err := s.pool.Exec(ctx, `UPDATE machines SET rack_controller=nullif($2,'') WHERE id=$1`, id, rack) return err } func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error { // a MAC another machine owns is not taken over: a box claiming a deployed // machine's MAC would otherwise replace that machine's boot file tag, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4) ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now() WHERE pxe_files.machine_id=$2`, mac, machineID, rack, content) if err == nil && tag.RowsAffected() == 0 { return fmt.Errorf("MAC %s belongs to another machine; delete that machine first", mac) } return err } func (s *Store) DeletePXE(ctx context.Context, mac string) error { _, err := s.pool.Exec(ctx, `DELETE FROM pxe_files WHERE mac=$1`, mac) return err } func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, error) { rows, err := s.pool.Query(ctx, `SELECT mac, content FROM pxe_files WHERE rack=$1`, rack) if err != nil { return nil, err } defer rows.Close() out := map[string]string{} for rows.Next() { var mac, content string if err := rows.Scan(&mac, &content); err != nil { return nil, err } out[pxeFileName(mac)] = content } return out, rows.Err() } var pxeNameRe = regexp.MustCompile(`^01(-[0-9a-f]{2}){6,20}$`) func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") } // ---------------------------------------------------------------- hub: state generations and the command queue type rackHub struct { mu sync.Mutex gen int64 changed chan struct{} work map[string][]*rackJob wake map[string]chan struct{} flight map[string]*rackJob kick chan struct{} // the embedded rack applies on this } func newRackHub() *rackHub { return &rackHub{gen: time.Now().UnixNano(), changed: make(chan struct{}), work: map[string][]*rackJob{}, wake: map[string]chan struct{}{}, flight: map[string]*rackJob{}, kick: make(chan struct{}, 1)} } func (h *rackHub) current() int64 { h.mu.Lock() defer h.mu.Unlock() return h.gen } // bump says the desired state of some rack changed. func (h *rackHub) bump() { h.mu.Lock() h.gen++ close(h.changed) h.changed = make(chan struct{}) h.mu.Unlock() select { case h.kick <- struct{}{}: default: } } // waitChange blocks until the generation moves past since, or the wait is over. func (h *rackHub) waitChange(ctx context.Context, since int64, wait time.Duration) int64 { t := time.NewTimer(wait) defer t.Stop() for { h.mu.Lock() g, ch := h.gen, h.changed h.mu.Unlock() if g != since { return g } select { case <-ch: case <-t.C: return g case <-ctx.Done(): return g } } } func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string, error) { j.ID = randomToken(8) j.done = make(chan rackResult, 1) h.mu.Lock() h.work[rack] = append(h.work[rack], j) if w, ok := h.wake[rack]; ok { close(w) delete(h.wake, rack) } h.mu.Unlock() wait := 90 * time.Second if j.Timeout > 0 { wait = time.Duration(j.Timeout) * time.Second } t := time.NewTimer(wait) defer t.Stop() select { case r := <-j.done: if r.Error != "" { return r.Output, errors.New(r.Error) } return r.Output, nil case <-t.C: case <-ctx.Done(): } h.mu.Lock() delete(h.flight, j.ID) q := h.work[rack][:0] for _, x := range h.work[rack] { if x != j { q = append(q, x) } } h.work[rack] = q h.mu.Unlock() return "", fmt.Errorf("rack controller %s did not answer in time", rack) } func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*rackJob { deadline := time.Now().Add(wait) for { h.mu.Lock() if q := h.work[rack]; len(q) > 0 { h.work[rack] = nil for _, j := range q { j.rack = rack h.flight[j.ID] = j } h.mu.Unlock() return q } w, ok := h.wake[rack] if !ok { w = make(chan struct{}) h.wake[rack] = w } h.mu.Unlock() left := time.Until(deadline) if left <= 0 { return nil } t := time.NewTimer(left) select { case <-w: case <-t.C: case <-ctx.Done(): t.Stop() return nil } t.Stop() } } func (h *rackHub) finish(rack string, r rackResult) bool { h.mu.Lock() j, ok := h.flight[r.ID] if ok && j.rack != rack { ok = false } else { delete(h.flight, r.ID) } h.mu.Unlock() if ok { j.done <- r } return ok } // ---------------------------------------------------------------- routing machines to racks func (a *App) racksSnapshot() map[string]RackController { a.netMu.RLock() defer a.netMu.RUnlock() out := make(map[string]RackController, len(a.racks)) for k, v := range a.racks { out[k] = v } return out } func (a *App) subnetsSnapshot() []Subnet { a.netMu.RLock() defer a.netMu.RUnlock() return a.subnets } // reloadNet refreshes the cached subnets and rack controllers after a change. func (a *App) reloadNet(ctx context.Context) { if sn, err := a.store.Subnets(ctx); err == nil { a.netMu.Lock() a.subnets = sn a.netMu.Unlock() } a.reloadRacks(ctx) } func (a *App) reloadRacks(ctx context.Context) { rs, err := a.store.RackControllers(ctx) if err != nil { log.Printf("racks: %v", err) return } m := map[string]RackController{} for _, r := range rs { m[r.Name] = r } a.netMu.Lock() a.racks = m a.netMu.Unlock() } func rackOnline(r RackController, local string) bool { return r.Name == local || (r.LastSeen != nil && time.Since(*r.LastSeen) < 90*time.Second) } // rackOf names the rack controller serving a machine: the one it last came // through, else the one serving the VLAN of its management or boot address, // else the embedded one. func (a *App) rackOf(m *Machine) string { racks := a.racksSnapshot() if m.RackController != "" { if _, ok := racks[m.RackController]; ok { return m.RackController } } if subnets := a.subnetsSnapshot(); len(subnets) > 0 { ips := []string{deref(m.MgmtIP)} if b := bootNIC(machineNICs(m), m.MACs); b != nil { ips = append(ips, b.IPv4...) } for _, ip := range ips { if sn := subnetFor(subnets, ip); sn != nil && sn.PrimaryRack != "" { if _, ok := racks[sn.PrimaryRack]; ok { return sn.PrimaryRack } } } } return a.cfg.RackName } func (a *App) rackURL(name string) string { if name == "" || name == a.cfg.RackName { return a.cfg.ServerURL } if r, ok := a.racksSnapshot()[name]; ok && r.URL != "" { return strings.TrimRight(r.URL, "/") } return a.cfg.ServerURL } // urlFor is the base URL a machine uses for images and the node API. func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) } // appendFor is commission_append with the server URL of the machine's rack. func (a *App) appendFor(base string) string { s := a.cfg.CommissionAppend if base != a.cfg.ServerURL { s = strings.ReplaceAll(s, a.cfg.ServerURL, base) } if t := a.agentTLSArgs(base); len(t) > 0 && !strings.Contains(s, "rollcall.tls=") { s += " " + strings.Join(t, " ") } return s } // rackAddresses are the rack controllers' own addresses, never handed to machines. func (a *App) rackAddresses() []string { var out []string for _, r := range a.racksSnapshot() { for _, i := range r.Interfaces { out = append(out, i.Addrs...) } } return out } // ---------------------------------------------------------------- BMC through the machine's rack // ---------------------------------------------------------------- desired state const defaultPXETmpl = `# generated by rollcall for rack controller %s # unknown machines land here; known ones get a per-MAC file (01-xx-xx-…) DEFAULT menu.c32 PROMPT 0 TIMEOUT 80 MENU TITLE rollcall · %s · unknown machine: commissioning in 8s LABEL commission MENU LABEL Commission this machine (inventory + BMC account) MENU DEFAULT KERNEL rollcall/vmlinuz INITRD rollcall/initrd.img APPEND %s LABEL commission-debug MENU LABEL Commission, but stay up afterwards (ssh root@) KERNEL rollcall/vmlinuz INITRD rollcall/initrd.img APPEND %s rollcall.nopoweroff LABEL local MENU LABEL Boot from local disk LOCALBOOT 0 ` func (a *App) rackState(ctx context.Context, name string) (*rackState, error) { gen := a.hub.current() st := &rackState{Gen: gen} if rc, ok := a.racksSnapshot()[name]; ok { st.Update = a.rackUpdateFor(rc) } base := a.rackURL(name) app := a.appendFor(base) st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app) var err error if st.PXE, err = a.store.PXEFiles(ctx, name); err != nil { return nil, err } subnets, err := a.store.Subnets(ctx) if err != nil { return nil, err } rc := a.racksSnapshot()[name] rc.Name = name dnsDir := a.cfg.DnsmasqDir if dnsDir == "" { dnsDir = "/var/lib/rollcall/dnsmasq" } if conf, ok := dnsmasqConfig(rc, subnets, dnsDir); ok { st.DnsmasqConf = conf } if st.HostsDHCP, st.Hosts, err = a.hostsFiles(ctx, name, subnets); err != nil { return nil, err } st.BootFiles = []string{"rollcall/vmlinuz", "rollcall/initrd.img"} if ents, err := os.ReadDir(a.cfg.ImagesDir); err == nil { for _, e := range ents { if e.IsDir() { st.BootFiles = append(st.BootFiles, "images/"+e.Name()+"/vmlinuz", "images/"+e.Name()+"/initrd.img") } } } return st, nil } // hostsFiles: static DHCP leases for the machines this rack serves, and names // for every machine: the addresses on the interface carrying the management // address get the plain name (A and AAAA), others name-, IB name-ib. var nonHostChars = regexp.MustCompile(`[^a-z0-9-]+`) func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) { ms, err := a.store.List(ctx) if err != nil { return "", "", err } all, _ := a.store.IPs(ctx, "", "") byMachine := map[string][]IPAddr{} for _, x := range all { if x.MachineID != "" { byMachine[x.MachineID] = append(byMachine[x.MachineID], x) } } var dhcp, hosts strings.Builder dhcp.WriteString("# generated by rollcall\n") hosts.WriteString("# generated by rollcall\n") domainOf := func(ip string) string { if sn := subnetFor(subnets, ip); sn != nil && sn.Domain != "" { return sn.Domain } return str(a.cfg.ClusterVars["domain"]) } line := func(ip, name string) { if d := domainOf(ip); d != "" { fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name) } else { fmt.Fprintf(&hosts, "%s %s\n", ip, name) } } // proxy subnets: the site's DHCP server hands out the addresses, so no dhcp-host // lines for them (they would do nothing, and the "known" tag they give a client // helps dnsmasq 2.91 build a cyclic tag list in its PXE proxy code: busy loop) proxied := func(ip string) bool { sn := subnetFor(subnets, ip) return sn != nil && sn.DHCPMode == "proxy" } sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) }) for _, m := range ms { if m.NodeName == nil || len(m.MACs) == 0 { continue } name, mgmt, ibip := *m.NodeName, deref(m.MgmtIP), deref(m.IBIP) mine := a.rackOf(m) == rack mgmtIface := "" for _, x := range byMachine[m.ID] { if x.IP == mgmt { mgmtIface = x.Iface } } if mgmt != "" { line(mgmt, name) if mine && familyOf(mgmt+"/32") == 4 && !proxied(mgmt) { fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], mgmt, name) } } for _, x := range byMachine[m.ID] { if x.IP == mgmt || x.IP == ibip { continue } v6 := strings.Contains(x.IP, ":") switch { case x.Iface == mgmtIface && mgmtIface != "": line(x.IP, name) if v6 && mine && !proxied(x.IP) { fmt.Fprintf(&dhcp, "%s,[%s],%s\n", m.MACs[0], x.IP, name) } case x.Iface != "": if sfx := strings.Trim(nonHostChars.ReplaceAllString(strings.ToLower(x.Iface), "-"), "-"); sfx != "" { line(x.IP, name+"-"+sfx) } } } if ibip != "" { fmt.Fprintf(&hosts, "%s %s-ib\n", ibip, name) } } return dhcp.String(), hosts.String(), nil } // writeIfChanged replaces a file only when its content differs (path units watch these). func writeIfChanged(p, content string, mode os.FileMode) (bool, error) { if old, err := os.ReadFile(p); err == nil && string(old) == content { return false, nil } if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { return false, err } if err := os.WriteFile(p+".tmp", []byte(content), mode); err != nil { return false, err } return true, os.Rename(p+".tmp", p) } // applyRackState writes a rack's state: PXE files under tftp/pxelinux.cfg // (stale per-MAC files removed), the dnsmasq configuration and host files. func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err error) { dir := filepath.Join(tftp, "pxelinux.cfg") note := func(p string, ch bool, e error) { if e != nil && err == nil { err = e } if ch { changed = append(changed, p) } } if st.DefaultPXE != "" { ch, e := writeIfChanged(filepath.Join(dir, "default"), st.DefaultPXE, 0o644) note("default", ch, e) } for name, body := range st.PXE { if !pxeNameRe.MatchString(name) { note(name, false, fmt.Errorf("not a pxelinux file name")) continue } ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644) note(name, ch, e) } if olds, e := filepath.Glob(filepath.Join(dir, "01-*")); e == nil { for _, p := range olds { if _, keep := st.PXE[filepath.Base(p)]; !keep && !strings.HasSuffix(p, ".tmp") { note("-"+filepath.Base(p), true, os.Remove(p)) } } } if dnsDir != "" { ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644) note("hosts.dhcp", ch, e) ch, e = writeIfChanged(filepath.Join(dnsDir, "hosts"), st.Hosts, 0o644) note("hosts", ch, e) conf := filepath.Join(dnsDir, "conf.d", "rollcall.conf") if st.DnsmasqConf != "" { ch, e = writeIfChanged(conf, st.DnsmasqConf+dnsmasqUpstream("/run/systemd/resolve/stub-resolv.conf", "/etc/resolv.conf", localAddrs()), 0o644) note("dnsmasq.conf", ch, e) } else if _, e := os.Stat(conf); e == nil { note("-dnsmasq.conf", true, os.Remove(conf)) } } return changed, err } // localRackLoop is the embedded rack controller: it applies this host's state on every change. func (a *App) localRackLoop() { apply := func() { ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) defer cancel() st, err := a.rackState(ctx, a.cfg.RackName) if err != nil { log.Printf("embedded rack: %v", err) return } if ch, err := applyRackState(st, a.cfg.TFTPRoot, a.cfg.DnsmasqDir); err != nil { log.Printf("embedded rack: %v", err) } else if len(ch) > 0 { log.Printf("embedded rack %s: wrote %s", a.cfg.RackName, strings.Join(ch, " ")) } _ = a.store.TouchRackController(ctx, a.cfg.RackName) } apply() tick := time.NewTicker(60 * time.Second) for { select { case <-a.hub.kick: time.Sleep(150 * time.Millisecond) // let a burst of changes settle select { case <-a.hub.kick: default: } case <-tick.C: } apply() } } func localInterfaces() []RackIface { var out []RackIface ifs, _ := net.Interfaces() for _, i := range ifs { if i.Flags&net.FlagLoopback != 0 { continue } ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()} addrs, _ := i.Addrs() for _, ad := range addrs { if ipn, ok := ad.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() { ri.Addrs = append(ri.Addrs, ipn.String()) } } if len(ri.Addrs) > 0 { out = append(out, ri) } } return out } // startRacks registers the embedded rack controller, carries over what an // older version kept on disk or in config, and starts the local apply loop. func (a *App) startRacks(ctx context.Context) error { if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, Arch: runtime.GOARCH, Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil { return err } a.reloadNet(ctx) // per-MAC files written before pxe_files existed var n int _ = a.store.pool.QueryRow(ctx, `SELECT count(*) FROM pxe_files`).Scan(&n) if n == 0 { files, _ := filepath.Glob(filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-*")) ms, _ := a.store.List(ctx) for _, f := range files { b, err := os.ReadFile(f) if err != nil { continue } mac := strings.ReplaceAll(strings.TrimPrefix(filepath.Base(f), "01-"), "-", ":") owner := "" for _, m := range ms { if hasTag(m.MACs, mac) { owner = m.ID } } _ = a.store.PutPXE(ctx, mac, owner, a.cfg.RackName, string(b)) } if len(files) > 0 { log.Printf("imported %d existing PXE files", len(files)) } } a.adoptIPs(ctx) go a.localRackLoop() return nil } // adoptIPs records addresses machines already have (node pools, before a // subnet was modeled) so IPAM never hands them out twice. func (a *App) adoptIPs(ctx context.Context) { subnets, err := a.store.Subnets(ctx) if err != nil || len(subnets) == 0 { return } ms, _ := a.store.List(ctx) for _, m := range ms { for _, ip := range []string{deref(m.MgmtIP), deref(m.IBIP)} { if sn := subnetFor(subnets, ip); sn != nil { iface := "" if b := bootNIC(machineNICs(m), m.MACs); b != nil && ip == deref(m.MgmtIP) { iface = b.Name } _, _ = a.claimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: m.ID, Iface: iface, Kind: "auto", Hostname: deref(m.NodeName)}) } } } } // ---------------------------------------------------------------- PXE files through the rack // writePXEBody records a machine's PXE file for its rack controller. func (a *App) writePXEBody(m *Machine, body string) error { rack := a.rackOf(m) for _, mac := range m.MACs { if err := a.store.PutPXE(context.Background(), normMAC(mac), m.ID, rack, body); err != nil { return err } } a.hub.bump() return nil } func (a *App) removePXE(m *Machine) { for _, mac := range m.MACs { _ = a.store.DeletePXE(context.Background(), normMAC(mac)) } a.hub.bump() } // exportDnsmasq: host files and dnsmasq configuration follow from the state now. func (a *App) exportDnsmasq(ctx context.Context) error { a.hub.bump() return nil } // ---------------------------------------------------------------- HTTP: rack controllers func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { if name := a.peerRack(r); name != "" { next(w, r, name) return } sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack") if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" { fail(w, 401, errors.New("rack controller certificate (or legacy secret) required")) return } next(w, r, name) } } // fromRack names the rack controller a node request came through (it proxies them). func (a *App) fromRack(r *http.Request) string { if name := a.peerRack(r); name != "" { return name } sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack") if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 { return name } return a.cfg.RackName } func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack string) { var rc RackController if err := json.NewDecoder(r.Body).Decode(&rc); err != nil { fail(w, 400, err) return } rc.Name, rc.Kind = rack, "remote" if rack == a.cfg.RackName { fail(w, 409, fmt.Errorf("%s is the embedded rack controller's name", rack)) return } if rc.URL == "" { fail(w, 400, errors.New("url is required")) return } rc.TLSURL = validTLSURL(rc) // its own host only if err := a.store.PutRackController(r.Context(), rc); err != nil { fail(w, 500, err) return } a.reloadRacks(r.Context()) a.hub.bump() log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces)) writeJSON(w, 200, obj{"ok": true, "gen": a.hub.current()}) } func (a *App) handleRackState(w http.ResponseWriter, r *http.Request, rack string) { if _, ok := a.racksSnapshot()[rack]; !ok { fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack)) return } since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64) _ = a.store.TouchRackController(r.Context(), rack) a.hub.waitChange(r.Context(), since, 25*time.Second) _ = a.store.TouchRackController(r.Context(), rack) go a.reloadRacks(context.Background()) // keep last_seen fresh for online checks st, err := a.rackState(r.Context(), rack) if err != nil { fail(w, 500, err) return } writeJSON(w, 200, st) } func (a *App) handleRackWork(w http.ResponseWriter, r *http.Request, rack string) { jobs := a.hub.take(r.Context(), rack, 25*time.Second) if jobs == nil { jobs = []*rackJob{} } writeJSON(w, 200, jobs) } func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack string) { var res rackResult if err := json.NewDecoder(r.Body).Decode(&res); err != nil { fail(w, 400, err) return } writeJSON(w, 200, obj{"accepted": a.hub.finish(rack, res)}) } func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) { rs, err := a.store.RackControllers(r.Context()) if err != nil { fail(w, 500, err) return } ms, _ := a.store.List(r.Context()) for i := range rs { rs[i].Online = rackOnline(rs[i], a.cfg.RackName) for _, m := range ms { if a.rackOf(m) == rs[i].Name { rs[i].Machines++ } } } writeJSON(w, 200, rs) } func (a *App) handleControllerDelete(w http.ResponseWriter, r *http.Request) { name := r.PathValue("name") if name == a.cfg.RackName { fail(w, 409, errors.New("the embedded rack controller cannot be removed")) return } if err := a.store.DeleteRackController(r.Context(), name); err != nil { fail(w, 500, err) return } a.reloadRacks(r.Context()) a.hub.bump() writeJSON(w, 200, obj{"deleted": name}) } // handleRackDHCP shows a rack's generated dnsmasq configuration (admin). func (a *App) handleRackDHCP(w http.ResponseWriter, r *http.Request) { st, err := a.rackState(r.Context(), r.PathValue("name")) if err != nil { fail(w, 500, err) return } w.Header().Set("Content-Type", "text/plain; charset=utf-8") if st.DnsmasqConf == "" { fmt.Fprintf(w, "# rack controller %s serves no VLAN with dhcp on: rollcall leaves dnsmasq's configuration alone\n", r.PathValue("name")) } fmt.Fprint(w, st.DnsmasqConf, "\n# --- hosts.dhcp\n", st.HostsDHCP) } // dnsmasqUpstream tells dnsmasq where to forward names it does not answer // itself. Without it Debian's dnsmasq looks for a resolvconf-made file that // usually does not exist and refuses every outside name, so nodes could not // resolve mirrors or site services. With systemd-resolved the stub is used, so // its per-link routing (split DNS such as ~int.example.com) keeps working; // otherwise the host's resolv.conf servers, never ourselves. func dnsmasqUpstream(stub, resolv string, own []string) string { if _, err := os.Stat(stub); err == nil { return "# upstream: systemd-resolved on this host (keeps its split DNS)\nno-resolv\nserver=127.0.0.53\n" } b, err := os.ReadFile(resolv) if err != nil { return "" } mine := map[string]bool{} for _, a := range own { mine[stripPrefix(a)] = true } var servers []string for _, line := range strings.Split(string(b), "\n") { f := strings.Fields(line) if len(f) < 2 || f[0] != "nameserver" { continue } ip, ok := parseAddr(f[1]) if !ok || mine[ip.String()] || (ip.IsLoopback() && ip.String() != "127.0.0.53") { continue } servers = append(servers, ip.String()) } if len(servers) == 0 { return "" } out := "# upstream: this host's resolv.conf\nno-resolv\n" for _, sv := range servers { out += "server=" + sv + "\n" } return out } // localAddrs: this host's interface addresses (CIDR form). func localAddrs() []string { var out []string addrs, _ := net.InterfaceAddrs() for _, a := range addrs { out = append(out, a.String()) } return out }