rollcall/cmd/rollcall/ipam.go

package main

// External IPAM: an authority outside Rollcall (NetBox, or Infoblox where a
// site runs it) in which every address Rollcall hands out is registered.
//
//   - Allocation asks the authority first: an address someone else already
//     has there is skipped, so Rollcall never collides with the site's plan.
//   - Releases are local; the reconcile loop then makes the authority match
//     Rollcall's table: it creates what is missing, fixes names, and deletes
//     what Rollcall owns there but no longer uses.
//   - Ownership is explicit: a tag plus the machine ID in a custom field
//     (an extensible attribute on Infoblox). Records without both are never
//     changed or deleted, only reported as conflicts.
//   - DNS follows from the record's dns_name (NetBox + PowerDNS sync, or an
//     Infoblox host record), so Rollcall writes one object per address.

import (
	"bytes"
	"context"
	"crypto/tls"
	"crypto/x509"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/netip"
	"net/url"
	"os"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"
)

type IPAMConfig struct {
	Provider     string `json:"provider"`      // "" = built-in only | netbox
	URL          string `json:"url"`           // https://netbox.example
	TokenFile    string `json:"token_file"`    // API token (NetBox v2 "nbt_..." or v1)
	CAFile       string `json:"ca_file"`       // CA that signed the server certificate ("" = system roots)
	DNSZone      string `json:"dns_zone"`      // dns_name = <host>.<zone> unless the subnet has a domain
	Tag          string `json:"tag"`           // ownership tag, default "rollcall"
	MachineField string `json:"machine_field"` // custom field holding the machine ID, default "rollcall_machine"
	SyncInterval string `json:"sync_interval"` // reconcile period, default 5m
}

// extAddr is one address as the authority holds it.
type extAddr struct {
	ID      string `json:"id,omitempty"`
	IP      string `json:"ip"`
	Bits    int    `json:"bits"`
	DNSName string `json:"dns_name,omitempty"`
	Machine string `json:"machine,omitempty"` // owner mark; "-" = Rollcall reservation without a machine
	Status  string `json:"status"`            // active | reserved
	Desc    string `json:"description,omitempty"`
}

type extPrefix struct {
	Prefix  string `json:"prefix"`
	Desc    string `json:"description,omitempty"`
	VID     int    `json:"vid,omitempty"`
	Site    string `json:"site,omitempty"`
	Local   bool   `json:"local"` // a Rollcall subnet exists for it
	Comment string `json:"comment,omitempty"`
}

type ipamProvider interface {
	Name() string
	// Reserve registers a; taken is true when the address exists there and
	// is not Rollcall's. An existing Rollcall-owned record is updated.
	Reserve(ctx context.Context, a extAddr) (taken bool, err error)
	Owned(ctx context.Context) ([]extAddr, error)
	Update(ctx context.Context, a extAddr) error
	Delete(ctx context.Context, a extAddr) error
	Prefixes(ctx context.Context) ([]extPrefix, error)
}

type ipamReport struct {
	Provider  string    `json:"provider"`
	At        time.Time `json:"at"`
	DryRun    bool      `json:"dry_run,omitempty"`
	Created   []string  `json:"created"`
	Updated   []string  `json:"updated"`
	Deleted   []string  `json:"deleted"`
	Conflicts []string  `json:"conflicts"`
	Error     string    `json:"error,omitempty"`
}

// fail notes one record's error; the reconcile goes on with the others.
func (r *ipamReport) fail(e string) { r.Error = joinErr(r.Error, e) }

// joinErr keeps the first few errors of a pass readable.
func joinErr(all, e string) string {
	switch n := strings.Count(all, "; ") + 1; {
	case all == "":
		return e
	case n < 5:
		return all + "; " + e
	case n == 5:
		return all + "; ..."
	}
	return all
}

type ipamState struct {
	mu   sync.Mutex // one reconcile at a time
	kick chan struct{}
	last ipamReport
}

func newIPAMProvider(c IPAMConfig) (ipamProvider, error) {
	switch c.Provider {
	case "":
		return nil, nil
	case "netbox":
		return newNetBox(c)
	default:
		return nil, fmt.Errorf("ipam provider %q: unknown (netbox)", c.Provider)
	}
}

func (c IPAMConfig) tag() string   { return orDefault(c.Tag, "rollcall") }
func (c IPAMConfig) field() string { return orDefault(c.MachineField, "rollcall_machine") }
func orDefault(s, d string) string {
	if s == "" {
		return d
	}
	return s
}

// ---------------------------------------------------------------- App side

var dnsLabel = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`)

// validDomain: lower-case dot-separated DNS labels, no trailing dot. Names end
// up in dnsmasq lines, cloud-init, PXE menus and DNS updates, so nothing else.
func validDomain(s string) bool {
	if s == "" || len(s) > 253 {
		return false
	}
	for _, l := range strings.Split(s, ".") {
		if !dnsLabel.MatchString(l) {
			return false
		}
	}
	return true
}

// extFor maps a Rollcall address to the record the authority should hold.
func (a *App) extFor(x IPAddr, sn *Subnet) extAddr {
	bits := 32
	if p, err := netip.ParsePrefix(x.Subnet); err == nil {
		bits = p.Bits()
	} else if ad, ok := parseAddr(x.IP); ok && ad.Is6() {
		bits = 128
	}
	e := extAddr{IP: x.IP, Bits: bits, Status: "active", Machine: x.MachineID}
	if x.Kind == "reserved" {
		e.Status = "reserved"
	}
	if e.Machine == "" {
		e.Machine = "-"
	}
	desc := "rollcall"
	if x.MachineID != "" {
		desc += " " + x.MachineID
	}
	if x.Iface != "" {
		desc += " " + x.Iface
	}
	if x.Kind == "reserved" && x.Hostname != "" {
		desc += ": " + x.Hostname
	}
	e.Desc = desc
	e.DNSName = a.extDNSName(x, sn)
	return e
}

// extDNSName: the machine's main address gets <host>.<zone>; addresses on
// other subnets (storage, IB, a tagged VLAN) get <host>-<iface>.<zone>. A
// subnet's domain replaces the zone when it lies inside it.
func (a *App) extDNSName(x IPAddr, sn *Subnet) string { return dnsNameIn(x, sn, a.cfg.IPAM.DNSZone) }

// dnsNameIn is the naming rule shared by the IPAM authority and the DNS provider.
func dnsNameIn(x IPAddr, sn *Subnet, zone string) string {
	zone = strings.TrimSuffix(strings.ToLower(zone), ".")
	if sn != nil && sn.Domain != "" { // a subnet domain counts only inside the authority's zone; "pxe.local" stays local
		d := strings.TrimSuffix(strings.ToLower(sn.Domain), ".")
		if zone == "" || d == zone || strings.HasSuffix(d, "."+zone) {
			zone = d
		}
	}
	host := strings.ToLower(strings.TrimSuffix(x.Hostname, "."))
	if zone == "" || host == "" {
		return ""
	}
	if strings.HasSuffix(host, "."+zone) {
		if !validDomain(host) {
			return "" // a wildcard or anything that is not a plain name never reaches the DNS
		}
		return host
	}
	if !dnsLabel.MatchString(host) {
		return "" // a free-text reservation name ("core switch") stays in the description
	}
	main := sn == nil || x.Iface == "" || sn.Purpose == "" || sn.Purpose == "provisioning" || sn.Purpose == "mgmt"
	if !main {
		suffix := strings.Trim(regexp.MustCompile(`[^a-z0-9]+`).ReplaceAllString(strings.ToLower(x.Iface), "-"), "-")
		if suffix != "" {
			host += "-" + suffix
		}
	}
	return host + "." + zone
}

// claimIP records an address in the authority (when one is configured) and
// then locally; false when it is taken in either.
func (a *App) claimIP(ctx context.Context, x IPAddr) (bool, error) {
	if a.ipam != nil && a.ipamSt != nil {
		// not while a reconcile runs: it read the address table before this
		// claim and would delete the record Reserve is about to make
		a.ipamSt.mu.Lock()
		defer a.ipamSt.mu.Unlock()
	}
	if a.ipam != nil {
		sn := subnetByRef(a.subnetsSnapshot(), x.Subnet)
		taken, err := a.ipam.Reserve(ctx, a.extFor(x, sn))
		if err != nil {
			return false, fmt.Errorf("%s: %w", a.ipam.Name(), err)
		}
		if taken {
			return false, nil
		}
	}
	ok, err := a.store.ClaimIP(ctx, x)
	if ok {
		a.kickDNS()
		a.kickDHCPRes()
	}
	if a.ipam != nil && (err != nil || !ok) {
		a.kickIPAM() // the record we just made is reconciled away or re-owned
	}
	return ok, err
}

func (a *App) kickIPAM() {
	a.kickDNS() // the same address changes move DNS records
	a.kickDHCPRes()
	if a.ipamSt == nil {
		return
	}
	select {
	case a.ipamSt.kick <- struct{}{}:
	default:
	}
}

func (a *App) startIPAM() error {
	p, err := newIPAMProvider(a.cfg.IPAM)
	if err != nil || p == nil {
		return err
	}
	a.ipam = p
	a.ipamSt = &ipamState{kick: make(chan struct{}, 1)}
	every := 5 * time.Minute
	if d, err := time.ParseDuration(a.cfg.IPAM.SyncInterval); err == nil && d >= 10*time.Second {
		every = d
	}
	go func() {
		a.kickIPAM() // first pass right after start
		for {
			select {
			case <-a.ipamSt.kick:
			case <-time.After(every):
			}
			ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
			rep := a.reconcileIPAM(ctx, false)
			cancel()
			if rep.Error != "" {
				log.Printf("ipam %s: %s", p.Name(), rep.Error)
			} else if n := len(rep.Created) + len(rep.Updated) + len(rep.Deleted) + len(rep.Conflicts); n > 0 {
				log.Printf("ipam %s: created %d updated %d deleted %d conflicts %d %v", p.Name(),
					len(rep.Created), len(rep.Updated), len(rep.Deleted), len(rep.Conflicts), rep.Conflicts)
			}
		}
	}()
	log.Printf("ipam: %s at %s (zone %q, owner tag %q)", p.Name(), a.cfg.IPAM.URL, a.cfg.IPAM.DNSZone, a.cfg.IPAM.tag())
	return nil
}

// reconcileIPAM makes the authority match Rollcall's address table.
func (a *App) reconcileIPAM(ctx context.Context, dry bool) ipamReport {
	rep := ipamReport{At: time.Now().UTC(), DryRun: dry, Created: []string{}, Updated: []string{}, Deleted: []string{}, Conflicts: []string{}}
	if a.ipam == nil {
		rep.Error = "no external IPAM configured (ipam.provider in config.json)"
		return rep
	}
	rep.Provider = a.ipam.Name()
	a.ipamSt.mu.Lock()
	defer a.ipamSt.mu.Unlock()
	defer func() {
		if !dry {
			a.ipamSt.last = rep
		}
	}()
	local, err := a.store.IPs(ctx, "", "")
	if err != nil {
		rep.Error = err.Error()
		return rep
	}
	a.reconcileAddrs(ctx, local, &rep)
	return rep
}

// reconcileAddrs is the reconcile proper, over a given address table.
func (a *App) reconcileAddrs(ctx context.Context, local []IPAddr, rep *ipamReport) {
	dry := rep.DryRun
	owned, err := a.ipam.Owned(ctx)
	if err != nil {
		rep.Error = err.Error()
		return
	}
	have := map[string]extAddr{}
	for _, o := range owned {
		have[o.IP] = o
	}
	subnets := a.subnetsSnapshot()
	for _, x := range local {
		want := a.extFor(x, subnetByRef(subnets, x.Subnet))
		if o, ok := have[x.IP]; ok {
			delete(have, x.IP)
			if o.DNSName != want.DNSName || o.Machine != want.Machine || o.Status != want.Status || o.Bits != want.Bits || o.Desc != want.Desc {
				want.ID = o.ID
				rep.Updated = append(rep.Updated, x.IP)
				if !dry {
					if err := a.ipam.Update(ctx, want); err != nil {
						rep.fail(fmt.Sprintf("update %s: %v", x.IP, err))
						rep.Updated = rep.Updated[:len(rep.Updated)-1]
						continue
					}
				}
			}
			continue
		}
		if dry {
			rep.Created = append(rep.Created, x.IP)
			continue
		}
		taken, err := a.ipam.Reserve(ctx, want)
		if err != nil {
			rep.fail(fmt.Sprintf("create %s: %v", x.IP, err))
			continue
		}
		if taken {
			rep.Conflicts = append(rep.Conflicts, x.IP+" exists there and is not Rollcall's")
		} else {
			rep.Created = append(rep.Created, x.IP)
		}
	}
	rest := make([]extAddr, 0, len(have))
	for _, o := range have {
		rest = append(rest, o)
	}
	sort.Slice(rest, func(i, j int) bool { return ipLess(rest[i].IP, rest[j].IP) })
	for _, o := range rest {
		rep.Deleted = append(rep.Deleted, o.IP)
		if !dry {
			if err := a.ipam.Delete(ctx, o); err != nil {
				rep.fail(fmt.Sprintf("delete %s: %v", o.IP, err))
				rep.Deleted = rep.Deleted[:len(rep.Deleted)-1]
				continue
			}
		}
	}
}

func (a *App) handleIPAM(w http.ResponseWriter, r *http.Request) {
	if a.ipam == nil {
		writeJSON(w, 200, obj{"provider": "", "comment": "built-in IPAM only"})
		return
	}
	a.ipamSt.mu.Lock()
	last := a.ipamSt.last
	a.ipamSt.mu.Unlock()
	writeJSON(w, 200, obj{"provider": a.ipam.Name(), "url": a.cfg.IPAM.URL, "dns_zone": a.cfg.IPAM.DNSZone,
		"tag": a.cfg.IPAM.tag(), "machine_field": a.cfg.IPAM.field(), "last_sync": last})
}

func (a *App) handleIPAMSync(w http.ResponseWriter, r *http.Request) {
	dry := r.URL.Query().Get("dry") != ""
	rep := a.reconcileIPAM(r.Context(), dry)
	code := 200
	if rep.Error != "" {
		code = 502
		if a.ipam == nil {
			code = 400
		}
	}
	writeJSON(w, code, rep)
}

// handleIPAMPrefixes lists the prefixes the authority delegates to Rollcall
// (tagged); POST imports the missing ones as subnets.
func (a *App) handleIPAMPrefixes(w http.ResponseWriter, r *http.Request) {
	if a.ipam == nil {
		fail(w, 400, fmt.Errorf("no external IPAM configured"))
		return
	}
	ps, err := a.ipam.Prefixes(r.Context())
	if err != nil {
		fail(w, 502, err)
		return
	}
	subnets := a.subnetsSnapshot()
	imported := []string{}
	for i := range ps {
		p, err := netip.ParsePrefix(ps[i].Prefix)
		if err != nil {
			continue
		}
		for _, sn := range subnets {
			if q, err := netip.ParsePrefix(sn.CIDR); err == nil && q.Masked() == p.Masked() {
				ps[i].Local = true
			}
		}
		if r.Method == http.MethodPost && !ps[i].Local {
			sn := Subnet{CIDR: p.Masked().String(), Name: ps[i].Desc, Purpose: "provisioning",
				Comment: "imported from " + a.ipam.Name(), DNS: []string{}}
			if len(sn.Name) > 60 {
				sn.Name = sn.Name[:60]
			}
			if err := a.store.PutSubnet(r.Context(), sn); err != nil {
				fail(w, 500, err)
				return
			}
			ps[i].Local = true
			ps[i].Comment = "imported"
			imported = append(imported, sn.CIDR)
		}
	}
	if len(imported) > 0 {
		a.netChanged(r)
	}
	writeJSON(w, 200, obj{"prefixes": ps, "imported": imported})
}

// ---------------------------------------------------------------- NetBox

type netBox struct {
	base   string
	token  string
	tag    string
	field  string
	client *http.Client
}

func newNetBox(c IPAMConfig) (*netBox, error) {
	if c.URL == "" || c.TokenFile == "" {
		return nil, fmt.Errorf("netbox: url and token_file are required")
	}
	tok, err := os.ReadFile(c.TokenFile)
	if err != nil {
		return nil, fmt.Errorf("netbox token: %w", err)
	}
	hc, err := httpClientCA(c.CAFile, 30*time.Second)
	if err != nil {
		return nil, fmt.Errorf("netbox ca_file: %w", err)
	}
	return &netBox{base: strings.TrimRight(c.URL, "/"), token: strings.TrimSpace(string(tok)),
		tag: c.tag(), field: c.field(), client: hc}, nil
}

// httpClientCA is an HTTP client that trusts caFile (PEM) instead of the
// system's CAs when it is given.
func httpClientCA(caFile string, timeout time.Duration) (*http.Client, error) {
	tr := http.DefaultTransport.(*http.Transport).Clone()
	if caFile != "" {
		pem, err := os.ReadFile(caFile)
		if err != nil {
			return nil, err
		}
		pool := x509.NewCertPool()
		if !pool.AppendCertsFromPEM(pem) {
			return nil, fmt.Errorf("%s: no certificates", caFile)
		}
		tr.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}
	}
	return &http.Client{Transport: tr, Timeout: timeout}, nil
}

func (n *netBox) Name() string { return "netbox" }

type nbError struct {
	Code int
	Body string
}

func (e *nbError) Error() string { return fmt.Sprintf("HTTP %d: %s", e.Code, trunc(e.Body, 300)) }

func (n *netBox) do(ctx context.Context, method, path string, body, out any) error {
	var rd io.Reader
	if body != nil {
		b, _ := json.Marshal(body)
		rd = bytes.NewReader(b)
	}
	u := n.base + "/api/" + strings.TrimPrefix(path, "/")
	if strings.HasPrefix(path, "http") { // a "next" link: keep our base, the server may name itself differently
		if p, err := url.Parse(path); err == nil {
			u = n.base + p.Path
			if p.RawQuery != "" {
				u += "?" + p.RawQuery
			}
		}
	}
	req, err := http.NewRequestWithContext(ctx, method, u, rd)
	if err != nil {
		return err
	}
	if strings.HasPrefix(n.token, "nbt_") {
		req.Header.Set("Authorization", "Bearer "+n.token)
	} else {
		req.Header.Set("Authorization", "Token "+n.token)
	}
	req.Header.Set("Accept", "application/json")
	req.Header.Set("Content-Type", "application/json")
	resp, err := n.client.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	b, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
	if resp.StatusCode >= 300 {
		return &nbError{resp.StatusCode, string(b)}
	}
	if out != nil && len(b) > 0 {
		return json.Unmarshal(b, out)
	}
	return nil
}

type nbIP struct {
	ID           int64                   `json:"id"`
	Address      string                  `json:"address"`
	DNSName      string                  `json:"dns_name"`
	Description  string                  `json:"description"`
	Status       struct{ Value string }  `json:"status"`
	Tags         []struct{ Name string } `json:"tags"`
	CustomFields map[string]any          `json:"custom_fields"`
}

type nbPage[T any] struct {
	Next    *string `json:"next"`
	Results []T     `json:"results"`
}

func (n *netBox) list(ctx context.Context, path string, q url.Values) ([]nbIP, error) {
	q.Set("limit", "500")
	next := path + "?" + q.Encode()
	var out []nbIP
	for next != "" {
		var p nbPage[nbIP]
		if err := n.do(ctx, "GET", next, nil, &p); err != nil {
			return nil, err
		}
		out = append(out, p.Results...)
		next = ""
		if p.Next != nil {
			next = *p.Next
		}
	}
	return out, nil
}

func (n *netBox) owner(x nbIP) string {
	tagged := false
	for _, t := range x.Tags {
		tagged = tagged || t.Name == n.tag
	}
	if !tagged {
		return ""
	}
	if v, ok := x.CustomFields[n.field].(string); ok {
		return v
	}
	return ""
}

func (n *netBox) toExt(x nbIP) extAddr {
	ip, bits := x.Address, 0
	if p, err := netip.ParsePrefix(x.Address); err == nil {
		ip, bits = p.Addr().String(), p.Bits()
	}
	return extAddr{ID: strconv.FormatInt(x.ID, 10), IP: ip, Bits: bits, DNSName: x.DNSName, Machine: n.owner(x),
		Status: x.Status.Value, Desc: x.Description}
}

func (n *netBox) body(a extAddr) obj {
	return obj{"address": fmt.Sprintf("%s/%d", a.IP, a.Bits), "status": a.Status, "dns_name": a.DNSName,
		"description": a.Desc, "tags": []obj{{"name": n.tag}}, "custom_fields": obj{n.field: a.Machine}}
}

func (n *netBox) find(ctx context.Context, ip string) ([]nbIP, error) {
	return n.list(ctx, "ipam/ip-addresses/", url.Values{"address": {ip}})
}

func (n *netBox) Reserve(ctx context.Context, a extAddr) (bool, error) {
	cur, err := n.find(ctx, a.IP)
	if err != nil {
		return false, err
	}
	for _, x := range cur {
		if n.owner(x) == "" {
			return true, nil // someone else's
		}
	}
	if len(cur) > 0 {
		a.ID = strconv.FormatInt(cur[0].ID, 10)
		return false, n.Update(ctx, a)
	}
	err = n.do(ctx, "POST", "ipam/ip-addresses/", n.body(a), nil)
	var ne *nbError
	if err != nil && errors.As(err, &ne) && ne.Code == 400 && strings.Contains(strings.ToLower(ne.Body), "duplicate") {
		return true, nil // created by someone else between our look and our write
	}
	return false, err
}

func (n *netBox) Owned(ctx context.Context) ([]extAddr, error) {
	xs, err := n.list(ctx, "ipam/ip-addresses/", url.Values{"tag": {n.tag}})
	if err != nil {
		return nil, err
	}
	out := []extAddr{}
	for _, x := range xs {
		if n.owner(x) != "" {
			out = append(out, n.toExt(x))
		}
	}
	return out, nil
}

func (n *netBox) Update(ctx context.Context, a extAddr) error {
	return n.do(ctx, "PATCH", "ipam/ip-addresses/"+a.ID+"/", n.body(a), nil)
}

func (n *netBox) Delete(ctx context.Context, a extAddr) error {
	var cur nbIP // re-read: only delete what is still ours
	if err := n.do(ctx, "GET", "ipam/ip-addresses/"+a.ID+"/", nil, &cur); err != nil {
		var ne *nbError
		if errors.As(err, &ne) && ne.Code == 404 {
			return nil
		}
		return err
	}
	if n.owner(cur) == "" {
		return nil
	}
	err := n.do(ctx, "DELETE", "ipam/ip-addresses/"+a.ID+"/", nil, nil)
	var ne *nbError
	if errors.As(err, &ne) && ne.Code == 404 {
		return nil
	}
	return err
}

func (n *netBox) Prefixes(ctx context.Context) ([]extPrefix, error) {
	type nbPrefix struct {
		Prefix      string `json:"prefix"`
		Description string `json:"description"`
		VLAN        *struct {
			VID int `json:"vid"`
		} `json:"vlan"`
		Scope *struct {
			Name string `json:"name"`
		} `json:"scope"`
	}
	next := "ipam/prefixes/?" + url.Values{"tag": {n.tag}, "limit": {"500"}}.Encode()
	out := []extPrefix{}
	for next != "" {
		var p nbPage[nbPrefix]
		if err := n.do(ctx, "GET", next, nil, &p); err != nil {
			return nil, err
		}
		for _, x := range p.Results {
			e := extPrefix{Prefix: x.Prefix, Desc: x.Description}
			if x.VLAN != nil {
				e.VID = x.VLAN.VID
			}
			if x.Scope != nil {
				e.Site = x.Scope.Name
			}
			out = append(out, e)
		}
		next = ""
		if p.Next != nil {
			next = *p.Next
		}
	}
	return out, nil
}