The Sites page shows sites on an OpenStreetMap map (Leaflet 1.9.4 in the binary, checked against cdnjs's SRI; the CSP allows OSM tiles and the tile layer sends a referrer as OSM's policy asks). A site opens its rooms, a room its floor plan: racks on 600 mm tiles, one wide and two deep, front edge marked, coloured by their machines with a bar for the units used; a rack opens its front, and the rack is part of the URL. An edit mode drags racks into place and rotates them; racks nobody placed are laid out in rows facing alternate aisles. With NetBox as the IPAM authority, sites (coordinates, address), locations and racks are read on every reconcile, never written back. Otherwise sites are placed by clicking the map. Rack positions are Rollcall's (NetBox has none). Checked in headless Chrome with a synthetic site, including a scripted drag, rotate and save of a rack. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
894 lines
32 KiB
Go
894 lines
32 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/subtle"
|
|
"crypto/tls"
|
|
"embed"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
//go:embed web
|
|
var webFS embed.FS
|
|
|
|
type App struct {
|
|
cfg *Config
|
|
store *Store
|
|
sealer *Sealer
|
|
hub *rackHub
|
|
ca *CA
|
|
|
|
netMu sync.RWMutex
|
|
racks map[string]RackController
|
|
subnets []Subnet
|
|
|
|
ipam ipamProvider // nil = built-in IPAM only
|
|
ipamSt *ipamState
|
|
|
|
adminCC adminCertCache
|
|
|
|
dnsp dnsProvider // nil = no direct DNS writes
|
|
dnsSt *dnsState
|
|
// fixed leases on the site's DHCP server (dhcpres.go); nil = none
|
|
dhcpRes dhcpResProvider
|
|
dhcpResSt *dhcpResState
|
|
}
|
|
|
|
type apiError struct {
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, code int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(code)
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
_ = enc.Encode(v)
|
|
}
|
|
|
|
func fail(w http.ResponseWriter, code int, err error) {
|
|
if errors.Is(err, errNotFound) {
|
|
code = http.StatusNotFound
|
|
}
|
|
writeJSON(w, code, apiError{err.Error()})
|
|
}
|
|
|
|
func tokenEqual(a, b string) bool {
|
|
return b != "" && subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
|
|
}
|
|
|
|
func (a *App) isAdmin(r *http.Request) bool {
|
|
h := r.Header.Get("Authorization")
|
|
if strings.HasPrefix(h, "Bearer ") && tokenEqual(strings.TrimPrefix(h, "Bearer "), a.cfg.AdminToken) {
|
|
return true
|
|
}
|
|
return a.peerAdmin(r) != "" // an admin client certificate (admincert.go)
|
|
}
|
|
|
|
// adminOverHTTP applies admin_http to requests that did not come over TLS.
|
|
func (a *App) adminOverHTTP(r *http.Request) error {
|
|
if r.TLS != nil || a.cfg.AdminHTTP == "allow" {
|
|
return nil
|
|
}
|
|
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
|
if ip := net.ParseIP(host); a.cfg.AdminHTTP == "loopback" && ip != nil && ip.IsLoopback() {
|
|
return nil
|
|
}
|
|
_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
|
|
h := r.Host
|
|
if hh, _, err := net.SplitHostPort(h); err == nil {
|
|
h = hh
|
|
}
|
|
return fmt.Errorf("the admin API is HTTPS-only here: https://%s (trust the CA from /ca.pem)", net.JoinHostPort(strings.Trim(h, "[]"), port))
|
|
}
|
|
|
|
// httpsURL is the region's HTTPS address for the host the client used.
|
|
func (a *App) httpsURL(r *http.Request) string {
|
|
_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
|
|
h := r.Host
|
|
if hh, _, err := net.SplitHostPort(h); err == nil {
|
|
h = hh
|
|
}
|
|
h = strings.Trim(h, "[]")
|
|
if port == "" || port == "443" {
|
|
if strings.Contains(h, ":") {
|
|
return "https://[" + h + "]"
|
|
}
|
|
return "https://" + h
|
|
}
|
|
return "https://" + net.JoinHostPort(h, port)
|
|
}
|
|
|
|
// uiOverHTTPS sends browsers that reach the web UI over plain HTTP to HTTPS when
|
|
// the admin API is not allowed over HTTP anyway (the page could not work).
|
|
// Only the UI is redirected: nodes keep using HTTP for images and the agent.
|
|
func (a *App) uiOverHTTPS(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.TLS == nil && a.cfg.AdminHTTP != "" && a.cfg.AdminHTTP != "allow" && a.cfg.TLSListen != "" && a.adminOverHTTP(r) != nil {
|
|
http.Redirect(w, r, a.httpsURL(r)+r.URL.RequestURI(), http.StatusPermanentRedirect)
|
|
return
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
func (a *App) adminAuth(next http.HandlerFunc) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
if err := a.adminOverHTTP(r); err != nil {
|
|
fail(w, 403, err)
|
|
return
|
|
}
|
|
if !a.isAdmin(r) {
|
|
w.Header().Set("WWW-Authenticate", `Bearer realm="rollcall"`)
|
|
fail(w, http.StatusUnauthorized, errors.New("admin token required"))
|
|
return
|
|
}
|
|
next(w, r)
|
|
}
|
|
}
|
|
|
|
func (a *App) agentAuth(next http.HandlerFunc) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
if a.cfg.AgentToken != "" && !tokenEqual(r.Header.Get("X-Rollcall-Token"), a.cfg.AgentToken) && !a.isAdmin(r) {
|
|
fail(w, http.StatusUnauthorized, errors.New("agent token required"))
|
|
return
|
|
}
|
|
next(w, r)
|
|
}
|
|
}
|
|
|
|
func logging(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
start := time.Now()
|
|
rw := &statusWriter{ResponseWriter: w, code: 200}
|
|
r.Body = http.MaxBytesReader(rw, r.Body, 16<<20) // handlers set tighter limits; nothing takes more
|
|
next.ServeHTTP(rw, r)
|
|
if (!strings.HasPrefix(r.URL.Path, "/rollcall/") && !strings.HasPrefix(r.URL.Path, "/images/") && !strings.HasPrefix(r.URL.Path, "/os/")) || rw.code >= 400 {
|
|
uri := r.URL.RequestURI()
|
|
if r.URL.Query().Has("token") { // /?token=... signs the UI in; it does not go to the journal
|
|
uri = r.URL.Path + "?token=REDACTED"
|
|
}
|
|
log.Printf("%s %s %s %d %s", r.RemoteAddr, r.Method, uri, rw.code, time.Since(start).Round(time.Millisecond))
|
|
}
|
|
})
|
|
}
|
|
|
|
type statusWriter struct {
|
|
http.ResponseWriter
|
|
code int
|
|
}
|
|
|
|
func (s *statusWriter) WriteHeader(c int) { s.code = c; s.ResponseWriter.WriteHeader(c) }
|
|
|
|
func (a *App) routes() http.Handler {
|
|
mux := http.NewServeMux()
|
|
// agent endpoints
|
|
mux.HandleFunc("POST /api/v1/commission", a.agentAuth(a.handleCommission))
|
|
mux.HandleFunc("GET /api/v1/bmc-hint", a.agentAuth(a.handleHint))
|
|
// management endpoints
|
|
mux.HandleFunc("GET /api/v1/machines", a.adminAuth(a.handleList))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}", a.adminAuth(a.handleGet))
|
|
mux.HandleFunc("DELETE /api/v1/machines/{id}", a.adminAuth(a.handleDelete))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/recommission", a.adminAuth(a.handleRecommission))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/power", a.adminAuth(a.handlePower))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/power", a.adminAuth(a.handlePower))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/bootdev", a.adminAuth(a.handleBootdev))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/bmc", a.adminAuth(a.handleBMC))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/bmc", a.adminAuth(a.handleBMC))
|
|
mux.HandleFunc("GET /api/v1/power-drivers", a.adminAuth(func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, powerDrivers) }))
|
|
mux.HandleFunc("GET /api/v1/whoami", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
|
writeJSON(w, 200, map[string]any{"ok": true})
|
|
}))
|
|
// stateless provisioning
|
|
mux.HandleFunc("GET /api/v1/profiles", a.adminAuth(a.handleProfiles))
|
|
mux.HandleFunc("POST /api/v1/profiles", a.adminAuth(a.handleProfiles))
|
|
mux.HandleFunc("DELETE /api/v1/profiles/{name}", a.adminAuth(a.handleProfileDelete))
|
|
mux.HandleFunc("GET /api/v1/images", a.adminAuth(a.handleImages))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/assign", a.adminAuth(a.handleAssign))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/unassign", a.adminAuth(a.handleUnassign))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/overlay", a.adminAuth(a.handleAdminOverlay))
|
|
mux.HandleFunc("GET /api/v1/render/{profile}/{path...}", a.adminAuth(a.handleRender))
|
|
mux.HandleFunc("GET /api/v1/nodes/self/overlay", a.handleNodeOverlay) // node token auth inside
|
|
// disk-based deployment
|
|
mux.HandleFunc("GET /api/v1/os", a.adminAuth(a.handleOS))
|
|
mux.HandleFunc("GET /api/v1/dhcp-reservations", a.adminAuth(a.handleDHCPRes))
|
|
mux.HandleFunc("POST /api/v1/dhcp-reservations/sync", a.adminAuth(a.handleDHCPResSync))
|
|
mux.HandleFunc("GET /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo))
|
|
mux.HandleFunc("POST /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo))
|
|
mux.HandleFunc("DELETE /api/v1/firmware/{pkg}", a.adminAuth(a.handleFirmwareRemove))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/firmware/update", a.adminAuth(a.handleFirmwareUpdate))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/deploy", a.adminAuth(a.handleDeploy))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/release", a.adminAuth(a.handleRelease))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/jobs", a.adminAuth(a.handleJobs))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/jobs/cancel", a.adminAuth(a.handleJobCancel))
|
|
mux.HandleFunc("GET /api/v1/jobs/self", a.handleJobSelf) // job token auth inside
|
|
mux.HandleFunc("POST /api/v1/jobs/self/progress", a.handleJobProgress) // job token auth inside
|
|
mux.HandleFunc("POST /api/v1/phone-home/{token}", a.handlePhoneHome)
|
|
// fleet-wide views for the UI
|
|
mux.HandleFunc("GET /api/v1/events", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
|
evs, err := a.store.RecentEvents(r.Context(), 100)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, evs)
|
|
}))
|
|
mux.HandleFunc("GET /api/v1/jobs", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
|
jobs, err := a.store.RecentJobs(r.Context(), 100)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, jobs)
|
|
}))
|
|
// network model and rack controllers
|
|
mux.HandleFunc("GET /api/v1/fabrics", a.adminAuth(a.handleFabrics))
|
|
mux.HandleFunc("POST /api/v1/fabrics", a.adminAuth(a.handleFabrics))
|
|
mux.HandleFunc("DELETE /api/v1/fabrics/{name}", a.adminAuth(a.handleFabricDelete))
|
|
mux.HandleFunc("GET /api/v1/vlans", a.adminAuth(a.handleVLANs))
|
|
mux.HandleFunc("POST /api/v1/vlans", a.adminAuth(a.handleVLANs))
|
|
mux.HandleFunc("DELETE /api/v1/vlans/{fabric}/{vid}", a.adminAuth(a.handleVLANDelete))
|
|
mux.HandleFunc("GET /api/v1/subnets", a.adminAuth(a.handleSubnets))
|
|
mux.HandleFunc("POST /api/v1/subnets", a.adminAuth(a.handleSubnets))
|
|
mux.HandleFunc("DELETE /api/v1/subnets/{cidr...}", a.adminAuth(a.handleSubnetDelete))
|
|
mux.HandleFunc("POST /api/v1/ranges", a.adminAuth(a.handleRanges))
|
|
mux.HandleFunc("DELETE /api/v1/ranges/{id}", a.adminAuth(a.handleRangeDelete))
|
|
mux.HandleFunc("GET /api/v1/ips", a.adminAuth(a.handleIPs))
|
|
mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs))
|
|
mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete))
|
|
mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM))
|
|
mux.HandleFunc("GET /api/v1/ca/rotation", a.adminAuth(a.handleCARotation))
|
|
mux.HandleFunc("GET /api/v1/dns", a.adminAuth(a.handleDNS))
|
|
mux.HandleFunc("POST /api/v1/dns/sync", a.adminAuth(a.handleDNSSync))
|
|
mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))
|
|
mux.HandleFunc("POST /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))
|
|
mux.HandleFunc("DELETE /api/v1/admin-certs/{ref}", a.adminAuth(a.handleAdminCertRevoke))
|
|
mux.HandleFunc("POST /api/v1/ipam/sync", a.adminAuth(a.handleIPAMSync))
|
|
mux.HandleFunc("GET /api/v1/ipam/prefixes", a.adminAuth(a.handleIPAMPrefixes))
|
|
mux.HandleFunc("POST /api/v1/ipam/prefixes", a.adminAuth(a.handleIPAMPrefixes))
|
|
mux.HandleFunc("GET /api/v1/layouts", a.adminAuth(a.handleLayouts))
|
|
mux.HandleFunc("POST /api/v1/layouts", a.adminAuth(a.handleLayouts))
|
|
mux.HandleFunc("DELETE /api/v1/layouts/{name}", a.adminAuth(a.handleLayoutDelete))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
|
|
mux.HandleFunc("GET /api/v1/controllers", a.adminAuth(a.handleControllers))
|
|
mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete))
|
|
mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP))
|
|
mux.HandleFunc("GET /ca.pem", a.handleCAPEM)
|
|
mux.HandleFunc("GET /ca-bundle.pem", a.handleCABundle)
|
|
mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode))
|
|
mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only
|
|
mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew))
|
|
mux.HandleFunc("POST /api/v1/rack/server-cert", a.rackAuth(a.handleRackServerCert))
|
|
mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister))
|
|
mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState))
|
|
mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork))
|
|
mux.HandleFunc("POST /api/v1/rack/result", a.rackAuth(a.handleRackResult))
|
|
mux.HandleFunc("GET /api/v1/rack/binary", a.rackAuth(a.handleRackBinary))
|
|
// physical location
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/location", a.adminAuth(a.handleLocation))
|
|
mux.HandleFunc("POST /api/v1/locations/import", a.adminAuth(a.handleLocationImport))
|
|
mux.HandleFunc("GET /api/v1/racks", a.adminAuth(a.handleRacks))
|
|
mux.HandleFunc("POST /api/v1/racks", a.adminAuth(a.handleRacks))
|
|
mux.HandleFunc("DELETE /api/v1/racks", a.adminAuth(a.handleRacks))
|
|
mux.HandleFunc("POST /api/v1/racks/positions", a.adminAuth(a.handleRackPositions))
|
|
mux.HandleFunc("GET /api/v1/sites", a.adminAuth(a.handleSites))
|
|
mux.HandleFunc("POST /api/v1/sites", a.adminAuth(a.handleSites))
|
|
mux.HandleFunc("DELETE /api/v1/sites", a.adminAuth(a.handleSites))
|
|
mux.HandleFunc("GET /api/v1/rooms", a.adminAuth(a.handleRooms))
|
|
mux.HandleFunc("POST /api/v1/rooms", a.adminAuth(a.handleRooms))
|
|
// tags
|
|
mux.HandleFunc("GET /api/v1/tags", a.adminAuth(a.handleTags))
|
|
mux.HandleFunc("POST /api/v1/tags", a.adminAuth(a.handleTags))
|
|
mux.HandleFunc("DELETE /api/v1/tags/{name}", a.adminAuth(a.handleTagDelete))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/tags", a.adminAuth(a.handleMachineTags))
|
|
// image files for the live boot (no auth: the kernel/initrd/squashfs must be fetchable)
|
|
imgDir := a.cfg.ImageDir
|
|
mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir))))
|
|
mux.Handle("GET /images/", http.StripPrefix("/images/", http.FileServer(http.Dir(a.cfg.ImagesDir))))
|
|
mux.Handle("GET /os/", http.StripPrefix("/os/", http.FileServer(http.Dir(a.cfg.OSDir))))
|
|
mux.HandleFunc("GET /firmware/{pkg}/{file}", a.handleFirmwareFile) // BMCs download packages (no listing)
|
|
// embedded web UI
|
|
sub, _ := fs.Sub(webFS, "web")
|
|
mux.Handle("GET /", a.uiOverHTTPS(uiHeaders(http.FileServer(http.FS(sub)))))
|
|
// Browsers send admin client certificates on their own, so a page elsewhere
|
|
// could post to the API with the admin's identity: cross-origin requests
|
|
// that change state are refused (Sec-Fetch-Site / Origin). The CLI, agents
|
|
// and racks send neither header and are not affected.
|
|
return logging(http.NewCrossOriginProtection().Handler(mux))
|
|
}
|
|
|
|
// uiHeaders: the UI loads only its own scripts and cannot be framed, so
|
|
// markup that slips through (machine-reported strings) does not run.
|
|
func uiHeaders(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
h := w.Header()
|
|
h.Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data: https://tile.openstreetmap.org; connect-src 'self'; object-src 'none'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'")
|
|
h.Set("X-Content-Type-Options", "nosniff")
|
|
h.Set("X-Frame-Options", "DENY")
|
|
h.Set("Referrer-Policy", "no-referrer")
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------- helpers over inventory JSON
|
|
|
|
type obj = map[string]any
|
|
|
|
func dig(m any, path ...string) any {
|
|
cur := m
|
|
for _, p := range path {
|
|
mm, ok := cur.(obj)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
cur = mm[p]
|
|
}
|
|
return cur
|
|
}
|
|
|
|
func str(v any) string {
|
|
switch t := v.(type) {
|
|
case nil:
|
|
return ""
|
|
case string:
|
|
return t
|
|
case float64:
|
|
return fmt.Sprintf("%v", t)
|
|
default:
|
|
return fmt.Sprintf("%v", t)
|
|
}
|
|
}
|
|
|
|
func strPtr(v any) *string {
|
|
s := str(v)
|
|
if s == "" {
|
|
return nil
|
|
}
|
|
return &s
|
|
}
|
|
|
|
func listLen(v any) int {
|
|
if l, ok := v.([]any); ok {
|
|
return len(l)
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// normMAC is the canonical form of a hardware address, "" when s is not one.
|
|
// MACs name files (pxelinux.cfg/01-...) and dnsmasq lines, so nothing else passes.
|
|
func normMAC(s string) string {
|
|
hw, err := net.ParseMAC(strings.TrimSpace(s))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return hw.String()
|
|
}
|
|
|
|
var nonAlnum = regexp.MustCompile(`[^a-z0-9]`)
|
|
|
|
func machineID(uuid, serial string, macs []string) string {
|
|
u := strings.ToLower(uuid)
|
|
if u != "" && u != "00000000-0000-0000-0000-000000000000" && !strings.Contains(u, "not ") &&
|
|
nonAlnum.ReplaceAllString(u, "") != "" {
|
|
u = nonAlnum.ReplaceAllString(u, "") // the ID names files and URLs: letters and digits only
|
|
if len(u) > 12 {
|
|
u = u[:12]
|
|
}
|
|
return "m-" + u
|
|
}
|
|
if s := nonAlnum.ReplaceAllString(strings.ToLower(serial), ""); s != "" && !strings.Contains(strings.ToLower(serial), "not ") {
|
|
if len(s) > 16 {
|
|
s = s[:16]
|
|
}
|
|
return "m-" + s
|
|
}
|
|
if len(macs) > 0 {
|
|
return "m-" + strings.ReplaceAll(macs[0], ":", "")
|
|
}
|
|
return "m-" + randomToken(6)
|
|
}
|
|
|
|
// Summary is the list-view projection of a machine (no secrets, no full inventory).
|
|
type Summary struct {
|
|
ID string `json:"id"`
|
|
State string `json:"state"`
|
|
Hostname *string `json:"hostname"`
|
|
Serial *string `json:"serial"`
|
|
UUID *string `json:"uuid"`
|
|
Manufacturer *string `json:"manufacturer"`
|
|
Product *string `json:"product"`
|
|
FirstSeen time.Time `json:"first_seen"`
|
|
LastSeen *time.Time `json:"last_seen"`
|
|
MACs []string `json:"macs"`
|
|
CPU string `json:"cpu"`
|
|
CPUs string `json:"cpus"`
|
|
MemoryBytes float64 `json:"memory_bytes"`
|
|
Disks float64 `json:"disks"`
|
|
StorageBytes float64 `json:"storage_bytes"`
|
|
NICs int `json:"nics"`
|
|
IBHCAs int `json:"ib_hcas"`
|
|
GPUs int `json:"gpus"`
|
|
BMC *obj `json:"bmc"`
|
|
Profile *string `json:"profile"`
|
|
NodeName *string `json:"node_name"`
|
|
MgmtIP *string `json:"mgmt_ip"`
|
|
IBIP *string `json:"ib_ip"`
|
|
BootedAt *time.Time `json:"booted_at"`
|
|
OS *string `json:"os"`
|
|
OSVersion *string `json:"os_version"`
|
|
DeployedAt *time.Time `json:"deployed_at"`
|
|
Job *Job `json:"job,omitempty"` // active deployment job, if any
|
|
Tags []string `json:"tags"` // effective (manual + auto)
|
|
ManualTags []string `json:"manual_tags"`
|
|
AutoTags []string `json:"auto_tags"`
|
|
Location Location `json:"location"`
|
|
Size rackSize `json:"size"` // rack units (hwsize.go)
|
|
Rack string `json:"rack_controller"`
|
|
NetLayout string `json:"net_layout"`
|
|
}
|
|
|
|
func num(v any) float64 {
|
|
f, _ := v.(float64)
|
|
return f
|
|
}
|
|
|
|
func summarize(m *Machine) Summary {
|
|
var inv obj
|
|
_ = json.Unmarshal(m.Inventory, &inv)
|
|
s := Summary{ID: m.ID, State: m.State, Hostname: m.Hostname, Serial: m.Serial, UUID: m.UUID,
|
|
Manufacturer: m.Manufacturer, Product: m.Product, FirstSeen: m.FirstSeen, LastSeen: m.LastSeen, MACs: m.MACs,
|
|
CPU: str(dig(inv, "cpu", "Model name")), CPUs: str(dig(inv, "cpu", "CPU(s)")),
|
|
MemoryBytes: num(dig(inv, "memory", "total_bytes")), Disks: num(dig(inv, "storage", "disk_count")),
|
|
StorageBytes: num(dig(inv, "storage", "total_bytes")), NICs: listLen(inv["network"]),
|
|
IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: nodeCtx(m).GPUs, // real accelerators only
|
|
Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt,
|
|
OS: m.OS, OSVersion: m.OSVersion, DeployedAt: m.DeployedAt,
|
|
Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location,
|
|
Rack: m.RackController, NetLayout: m.NetLayout}
|
|
if m.BMC != nil {
|
|
var b obj
|
|
if json.Unmarshal(m.BMC, &b) == nil {
|
|
drv := str(b["driver"])
|
|
if drv == "" {
|
|
drv = "ipmi"
|
|
}
|
|
s.BMC = &obj{"source": b["source"], "address": b["address"], "user": b["user"], "driver": drv,
|
|
"configured": b["configured"], "verified_oob": b["verified_oob"]}
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
// ---------------------------------------------------------------- pxelinux per-MAC files
|
|
|
|
const commissionedTmpl = `# generated by rollcall %s
|
|
# machine %s %s %s serial=%s
|
|
DEFAULT menu.c32
|
|
PROMPT 0
|
|
TIMEOUT 150
|
|
MENU TITLE %s - commissioned (%s %s, S/N %s)
|
|
|
|
LABEL poweroff
|
|
MENU LABEL Commissioned: power off (default in 15s)
|
|
MENU DEFAULT
|
|
KERNEL poweroff.c32
|
|
|
|
LABEL recommission
|
|
MENU LABEL Re-run commissioning
|
|
KERNEL rollcall/vmlinuz
|
|
INITRD rollcall/initrd.img
|
|
APPEND %s
|
|
|
|
LABEL local
|
|
MENU LABEL Boot from local disk
|
|
LOCALBOOT 0
|
|
`
|
|
|
|
func deref(p *string) string {
|
|
if p == nil {
|
|
return ""
|
|
}
|
|
return *p
|
|
}
|
|
|
|
func (a *App) writePXE(m *Machine) error {
|
|
man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial)
|
|
body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser,
|
|
m.ID, man, prod, ser, a.appendFor(a.urlFor(m)))
|
|
return a.writePXEBody(m, body)
|
|
}
|
|
|
|
// ---------------------------------------------------------------- handlers
|
|
|
|
func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
|
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 4<<20))
|
|
if err != nil {
|
|
fail(w, 400, err)
|
|
return
|
|
}
|
|
var inv obj
|
|
if err := json.Unmarshal(body, &inv); err != nil {
|
|
fail(w, 400, fmt.Errorf("bad json: %w", err))
|
|
return
|
|
}
|
|
ident, _ := inv["identity"].(obj)
|
|
var macs []string
|
|
if l, ok := ident["macs"].([]any); ok {
|
|
for _, v := range l {
|
|
if s := normMAC(str(v)); s != "" {
|
|
macs = append(macs, s)
|
|
}
|
|
}
|
|
}
|
|
if boot := normMAC(str(ident["boot_mac"])); boot != "" {
|
|
for i, mac := range macs {
|
|
if mac == boot && i != 0 {
|
|
macs = append([]string{boot}, append(macs[:i], macs[i+1:]...)...)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
// identity strings come from the machine: one line each, they go into PXE menus and logs
|
|
line := func(v any) *string {
|
|
s := strings.Map(func(c rune) rune {
|
|
if c < ' ' || c == 0x7f {
|
|
return ' '
|
|
}
|
|
return c
|
|
}, str(v))
|
|
if s = strings.TrimSpace(s); len(s) > 128 {
|
|
s = s[:128]
|
|
}
|
|
return strPtr(s)
|
|
}
|
|
m := &Machine{
|
|
ID: machineID(str(ident["uuid"]), str(ident["serial"]), macs),
|
|
UUID: line(ident["uuid"]),
|
|
Serial: line(ident["serial"]),
|
|
Hostname: line(inv["hostname"]),
|
|
Manufacturer: line(dig(inv, "system", "system-manufacturer")),
|
|
Product: line(dig(inv, "system", "system-product-name")),
|
|
MACs: macs,
|
|
}
|
|
// Pull the BMC password out of the payload before anything is persisted.
|
|
var secret []byte
|
|
detail := "bmc=none"
|
|
if bmc, _ := inv["bmc"].(obj); bmc == nil || str(bmc["source"]) == "hint" || str(bmc["address"]) == "" {
|
|
// a configured out-of-band BMC: the region's credentials, whatever the agent sent
|
|
if h := a.hintFor(macs); h != nil {
|
|
port := h.Port
|
|
if port == 0 {
|
|
port = 623
|
|
}
|
|
nb := obj{"source": "hint", "address": h.Address, "port": port, "user": h.User, "configured": true, "password": "<from bmc_hints>"}
|
|
if bmc != nil {
|
|
nb["agent"] = obj{"source": bmc["source"], "errors": bmc["errors"]}
|
|
}
|
|
inv["bmc"] = nb
|
|
if h.Password != "" {
|
|
if secret, err = a.sealer.Seal(h.Password, m.ID); err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if bmc, ok := inv["bmc"].(obj); ok {
|
|
if pw := str(bmc["password"]); pw != "" && str(bmc["source"]) != "hint" {
|
|
secret, err = a.sealer.Seal(pw, m.ID)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
bmc["password"] = "<stored encrypted>"
|
|
}
|
|
m.BMC, _ = json.Marshal(bmc)
|
|
detail = fmt.Sprintf("bmc=%s:%s configured=%v", str(bmc["source"]), str(bmc["address"]), bmc["configured"])
|
|
}
|
|
m.Inventory, _ = json.Marshal(inv)
|
|
|
|
// Anyone on the provisioning network can post here, so a known machine is
|
|
// only re-commissioned when that is expected: an operator asked
|
|
// (recommission), or it has nothing to lose (ready, a failed job). A report
|
|
// for a deployed or running machine is refused, not merged.
|
|
old, oldErr := a.store.Get(r.Context(), m.ID)
|
|
if oldErr == nil && !recommissionable(old.State) {
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "commission refused", fmt.Sprintf("state %s; from %s", old.State, r.RemoteAddr))
|
|
log.Printf("commission refused for %s (state %s) from %s", m.ID, old.State, r.RemoteAddr)
|
|
fail(w, 409, fmt.Errorf("machine %s is %s; recommission it first", m.ID, old.State))
|
|
return
|
|
}
|
|
// a BMC an operator configured (driver_set) survives commissioning; what the agent
|
|
// found is kept next to it for reference
|
|
if oldErr == nil && old.BMC != nil {
|
|
var ob obj
|
|
if json.Unmarshal(old.BMC, &ob) == nil && ob["driver_set"] == true {
|
|
var found obj
|
|
_ = json.Unmarshal(m.BMC, &found)
|
|
delete(found, "password")
|
|
ob["commissioning"] = found
|
|
m.BMC, _ = json.Marshal(ob)
|
|
secret = old.bmcSecret
|
|
detail += " (operator BMC settings kept)"
|
|
}
|
|
}
|
|
|
|
isNew, err := a.store.UpsertCommissioned(r.Context(), m, secret)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail)
|
|
if isNew || oldErr != nil || old.State == "recommission" || old.RackController == "" {
|
|
_ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r)) // a ready machine is not moved by whoever reports it
|
|
}
|
|
go func(id string) { // check a hinted BMC from where it is reachable, then look for Redfish
|
|
ctx := context.Background()
|
|
if full, err := a.store.Get(ctx, id); err == nil {
|
|
a.verifyHintedBMC(full)
|
|
}
|
|
if full, err := a.store.Get(ctx, id); err == nil {
|
|
a.probeBMC(full)
|
|
}
|
|
}(m.ID)
|
|
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
|
|
m = full // carries profile / node token when this is a re-commission of a stateless node
|
|
}
|
|
a.retag(r.Context(), []*Machine{m})
|
|
if err := a.writePXEAny(r.Context(), m); err != nil {
|
|
log.Printf("pxe write failed for %s: %v", m.ID, err)
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "pxe-write-failed", err.Error())
|
|
}
|
|
log.Printf("commissioned %s new=%v (%s %s, serial %s, %d macs) %s", m.ID, isNew, deref(m.Manufacturer),
|
|
deref(m.Product), deref(m.Serial), len(macs), detail)
|
|
writeJSON(w, 200, map[string]any{"id": m.ID, "state": "ready", "new": isNew})
|
|
}
|
|
|
|
func recommissionable(state string) bool {
|
|
return state == "new" || state == "ready" || state == "recommission" || strings.HasSuffix(state, "-failed")
|
|
}
|
|
|
|
// hintFor is the configured out-of-band BMC for one of these MACs.
|
|
func (a *App) hintFor(macs []string) *BMCHint {
|
|
want := map[string]bool{}
|
|
for _, m := range macs {
|
|
if n := normMAC(m); n != "" {
|
|
want[n] = true
|
|
}
|
|
}
|
|
for i, h := range a.cfg.BMCHints {
|
|
if want[normMAC(h.MAC)] {
|
|
return &a.cfg.BMCHints[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// handleHint tells a commissioning agent that a BMC is configured for it: the
|
|
// address, never the password (the agent token is public, MACs are visible on
|
|
// the wire). The region attaches the credentials itself when the machine
|
|
// reports in (handleCommission) and checks them from the rack controller.
|
|
func (a *App) handleHint(w http.ResponseWriter, r *http.Request) {
|
|
h := a.hintFor(strings.Split(r.URL.Query().Get("macs"), ","))
|
|
if h == nil {
|
|
writeJSON(w, 200, obj{})
|
|
return
|
|
}
|
|
port := h.Port
|
|
if port == 0 {
|
|
port = 623
|
|
}
|
|
writeJSON(w, 200, obj{"address": h.Address, "port": port, "user": h.User, "credentials": "on the region"})
|
|
}
|
|
|
|
func (a *App) handleList(w http.ResponseWriter, r *http.Request) {
|
|
ms, err := a.store.List(r.Context())
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
active, _ := a.store.ActiveJobs(r.Context())
|
|
wantTag := r.URL.Query().Get("tag")
|
|
out := make([]Summary, 0, len(ms))
|
|
for _, m := range ms {
|
|
if wantTag != "" && !hasTag(effectiveTags(m), wantTag) {
|
|
continue
|
|
}
|
|
if !locationMatch(m, r.URL.Query()) {
|
|
continue
|
|
}
|
|
s := summarize(m)
|
|
s.Size = machineSize(m, a.cfg.RackUnits)
|
|
s.Job = active[m.ID]
|
|
out = append(out, s)
|
|
}
|
|
writeJSON(w, 200, out)
|
|
}
|
|
|
|
func (a *App) handleGet(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
evs, _ := a.store.Events(r.Context(), m.ID)
|
|
jobs, _ := a.store.Jobs(r.Context(), m.ID, 10)
|
|
s := summarize(m)
|
|
s.Size = machineSize(m, a.cfg.RackUnits)
|
|
if j, err := a.store.ActiveJob(r.Context(), m.ID); err == nil {
|
|
s.Job = j
|
|
}
|
|
var bmc, inv any
|
|
_ = json.Unmarshal(m.BMC, &bmc)
|
|
_ = json.Unmarshal(m.Inventory, &inv)
|
|
writeJSON(w, 200, obj{"summary": s, "bmc": bmc, "has_bmc_secret": m.bmcSecret != nil,
|
|
"inventory": inv, "events": evs, "jobs": jobs})
|
|
}
|
|
|
|
func (a *App) handleDelete(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
a.removePXE(m)
|
|
defer a.kickIPAM()
|
|
if err := a.store.Delete(r.Context(), m.ID); err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
log.Printf("deleted %s", m.ID)
|
|
writeJSON(w, 200, obj{"deleted": m.ID})
|
|
}
|
|
|
|
func (a *App) handleRecommission(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
a.removePXE(m)
|
|
_ = a.store.SetState(r.Context(), m.ID, "recommission")
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "recommission requested", "")
|
|
// optional: power-cycle it right away when the caller asks (?now=1)
|
|
out := obj{"id": m.ID, "state": "recommission"}
|
|
if r.URL.Query().Get("now") == "1" {
|
|
out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m)
|
|
}
|
|
writeJSON(w, 200, out)
|
|
}
|
|
|
|
func (a *App) handlePower(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
action := "status"
|
|
if r.Method == http.MethodPost {
|
|
var req struct {
|
|
Action string `json:"action"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
fail(w, 400, err)
|
|
return
|
|
}
|
|
action = req.Action
|
|
}
|
|
if !powerOps[action] || action == "bootdev" || action == "probe" {
|
|
fail(w, 400, fmt.Errorf("unknown power action %q", action))
|
|
return
|
|
}
|
|
out, err := a.power(r.Context(), m, action, "")
|
|
if errors.Is(err, errNoBMC) {
|
|
fail(w, 409, err)
|
|
return
|
|
}
|
|
if action != "status" {
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "power "+action, out)
|
|
}
|
|
res := obj{"id": m.ID, "action": action, "ok": err == nil, "output": out}
|
|
if err != nil {
|
|
res["error"] = err.Error()
|
|
}
|
|
writeJSON(w, 200, res)
|
|
}
|
|
|
|
func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
var req struct {
|
|
Device string `json:"device"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || !bootDevices[req.Device] {
|
|
fail(w, 400, fmt.Errorf("device must be one of pxe, disk, cdrom, bios, none, safe"))
|
|
return
|
|
}
|
|
out, err := a.power(r.Context(), m, "bootdev", req.Device)
|
|
if errors.Is(err, errNoBMC) {
|
|
fail(w, 409, err)
|
|
return
|
|
}
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "bootdev "+req.Device, out)
|
|
writeJSON(w, 200, obj{"id": m.ID, "device": req.Device, "ok": err == nil, "output": out})
|
|
}
|
|
|
|
func serve(cfgPath string) error {
|
|
cfg, err := loadConfig(cfgPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sealer, err := loadOrCreateKey(cfg.MasterKeyFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
store, err := openStore(ctx, cfg.DatabaseURL)
|
|
cancel()
|
|
if err != nil {
|
|
return fmt.Errorf("database: %w", err)
|
|
}
|
|
app := &App{cfg: cfg, store: store, sealer: sealer, hub: newRackHub()}
|
|
if app.ca, err = loadOrCreateCA(cfg.CADir, "rollcall CA"); err != nil {
|
|
return fmt.Errorf("certificate authority: %w", err)
|
|
}
|
|
ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
|
|
err = app.startRacks(ctx2)
|
|
cancel2()
|
|
if err != nil {
|
|
return fmt.Errorf("rack controllers: %w", err)
|
|
}
|
|
if err := app.startIPAM(); err != nil {
|
|
return fmt.Errorf("ipam: %w", err)
|
|
}
|
|
if err := app.startDHCPRes(); err != nil {
|
|
return err
|
|
}
|
|
if err := app.startDNS(); err != nil {
|
|
return fmt.Errorf("dns: %w", err)
|
|
}
|
|
go app.jobWatchdog()
|
|
go app.resumeFirmware()
|
|
handler := app.routes()
|
|
if cfg.TLSListen != "" {
|
|
names := append([]string{}, cfg.TLSNames...)
|
|
if u, err := url.Parse(cfg.ServerURL); err == nil && u.Hostname() != "" {
|
|
names = append(names, u.Hostname())
|
|
}
|
|
pair, err := regionCert(app.ca, cfg.CADir, names)
|
|
if err != nil {
|
|
return fmt.Errorf("region certificate: %w", err)
|
|
}
|
|
pool := app.ca.pool() // a rotation's other root too: racks and admins keep working
|
|
tsrv := &http.Server{Addr: cfg.TLSListen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute,
|
|
TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{pair}, ClientCAs: pool, ClientAuth: tls.VerifyClientCertIfGiven}}
|
|
go func() {
|
|
log.Printf("rollcall HTTPS on %s (CA %s)", cfg.TLSListen, app.ca.Fingerprint())
|
|
if err := tsrv.ListenAndServeTLS("", ""); err != nil {
|
|
log.Printf("HTTPS listener: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
srv := &http.Server{Addr: cfg.Listen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
|
|
log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "")
|
|
return srv.ListenAndServe()
|
|
}
|