Ilker ManapandClaude Opus 5.5
d0ea29d2a9
Found in the lab test of certificate pinning: the CLI printed only 'ipmitool failed on 1 machine(s)'. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
876 lines
31 KiB
Go
876 lines
31 KiB
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"crypto/subtle"
|
|
"crypto/tls"
|
|
"crypto/x509"
|
|
"embed"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"io/fs"
|
|
"log"
|
|
"net"
|
|
"net/http"
|
|
"net/url"
|
|
"regexp"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
)
|
|
|
|
//go:embed web
|
|
var webFS embed.FS
|
|
|
|
type App struct {
|
|
cfg *Config
|
|
store *Store
|
|
sealer *Sealer
|
|
hub *rackHub
|
|
ca *CA
|
|
|
|
netMu sync.RWMutex
|
|
racks map[string]RackController
|
|
subnets []Subnet
|
|
|
|
ipam ipamProvider // nil = built-in IPAM only
|
|
ipamSt *ipamState
|
|
|
|
adminCC adminCertCache
|
|
|
|
dnsp dnsProvider // nil = no direct DNS writes
|
|
dnsSt *dnsState
|
|
}
|
|
|
|
type apiError struct {
|
|
Error string `json:"error"`
|
|
}
|
|
|
|
func writeJSON(w http.ResponseWriter, code int, v any) {
|
|
w.Header().Set("Content-Type", "application/json")
|
|
w.WriteHeader(code)
|
|
enc := json.NewEncoder(w)
|
|
enc.SetIndent("", " ")
|
|
_ = enc.Encode(v)
|
|
}
|
|
|
|
func fail(w http.ResponseWriter, code int, err error) {
|
|
if errors.Is(err, errNotFound) {
|
|
code = http.StatusNotFound
|
|
}
|
|
writeJSON(w, code, apiError{err.Error()})
|
|
}
|
|
|
|
func tokenEqual(a, b string) bool {
|
|
return b != "" && subtle.ConstantTimeCompare([]byte(a), []byte(b)) == 1
|
|
}
|
|
|
|
func (a *App) isAdmin(r *http.Request) bool {
|
|
h := r.Header.Get("Authorization")
|
|
if strings.HasPrefix(h, "Bearer ") && tokenEqual(strings.TrimPrefix(h, "Bearer "), a.cfg.AdminToken) {
|
|
return true
|
|
}
|
|
return a.peerAdmin(r) != "" // an admin client certificate (admincert.go)
|
|
}
|
|
|
|
// adminOverHTTP applies admin_http to requests that did not come over TLS.
|
|
func (a *App) adminOverHTTP(r *http.Request) error {
|
|
if r.TLS != nil || a.cfg.AdminHTTP == "allow" {
|
|
return nil
|
|
}
|
|
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
|
if ip := net.ParseIP(host); a.cfg.AdminHTTP == "loopback" && ip != nil && ip.IsLoopback() {
|
|
return nil
|
|
}
|
|
_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
|
|
h := r.Host
|
|
if hh, _, err := net.SplitHostPort(h); err == nil {
|
|
h = hh
|
|
}
|
|
return fmt.Errorf("the admin API is HTTPS-only here: https://%s (trust the CA from /ca.pem)", net.JoinHostPort(strings.Trim(h, "[]"), port))
|
|
}
|
|
|
|
// httpsURL is the region's HTTPS address for the host the client used.
|
|
func (a *App) httpsURL(r *http.Request) string {
|
|
_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
|
|
h := r.Host
|
|
if hh, _, err := net.SplitHostPort(h); err == nil {
|
|
h = hh
|
|
}
|
|
h = strings.Trim(h, "[]")
|
|
if port == "" || port == "443" {
|
|
if strings.Contains(h, ":") {
|
|
return "https://[" + h + "]"
|
|
}
|
|
return "https://" + h
|
|
}
|
|
return "https://" + net.JoinHostPort(h, port)
|
|
}
|
|
|
|
// uiOverHTTPS sends browsers that reach the web UI over plain HTTP to HTTPS when
|
|
// the admin API is not allowed over HTTP anyway (the page could not work).
|
|
// Only the UI is redirected: nodes keep using HTTP for images and the agent.
|
|
func (a *App) uiOverHTTPS(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
if r.TLS == nil && a.cfg.AdminHTTP != "" && a.cfg.AdminHTTP != "allow" && a.cfg.TLSListen != "" && a.adminOverHTTP(r) != nil {
|
|
http.Redirect(w, r, a.httpsURL(r)+r.URL.RequestURI(), http.StatusPermanentRedirect)
|
|
return
|
|
}
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
func (a *App) adminAuth(next http.HandlerFunc) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
if err := a.adminOverHTTP(r); err != nil {
|
|
fail(w, 403, err)
|
|
return
|
|
}
|
|
if !a.isAdmin(r) {
|
|
w.Header().Set("WWW-Authenticate", `Bearer realm="rollcall"`)
|
|
fail(w, http.StatusUnauthorized, errors.New("admin token required"))
|
|
return
|
|
}
|
|
next(w, r)
|
|
}
|
|
}
|
|
|
|
func (a *App) agentAuth(next http.HandlerFunc) http.HandlerFunc {
|
|
return func(w http.ResponseWriter, r *http.Request) {
|
|
if a.cfg.AgentToken != "" && !tokenEqual(r.Header.Get("X-Rollcall-Token"), a.cfg.AgentToken) && !a.isAdmin(r) {
|
|
fail(w, http.StatusUnauthorized, errors.New("agent token required"))
|
|
return
|
|
}
|
|
next(w, r)
|
|
}
|
|
}
|
|
|
|
func logging(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
start := time.Now()
|
|
rw := &statusWriter{ResponseWriter: w, code: 200}
|
|
r.Body = http.MaxBytesReader(rw, r.Body, 16<<20) // handlers set tighter limits; nothing takes more
|
|
next.ServeHTTP(rw, r)
|
|
if (!strings.HasPrefix(r.URL.Path, "/rollcall/") && !strings.HasPrefix(r.URL.Path, "/images/") && !strings.HasPrefix(r.URL.Path, "/os/")) || rw.code >= 400 {
|
|
uri := r.URL.RequestURI()
|
|
if r.URL.Query().Has("token") { // /?token=... signs the UI in; it does not go to the journal
|
|
uri = r.URL.Path + "?token=REDACTED"
|
|
}
|
|
log.Printf("%s %s %s %d %s", r.RemoteAddr, r.Method, uri, rw.code, time.Since(start).Round(time.Millisecond))
|
|
}
|
|
})
|
|
}
|
|
|
|
type statusWriter struct {
|
|
http.ResponseWriter
|
|
code int
|
|
}
|
|
|
|
func (s *statusWriter) WriteHeader(c int) { s.code = c; s.ResponseWriter.WriteHeader(c) }
|
|
|
|
func (a *App) routes() http.Handler {
|
|
mux := http.NewServeMux()
|
|
// agent endpoints
|
|
mux.HandleFunc("POST /api/v1/commission", a.agentAuth(a.handleCommission))
|
|
mux.HandleFunc("GET /api/v1/bmc-hint", a.agentAuth(a.handleHint))
|
|
// management endpoints
|
|
mux.HandleFunc("GET /api/v1/machines", a.adminAuth(a.handleList))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}", a.adminAuth(a.handleGet))
|
|
mux.HandleFunc("DELETE /api/v1/machines/{id}", a.adminAuth(a.handleDelete))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/recommission", a.adminAuth(a.handleRecommission))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/power", a.adminAuth(a.handlePower))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/power", a.adminAuth(a.handlePower))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/bootdev", a.adminAuth(a.handleBootdev))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/bmc", a.adminAuth(a.handleBMC))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/bmc", a.adminAuth(a.handleBMC))
|
|
mux.HandleFunc("GET /api/v1/power-drivers", a.adminAuth(func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, powerDrivers) }))
|
|
mux.HandleFunc("GET /api/v1/whoami", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
|
writeJSON(w, 200, map[string]any{"ok": true})
|
|
}))
|
|
// stateless provisioning
|
|
mux.HandleFunc("GET /api/v1/profiles", a.adminAuth(a.handleProfiles))
|
|
mux.HandleFunc("POST /api/v1/profiles", a.adminAuth(a.handleProfiles))
|
|
mux.HandleFunc("DELETE /api/v1/profiles/{name}", a.adminAuth(a.handleProfileDelete))
|
|
mux.HandleFunc("GET /api/v1/images", a.adminAuth(a.handleImages))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/assign", a.adminAuth(a.handleAssign))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/unassign", a.adminAuth(a.handleUnassign))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/overlay", a.adminAuth(a.handleAdminOverlay))
|
|
mux.HandleFunc("GET /api/v1/render/{profile}/{path...}", a.adminAuth(a.handleRender))
|
|
mux.HandleFunc("GET /api/v1/nodes/self/overlay", a.handleNodeOverlay) // node token auth inside
|
|
// disk-based deployment
|
|
mux.HandleFunc("GET /api/v1/os", a.adminAuth(a.handleOS))
|
|
mux.HandleFunc("GET /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo))
|
|
mux.HandleFunc("POST /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo))
|
|
mux.HandleFunc("DELETE /api/v1/firmware/{pkg}", a.adminAuth(a.handleFirmwareRemove))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/firmware/update", a.adminAuth(a.handleFirmwareUpdate))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/deploy", a.adminAuth(a.handleDeploy))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/release", a.adminAuth(a.handleRelease))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/jobs", a.adminAuth(a.handleJobs))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/jobs/cancel", a.adminAuth(a.handleJobCancel))
|
|
mux.HandleFunc("GET /api/v1/jobs/self", a.handleJobSelf) // job token auth inside
|
|
mux.HandleFunc("POST /api/v1/jobs/self/progress", a.handleJobProgress) // job token auth inside
|
|
mux.HandleFunc("POST /api/v1/phone-home/{token}", a.handlePhoneHome)
|
|
// fleet-wide views for the UI
|
|
mux.HandleFunc("GET /api/v1/events", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
|
evs, err := a.store.RecentEvents(r.Context(), 100)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, evs)
|
|
}))
|
|
mux.HandleFunc("GET /api/v1/jobs", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
|
jobs, err := a.store.RecentJobs(r.Context(), 100)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
writeJSON(w, 200, jobs)
|
|
}))
|
|
// network model and rack controllers
|
|
mux.HandleFunc("GET /api/v1/fabrics", a.adminAuth(a.handleFabrics))
|
|
mux.HandleFunc("POST /api/v1/fabrics", a.adminAuth(a.handleFabrics))
|
|
mux.HandleFunc("DELETE /api/v1/fabrics/{name}", a.adminAuth(a.handleFabricDelete))
|
|
mux.HandleFunc("GET /api/v1/vlans", a.adminAuth(a.handleVLANs))
|
|
mux.HandleFunc("POST /api/v1/vlans", a.adminAuth(a.handleVLANs))
|
|
mux.HandleFunc("DELETE /api/v1/vlans/{fabric}/{vid}", a.adminAuth(a.handleVLANDelete))
|
|
mux.HandleFunc("GET /api/v1/subnets", a.adminAuth(a.handleSubnets))
|
|
mux.HandleFunc("POST /api/v1/subnets", a.adminAuth(a.handleSubnets))
|
|
mux.HandleFunc("DELETE /api/v1/subnets/{cidr...}", a.adminAuth(a.handleSubnetDelete))
|
|
mux.HandleFunc("POST /api/v1/ranges", a.adminAuth(a.handleRanges))
|
|
mux.HandleFunc("DELETE /api/v1/ranges/{id}", a.adminAuth(a.handleRangeDelete))
|
|
mux.HandleFunc("GET /api/v1/ips", a.adminAuth(a.handleIPs))
|
|
mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs))
|
|
mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete))
|
|
mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM))
|
|
mux.HandleFunc("GET /api/v1/dns", a.adminAuth(a.handleDNS))
|
|
mux.HandleFunc("POST /api/v1/dns/sync", a.adminAuth(a.handleDNSSync))
|
|
mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))
|
|
mux.HandleFunc("POST /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))
|
|
mux.HandleFunc("DELETE /api/v1/admin-certs/{ref}", a.adminAuth(a.handleAdminCertRevoke))
|
|
mux.HandleFunc("POST /api/v1/ipam/sync", a.adminAuth(a.handleIPAMSync))
|
|
mux.HandleFunc("GET /api/v1/ipam/prefixes", a.adminAuth(a.handleIPAMPrefixes))
|
|
mux.HandleFunc("POST /api/v1/ipam/prefixes", a.adminAuth(a.handleIPAMPrefixes))
|
|
mux.HandleFunc("GET /api/v1/layouts", a.adminAuth(a.handleLayouts))
|
|
mux.HandleFunc("POST /api/v1/layouts", a.adminAuth(a.handleLayouts))
|
|
mux.HandleFunc("DELETE /api/v1/layouts/{name}", a.adminAuth(a.handleLayoutDelete))
|
|
mux.HandleFunc("GET /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
|
|
mux.HandleFunc("GET /api/v1/controllers", a.adminAuth(a.handleControllers))
|
|
mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete))
|
|
mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP))
|
|
mux.HandleFunc("GET /ca.pem", a.handleCAPEM)
|
|
mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode))
|
|
mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only
|
|
mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew))
|
|
mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister))
|
|
mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState))
|
|
mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork))
|
|
mux.HandleFunc("POST /api/v1/rack/result", a.rackAuth(a.handleRackResult))
|
|
mux.HandleFunc("GET /api/v1/rack/binary", a.rackAuth(a.handleRackBinary))
|
|
// physical location
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/location", a.adminAuth(a.handleLocation))
|
|
mux.HandleFunc("POST /api/v1/locations/import", a.adminAuth(a.handleLocationImport))
|
|
mux.HandleFunc("GET /api/v1/racks", a.adminAuth(a.handleRacks))
|
|
mux.HandleFunc("POST /api/v1/racks", a.adminAuth(a.handleRacks))
|
|
mux.HandleFunc("DELETE /api/v1/racks", a.adminAuth(a.handleRacks))
|
|
// tags
|
|
mux.HandleFunc("GET /api/v1/tags", a.adminAuth(a.handleTags))
|
|
mux.HandleFunc("POST /api/v1/tags", a.adminAuth(a.handleTags))
|
|
mux.HandleFunc("DELETE /api/v1/tags/{name}", a.adminAuth(a.handleTagDelete))
|
|
mux.HandleFunc("POST /api/v1/machines/{id}/tags", a.adminAuth(a.handleMachineTags))
|
|
// image files for the live boot (no auth: the kernel/initrd/squashfs must be fetchable)
|
|
imgDir := a.cfg.ImageDir
|
|
mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir))))
|
|
mux.Handle("GET /images/", http.StripPrefix("/images/", http.FileServer(http.Dir(a.cfg.ImagesDir))))
|
|
mux.Handle("GET /os/", http.StripPrefix("/os/", http.FileServer(http.Dir(a.cfg.OSDir))))
|
|
mux.HandleFunc("GET /firmware/{pkg}/{file}", a.handleFirmwareFile) // BMCs download packages (no listing)
|
|
// embedded web UI
|
|
sub, _ := fs.Sub(webFS, "web")
|
|
mux.Handle("GET /", a.uiOverHTTPS(uiHeaders(http.FileServer(http.FS(sub)))))
|
|
// Browsers send admin client certificates on their own, so a page elsewhere
|
|
// could post to the API with the admin's identity: cross-origin requests
|
|
// that change state are refused (Sec-Fetch-Site / Origin). The CLI, agents
|
|
// and racks send neither header and are not affected.
|
|
return logging(http.NewCrossOriginProtection().Handler(mux))
|
|
}
|
|
|
|
// uiHeaders: the UI loads only its own scripts and cannot be framed, so
|
|
// markup that slips through (machine-reported strings) does not run.
|
|
func uiHeaders(next http.Handler) http.Handler {
|
|
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
|
h := w.Header()
|
|
h.Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; object-src 'none'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'")
|
|
h.Set("X-Content-Type-Options", "nosniff")
|
|
h.Set("X-Frame-Options", "DENY")
|
|
h.Set("Referrer-Policy", "no-referrer")
|
|
next.ServeHTTP(w, r)
|
|
})
|
|
}
|
|
|
|
// ---------------------------------------------------------------- helpers over inventory JSON
|
|
|
|
type obj = map[string]any
|
|
|
|
func dig(m any, path ...string) any {
|
|
cur := m
|
|
for _, p := range path {
|
|
mm, ok := cur.(obj)
|
|
if !ok {
|
|
return nil
|
|
}
|
|
cur = mm[p]
|
|
}
|
|
return cur
|
|
}
|
|
|
|
func str(v any) string {
|
|
switch t := v.(type) {
|
|
case nil:
|
|
return ""
|
|
case string:
|
|
return t
|
|
case float64:
|
|
return fmt.Sprintf("%v", t)
|
|
default:
|
|
return fmt.Sprintf("%v", t)
|
|
}
|
|
}
|
|
|
|
func strPtr(v any) *string {
|
|
s := str(v)
|
|
if s == "" {
|
|
return nil
|
|
}
|
|
return &s
|
|
}
|
|
|
|
func listLen(v any) int {
|
|
if l, ok := v.([]any); ok {
|
|
return len(l)
|
|
}
|
|
return 0
|
|
}
|
|
|
|
// normMAC is the canonical form of a hardware address, "" when s is not one.
|
|
// MACs name files (pxelinux.cfg/01-...) and dnsmasq lines, so nothing else passes.
|
|
func normMAC(s string) string {
|
|
hw, err := net.ParseMAC(strings.TrimSpace(s))
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
return hw.String()
|
|
}
|
|
|
|
var nonAlnum = regexp.MustCompile(`[^a-z0-9]`)
|
|
|
|
func machineID(uuid, serial string, macs []string) string {
|
|
u := strings.ToLower(uuid)
|
|
if u != "" && u != "00000000-0000-0000-0000-000000000000" && !strings.Contains(u, "not ") &&
|
|
nonAlnum.ReplaceAllString(u, "") != "" {
|
|
u = nonAlnum.ReplaceAllString(u, "") // the ID names files and URLs: letters and digits only
|
|
if len(u) > 12 {
|
|
u = u[:12]
|
|
}
|
|
return "m-" + u
|
|
}
|
|
if s := nonAlnum.ReplaceAllString(strings.ToLower(serial), ""); s != "" && !strings.Contains(strings.ToLower(serial), "not ") {
|
|
if len(s) > 16 {
|
|
s = s[:16]
|
|
}
|
|
return "m-" + s
|
|
}
|
|
if len(macs) > 0 {
|
|
return "m-" + strings.ReplaceAll(macs[0], ":", "")
|
|
}
|
|
return "m-" + randomToken(6)
|
|
}
|
|
|
|
// Summary is the list-view projection of a machine (no secrets, no full inventory).
|
|
type Summary struct {
|
|
ID string `json:"id"`
|
|
State string `json:"state"`
|
|
Hostname *string `json:"hostname"`
|
|
Serial *string `json:"serial"`
|
|
UUID *string `json:"uuid"`
|
|
Manufacturer *string `json:"manufacturer"`
|
|
Product *string `json:"product"`
|
|
FirstSeen time.Time `json:"first_seen"`
|
|
LastSeen *time.Time `json:"last_seen"`
|
|
MACs []string `json:"macs"`
|
|
CPU string `json:"cpu"`
|
|
CPUs string `json:"cpus"`
|
|
MemoryBytes float64 `json:"memory_bytes"`
|
|
Disks float64 `json:"disks"`
|
|
StorageBytes float64 `json:"storage_bytes"`
|
|
NICs int `json:"nics"`
|
|
IBHCAs int `json:"ib_hcas"`
|
|
GPUs int `json:"gpus"`
|
|
BMC *obj `json:"bmc"`
|
|
Profile *string `json:"profile"`
|
|
NodeName *string `json:"node_name"`
|
|
MgmtIP *string `json:"mgmt_ip"`
|
|
IBIP *string `json:"ib_ip"`
|
|
BootedAt *time.Time `json:"booted_at"`
|
|
OS *string `json:"os"`
|
|
OSVersion *string `json:"os_version"`
|
|
DeployedAt *time.Time `json:"deployed_at"`
|
|
Job *Job `json:"job,omitempty"` // active deployment job, if any
|
|
Tags []string `json:"tags"` // effective (manual + auto)
|
|
ManualTags []string `json:"manual_tags"`
|
|
AutoTags []string `json:"auto_tags"`
|
|
Location Location `json:"location"`
|
|
Rack string `json:"rack_controller"`
|
|
NetLayout string `json:"net_layout"`
|
|
}
|
|
|
|
func num(v any) float64 {
|
|
f, _ := v.(float64)
|
|
return f
|
|
}
|
|
|
|
func summarize(m *Machine) Summary {
|
|
var inv obj
|
|
_ = json.Unmarshal(m.Inventory, &inv)
|
|
s := Summary{ID: m.ID, State: m.State, Hostname: m.Hostname, Serial: m.Serial, UUID: m.UUID,
|
|
Manufacturer: m.Manufacturer, Product: m.Product, FirstSeen: m.FirstSeen, LastSeen: m.LastSeen, MACs: m.MACs,
|
|
CPU: str(dig(inv, "cpu", "Model name")), CPUs: str(dig(inv, "cpu", "CPU(s)")),
|
|
MemoryBytes: num(dig(inv, "memory", "total_bytes")), Disks: num(dig(inv, "storage", "disk_count")),
|
|
StorageBytes: num(dig(inv, "storage", "total_bytes")), NICs: listLen(inv["network"]),
|
|
IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: nodeCtx(m).GPUs, // real accelerators only
|
|
Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt,
|
|
OS: m.OS, OSVersion: m.OSVersion, DeployedAt: m.DeployedAt,
|
|
Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location,
|
|
Rack: m.RackController, NetLayout: m.NetLayout}
|
|
if m.BMC != nil {
|
|
var b obj
|
|
if json.Unmarshal(m.BMC, &b) == nil {
|
|
drv := str(b["driver"])
|
|
if drv == "" {
|
|
drv = "ipmi"
|
|
}
|
|
s.BMC = &obj{"source": b["source"], "address": b["address"], "user": b["user"], "driver": drv,
|
|
"configured": b["configured"], "verified_oob": b["verified_oob"]}
|
|
}
|
|
}
|
|
return s
|
|
}
|
|
|
|
// ---------------------------------------------------------------- pxelinux per-MAC files
|
|
|
|
const commissionedTmpl = `# generated by rollcall %s
|
|
# machine %s %s %s serial=%s
|
|
DEFAULT menu.c32
|
|
PROMPT 0
|
|
TIMEOUT 150
|
|
MENU TITLE %s - commissioned (%s %s, S/N %s)
|
|
|
|
LABEL poweroff
|
|
MENU LABEL Commissioned: power off (default in 15s)
|
|
MENU DEFAULT
|
|
KERNEL poweroff.c32
|
|
|
|
LABEL recommission
|
|
MENU LABEL Re-run commissioning
|
|
KERNEL rollcall/vmlinuz
|
|
INITRD rollcall/initrd.img
|
|
APPEND %s
|
|
|
|
LABEL local
|
|
MENU LABEL Boot from local disk
|
|
LOCALBOOT 0
|
|
`
|
|
|
|
func deref(p *string) string {
|
|
if p == nil {
|
|
return ""
|
|
}
|
|
return *p
|
|
}
|
|
|
|
func (a *App) writePXE(m *Machine) error {
|
|
man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial)
|
|
body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser,
|
|
m.ID, man, prod, ser, a.appendFor(a.urlFor(m)))
|
|
return a.writePXEBody(m, body)
|
|
}
|
|
|
|
// ---------------------------------------------------------------- handlers
|
|
|
|
func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
|
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 4<<20))
|
|
if err != nil {
|
|
fail(w, 400, err)
|
|
return
|
|
}
|
|
var inv obj
|
|
if err := json.Unmarshal(body, &inv); err != nil {
|
|
fail(w, 400, fmt.Errorf("bad json: %w", err))
|
|
return
|
|
}
|
|
ident, _ := inv["identity"].(obj)
|
|
var macs []string
|
|
if l, ok := ident["macs"].([]any); ok {
|
|
for _, v := range l {
|
|
if s := normMAC(str(v)); s != "" {
|
|
macs = append(macs, s)
|
|
}
|
|
}
|
|
}
|
|
if boot := normMAC(str(ident["boot_mac"])); boot != "" {
|
|
for i, mac := range macs {
|
|
if mac == boot && i != 0 {
|
|
macs = append([]string{boot}, append(macs[:i], macs[i+1:]...)...)
|
|
break
|
|
}
|
|
}
|
|
}
|
|
// identity strings come from the machine: one line each, they go into PXE menus and logs
|
|
line := func(v any) *string {
|
|
s := strings.Map(func(c rune) rune {
|
|
if c < ' ' || c == 0x7f {
|
|
return ' '
|
|
}
|
|
return c
|
|
}, str(v))
|
|
if s = strings.TrimSpace(s); len(s) > 128 {
|
|
s = s[:128]
|
|
}
|
|
return strPtr(s)
|
|
}
|
|
m := &Machine{
|
|
ID: machineID(str(ident["uuid"]), str(ident["serial"]), macs),
|
|
UUID: line(ident["uuid"]),
|
|
Serial: line(ident["serial"]),
|
|
Hostname: line(inv["hostname"]),
|
|
Manufacturer: line(dig(inv, "system", "system-manufacturer")),
|
|
Product: line(dig(inv, "system", "system-product-name")),
|
|
MACs: macs,
|
|
}
|
|
// Pull the BMC password out of the payload before anything is persisted.
|
|
var secret []byte
|
|
detail := "bmc=none"
|
|
if bmc, _ := inv["bmc"].(obj); bmc == nil || str(bmc["source"]) == "hint" || str(bmc["address"]) == "" {
|
|
// a configured out-of-band BMC: the region's credentials, whatever the agent sent
|
|
if h := a.hintFor(macs); h != nil {
|
|
port := h.Port
|
|
if port == 0 {
|
|
port = 623
|
|
}
|
|
nb := obj{"source": "hint", "address": h.Address, "port": port, "user": h.User, "configured": true, "password": "<from bmc_hints>"}
|
|
if bmc != nil {
|
|
nb["agent"] = obj{"source": bmc["source"], "errors": bmc["errors"]}
|
|
}
|
|
inv["bmc"] = nb
|
|
if h.Password != "" {
|
|
if secret, err = a.sealer.Seal(h.Password, m.ID); err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
}
|
|
}
|
|
}
|
|
if bmc, ok := inv["bmc"].(obj); ok {
|
|
if pw := str(bmc["password"]); pw != "" && str(bmc["source"]) != "hint" {
|
|
secret, err = a.sealer.Seal(pw, m.ID)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
bmc["password"] = "<stored encrypted>"
|
|
}
|
|
m.BMC, _ = json.Marshal(bmc)
|
|
detail = fmt.Sprintf("bmc=%s:%s configured=%v", str(bmc["source"]), str(bmc["address"]), bmc["configured"])
|
|
}
|
|
m.Inventory, _ = json.Marshal(inv)
|
|
|
|
// Anyone on the provisioning network can post here, so a known machine is
|
|
// only re-commissioned when that is expected: an operator asked
|
|
// (recommission), or it has nothing to lose (ready, a failed job). A report
|
|
// for a deployed or running machine is refused, not merged.
|
|
old, oldErr := a.store.Get(r.Context(), m.ID)
|
|
if oldErr == nil && !recommissionable(old.State) {
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "commission refused", fmt.Sprintf("state %s; from %s", old.State, r.RemoteAddr))
|
|
log.Printf("commission refused for %s (state %s) from %s", m.ID, old.State, r.RemoteAddr)
|
|
fail(w, 409, fmt.Errorf("machine %s is %s; recommission it first", m.ID, old.State))
|
|
return
|
|
}
|
|
// a BMC an operator configured (driver_set) survives commissioning; what the agent
|
|
// found is kept next to it for reference
|
|
if oldErr == nil && old.BMC != nil {
|
|
var ob obj
|
|
if json.Unmarshal(old.BMC, &ob) == nil && ob["driver_set"] == true {
|
|
var found obj
|
|
_ = json.Unmarshal(m.BMC, &found)
|
|
delete(found, "password")
|
|
ob["commissioning"] = found
|
|
m.BMC, _ = json.Marshal(ob)
|
|
secret = old.bmcSecret
|
|
detail += " (operator BMC settings kept)"
|
|
}
|
|
}
|
|
|
|
isNew, err := a.store.UpsertCommissioned(r.Context(), m, secret)
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail)
|
|
if isNew || oldErr != nil || old.State == "recommission" || old.RackController == "" {
|
|
_ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r)) // a ready machine is not moved by whoever reports it
|
|
}
|
|
go func(id string) { // check a hinted BMC from where it is reachable, then look for Redfish
|
|
ctx := context.Background()
|
|
if full, err := a.store.Get(ctx, id); err == nil {
|
|
a.verifyHintedBMC(full)
|
|
}
|
|
if full, err := a.store.Get(ctx, id); err == nil {
|
|
a.probeBMC(full)
|
|
}
|
|
}(m.ID)
|
|
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
|
|
m = full // carries profile / node token when this is a re-commission of a stateless node
|
|
}
|
|
a.retag(r.Context(), []*Machine{m})
|
|
if err := a.writePXEAny(r.Context(), m); err != nil {
|
|
log.Printf("pxe write failed for %s: %v", m.ID, err)
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "pxe-write-failed", err.Error())
|
|
}
|
|
log.Printf("commissioned %s new=%v (%s %s, serial %s, %d macs) %s", m.ID, isNew, deref(m.Manufacturer),
|
|
deref(m.Product), deref(m.Serial), len(macs), detail)
|
|
writeJSON(w, 200, map[string]any{"id": m.ID, "state": "ready", "new": isNew})
|
|
}
|
|
|
|
func recommissionable(state string) bool {
|
|
return state == "new" || state == "ready" || state == "recommission" || strings.HasSuffix(state, "-failed")
|
|
}
|
|
|
|
// hintFor is the configured out-of-band BMC for one of these MACs.
|
|
func (a *App) hintFor(macs []string) *BMCHint {
|
|
want := map[string]bool{}
|
|
for _, m := range macs {
|
|
if n := normMAC(m); n != "" {
|
|
want[n] = true
|
|
}
|
|
}
|
|
for i, h := range a.cfg.BMCHints {
|
|
if want[normMAC(h.MAC)] {
|
|
return &a.cfg.BMCHints[i]
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// handleHint tells a commissioning agent that a BMC is configured for it: the
|
|
// address, never the password (the agent token is public, MACs are visible on
|
|
// the wire). The region attaches the credentials itself when the machine
|
|
// reports in (handleCommission) and checks them from the rack controller.
|
|
func (a *App) handleHint(w http.ResponseWriter, r *http.Request) {
|
|
h := a.hintFor(strings.Split(r.URL.Query().Get("macs"), ","))
|
|
if h == nil {
|
|
writeJSON(w, 200, obj{})
|
|
return
|
|
}
|
|
port := h.Port
|
|
if port == 0 {
|
|
port = 623
|
|
}
|
|
writeJSON(w, 200, obj{"address": h.Address, "port": port, "user": h.User, "credentials": "on the region"})
|
|
}
|
|
|
|
func (a *App) handleList(w http.ResponseWriter, r *http.Request) {
|
|
ms, err := a.store.List(r.Context())
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
active, _ := a.store.ActiveJobs(r.Context())
|
|
wantTag := r.URL.Query().Get("tag")
|
|
out := make([]Summary, 0, len(ms))
|
|
for _, m := range ms {
|
|
if wantTag != "" && !hasTag(effectiveTags(m), wantTag) {
|
|
continue
|
|
}
|
|
if !locationMatch(m, r.URL.Query()) {
|
|
continue
|
|
}
|
|
s := summarize(m)
|
|
s.Job = active[m.ID]
|
|
out = append(out, s)
|
|
}
|
|
writeJSON(w, 200, out)
|
|
}
|
|
|
|
func (a *App) handleGet(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
evs, _ := a.store.Events(r.Context(), m.ID)
|
|
jobs, _ := a.store.Jobs(r.Context(), m.ID, 10)
|
|
s := summarize(m)
|
|
if j, err := a.store.ActiveJob(r.Context(), m.ID); err == nil {
|
|
s.Job = j
|
|
}
|
|
var bmc, inv any
|
|
_ = json.Unmarshal(m.BMC, &bmc)
|
|
_ = json.Unmarshal(m.Inventory, &inv)
|
|
writeJSON(w, 200, obj{"summary": s, "bmc": bmc, "has_bmc_secret": m.bmcSecret != nil,
|
|
"inventory": inv, "events": evs, "jobs": jobs})
|
|
}
|
|
|
|
func (a *App) handleDelete(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
a.removePXE(m)
|
|
defer a.kickIPAM()
|
|
if err := a.store.Delete(r.Context(), m.ID); err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
log.Printf("deleted %s", m.ID)
|
|
writeJSON(w, 200, obj{"deleted": m.ID})
|
|
}
|
|
|
|
func (a *App) handleRecommission(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
a.removePXE(m)
|
|
_ = a.store.SetState(r.Context(), m.ID, "recommission")
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "recommission requested", "")
|
|
// optional: power-cycle it right away when the caller asks (?now=1)
|
|
out := obj{"id": m.ID, "state": "recommission"}
|
|
if r.URL.Query().Get("now") == "1" {
|
|
out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m)
|
|
}
|
|
writeJSON(w, 200, out)
|
|
}
|
|
|
|
func (a *App) handlePower(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
action := "status"
|
|
if r.Method == http.MethodPost {
|
|
var req struct {
|
|
Action string `json:"action"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
|
fail(w, 400, err)
|
|
return
|
|
}
|
|
action = req.Action
|
|
}
|
|
if !powerOps[action] || action == "bootdev" || action == "probe" {
|
|
fail(w, 400, fmt.Errorf("unknown power action %q", action))
|
|
return
|
|
}
|
|
out, err := a.power(r.Context(), m, action, "")
|
|
if errors.Is(err, errNoBMC) {
|
|
fail(w, 409, err)
|
|
return
|
|
}
|
|
if action != "status" {
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "power "+action, out)
|
|
}
|
|
res := obj{"id": m.ID, "action": action, "ok": err == nil, "output": out}
|
|
if err != nil {
|
|
res["error"] = err.Error()
|
|
}
|
|
writeJSON(w, 200, res)
|
|
}
|
|
|
|
func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) {
|
|
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
|
if err != nil {
|
|
fail(w, 500, err)
|
|
return
|
|
}
|
|
var req struct {
|
|
Device string `json:"device"`
|
|
}
|
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || !bootDevices[req.Device] {
|
|
fail(w, 400, fmt.Errorf("device must be one of pxe, disk, cdrom, bios, none, safe"))
|
|
return
|
|
}
|
|
out, err := a.power(r.Context(), m, "bootdev", req.Device)
|
|
if errors.Is(err, errNoBMC) {
|
|
fail(w, 409, err)
|
|
return
|
|
}
|
|
_ = a.store.AddEvent(r.Context(), m.ID, "bootdev "+req.Device, out)
|
|
writeJSON(w, 200, obj{"id": m.ID, "device": req.Device, "ok": err == nil, "output": out})
|
|
}
|
|
|
|
func serve(cfgPath string) error {
|
|
cfg, err := loadConfig(cfgPath)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
sealer, err := loadOrCreateKey(cfg.MasterKeyFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
|
store, err := openStore(ctx, cfg.DatabaseURL)
|
|
cancel()
|
|
if err != nil {
|
|
return fmt.Errorf("database: %w", err)
|
|
}
|
|
app := &App{cfg: cfg, store: store, sealer: sealer, hub: newRackHub()}
|
|
if app.ca, err = loadOrCreateCA(cfg.CADir, "rollcall CA"); err != nil {
|
|
return fmt.Errorf("certificate authority: %w", err)
|
|
}
|
|
ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
|
|
err = app.startRacks(ctx2)
|
|
cancel2()
|
|
if err != nil {
|
|
return fmt.Errorf("rack controllers: %w", err)
|
|
}
|
|
if err := app.startIPAM(); err != nil {
|
|
return fmt.Errorf("ipam: %w", err)
|
|
}
|
|
if err := app.startDNS(); err != nil {
|
|
return fmt.Errorf("dns: %w", err)
|
|
}
|
|
go app.jobWatchdog()
|
|
go app.resumeFirmware()
|
|
handler := app.routes()
|
|
if cfg.TLSListen != "" {
|
|
names := append([]string{}, cfg.TLSNames...)
|
|
if u, err := url.Parse(cfg.ServerURL); err == nil && u.Hostname() != "" {
|
|
names = append(names, u.Hostname())
|
|
}
|
|
pair, err := regionCert(app.ca, cfg.CADir, names)
|
|
if err != nil {
|
|
return fmt.Errorf("region certificate: %w", err)
|
|
}
|
|
pool := x509.NewCertPool()
|
|
pool.AddCert(app.ca.Cert)
|
|
tsrv := &http.Server{Addr: cfg.TLSListen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute,
|
|
TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{pair}, ClientCAs: pool, ClientAuth: tls.VerifyClientCertIfGiven}}
|
|
go func() {
|
|
log.Printf("rollcall HTTPS on %s (CA %s)", cfg.TLSListen, app.ca.Fingerprint())
|
|
if err := tsrv.ListenAndServeTLS("", ""); err != nil {
|
|
log.Printf("HTTPS listener: %v", err)
|
|
}
|
|
}()
|
|
}
|
|
srv := &http.Server{Addr: cfg.Listen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
|
|
log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "")
|
|
return srv.ListenAndServe()
|
|
}
|