Network model, IPAM, interface layouts and the region/rack split (MAAS-style)
- fabrics, VLANs, subnets, IP ranges (dynamic/reserved/static), IPAM with adoption of addresses machines already have; rollcall fabric|vlan|subnet|range|ip - interface layouts (physical/bond/vlan, NICs by selector: boot, name, mac, fastest:N, ib:N) per machine or per tag; deployment allocates addresses and writes cloud-init network-config plus udev name rules (RHEL cloud images boot with net.ifnames=0); release frees them; rollcall layout, rollcall net show|set - region/rack: rollcall rack long-polls its desired state (per-MAC PXE files kept in the database, dnsmasq configuration and host files) and a BMC command queue, caches boot and OS images (kernels/initrds prefetched for TFTP), proxies the node API and tags requests with its name; the region runs an embedded rack controller; install-rack.sh, rollcall-rack.service - dnsmasq configuration generated per rack from the VLANs it serves (split pool for two racks); reload script restarts only on change and restores the last working config if dnsmasq does not come back (drilled in the lab) - web: Network page, per-machine network tab, layout action, tag layout; console: Network page, network section in the machine view, layout action - tests for candidates, plans (default, bond+VLAN, pinned, DHCP fallback), network-config, udev rules and dnsmasq rendering - lab: pve-vbmc address script finds the interface per BMC address Verified in the lab: embedded rack (10.213.77.0/24) and a remote rack controller on a second SDN segment (10.213.78.0/24): enlistment, commissioning, BMC through the rack's queue, image cache, deployment and phone-home through the rack; Debian 13, Ubuntu 24.04 and Rocky 9 with a static boot NIC and with an active-backup bond carrying VLAN 200. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
e3d9373201
commit
736a7e38a3
31 files changed
+4357
-176
No files matched your search
+51
-48
@@ -11,10 +11,9 @@ import (
|
||||
"io/fs"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -25,6 +24,11 @@ type App struct {
|
||||
cfg *Config
|
||||
store *Store
|
||||
sealer *Sealer
|
||||
hub *rackHub
|
||||
|
||||
netMu sync.RWMutex
|
||||
racks map[string]RackController
|
||||
subnets []Subnet
|
||||
}
|
||||
|
||||
type apiError struct {
|
||||
@@ -146,6 +150,33 @@ func (a *App) routes() http.Handler {
|
||||
}
|
||||
writeJSON(w, 200, jobs)
|
||||
}))
|
||||
// network model and rack controllers
|
||||
mux.HandleFunc("GET /api/v1/fabrics", a.adminAuth(a.handleFabrics))
|
||||
mux.HandleFunc("POST /api/v1/fabrics", a.adminAuth(a.handleFabrics))
|
||||
mux.HandleFunc("DELETE /api/v1/fabrics/{name}", a.adminAuth(a.handleFabricDelete))
|
||||
mux.HandleFunc("GET /api/v1/vlans", a.adminAuth(a.handleVLANs))
|
||||
mux.HandleFunc("POST /api/v1/vlans", a.adminAuth(a.handleVLANs))
|
||||
mux.HandleFunc("DELETE /api/v1/vlans/{fabric}/{vid}", a.adminAuth(a.handleVLANDelete))
|
||||
mux.HandleFunc("GET /api/v1/subnets", a.adminAuth(a.handleSubnets))
|
||||
mux.HandleFunc("POST /api/v1/subnets", a.adminAuth(a.handleSubnets))
|
||||
mux.HandleFunc("DELETE /api/v1/subnets/{cidr...}", a.adminAuth(a.handleSubnetDelete))
|
||||
mux.HandleFunc("POST /api/v1/ranges", a.adminAuth(a.handleRanges))
|
||||
mux.HandleFunc("DELETE /api/v1/ranges/{id}", a.adminAuth(a.handleRangeDelete))
|
||||
mux.HandleFunc("GET /api/v1/ips", a.adminAuth(a.handleIPs))
|
||||
mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs))
|
||||
mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete))
|
||||
mux.HandleFunc("GET /api/v1/layouts", a.adminAuth(a.handleLayouts))
|
||||
mux.HandleFunc("POST /api/v1/layouts", a.adminAuth(a.handleLayouts))
|
||||
mux.HandleFunc("DELETE /api/v1/layouts/{name}", a.adminAuth(a.handleLayoutDelete))
|
||||
mux.HandleFunc("GET /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
|
||||
mux.HandleFunc("GET /api/v1/controllers", a.adminAuth(a.handleControllers))
|
||||
mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete))
|
||||
mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP))
|
||||
mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister))
|
||||
mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState))
|
||||
mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork))
|
||||
mux.HandleFunc("POST /api/v1/rack/result", a.rackAuth(a.handleRackResult))
|
||||
// physical location
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/location", a.adminAuth(a.handleLocation))
|
||||
mux.HandleFunc("POST /api/v1/locations/import", a.adminAuth(a.handleLocationImport))
|
||||
@@ -271,6 +302,8 @@ type Summary struct {
|
||||
ManualTags []string `json:"manual_tags"`
|
||||
AutoTags []string `json:"auto_tags"`
|
||||
Location Location `json:"location"`
|
||||
Rack string `json:"rack_controller"`
|
||||
NetLayout string `json:"net_layout"`
|
||||
}
|
||||
|
||||
func num(v any) float64 {
|
||||
@@ -289,7 +322,8 @@ func summarize(m *Machine) Summary {
|
||||
IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: nodeCtx(m).GPUs, // real accelerators only
|
||||
Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt,
|
||||
OS: m.OS, OSVersion: m.OSVersion, DeployedAt: m.DeployedAt,
|
||||
Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location}
|
||||
Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location,
|
||||
Rack: m.RackController, NetLayout: m.NetLayout}
|
||||
if m.BMC != nil {
|
||||
var b obj
|
||||
if json.Unmarshal(m.BMC, &b) == nil {
|
||||
@@ -325,10 +359,6 @@ LABEL local
|
||||
LOCALBOOT 0
|
||||
`
|
||||
|
||||
func (a *App) pxePath(mac string) string {
|
||||
return filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-"+strings.ReplaceAll(normMAC(mac), ":", "-"))
|
||||
}
|
||||
|
||||
func deref(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
@@ -339,32 +369,10 @@ func deref(p *string) string {
|
||||
func (a *App) writePXE(m *Machine) error {
|
||||
man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial)
|
||||
body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser,
|
||||
m.ID, man, prod, ser, a.cfg.CommissionAppend)
|
||||
m.ID, man, prod, ser, a.appendFor(a.urlFor(m)))
|
||||
return a.writePXEBody(m, body)
|
||||
}
|
||||
|
||||
func (a *App) writePXEBody(m *Machine, body string) error {
|
||||
for _, mac := range m.MACs {
|
||||
p := a.pxePath(mac)
|
||||
if err := os.WriteFile(p+".tmp", []byte(body), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(p+".tmp", p); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("wrote %s", p)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) removePXE(m *Machine) {
|
||||
for _, mac := range m.MACs {
|
||||
if err := os.Remove(a.pxePath(mac)); err == nil {
|
||||
log.Printf("removed %s", a.pxePath(mac))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- handlers
|
||||
|
||||
func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -427,6 +435,7 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail)
|
||||
_ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r))
|
||||
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
|
||||
m = full // carries profile / node token when this is a re-commission of a stateless node
|
||||
}
|
||||
@@ -527,17 +536,7 @@ func (a *App) handleRecommission(w http.ResponseWriter, r *http.Request) {
|
||||
// optional: power-cycle it right away when the caller asks (?now=1)
|
||||
out := obj{"id": m.ID, "state": "recommission"}
|
||||
if r.URL.Query().Get("now") == "1" {
|
||||
if c, err := a.credsFor(m); err == nil {
|
||||
o, ierr := a.ipmi(r.Context(), c, "chassis", "bootdev", "pxe")
|
||||
o2, ierr2 := a.ipmi(r.Context(), c, "chassis", "power", "cycle")
|
||||
if ierr2 != nil && strings.Contains(o2, "off") { // cycle fails when off; power on instead
|
||||
o2, ierr2 = a.ipmi(r.Context(), c, "chassis", "power", "on")
|
||||
}
|
||||
out["ipmi"] = strings.TrimSpace(o + "\n" + o2)
|
||||
out["ipmi_ok"] = ierr == nil && ierr2 == nil
|
||||
} else {
|
||||
out["ipmi"] = err.Error()
|
||||
}
|
||||
out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m)
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
@@ -564,12 +563,11 @@ func (a *App) handlePower(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 400, fmt.Errorf("unknown power action %q", action))
|
||||
return
|
||||
}
|
||||
c, err := a.credsFor(m)
|
||||
if err != nil {
|
||||
out, err := a.bmc(r.Context(), m, args...)
|
||||
if errors.Is(err, errNoBMC) {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
out, err := a.ipmi(r.Context(), c, args...)
|
||||
if action != "status" {
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "power "+action, out)
|
||||
}
|
||||
@@ -589,12 +587,11 @@ func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 400, fmt.Errorf("device must be one of pxe, disk, cdrom, bios, none, safe"))
|
||||
return
|
||||
}
|
||||
c, err := a.credsFor(m)
|
||||
if err != nil {
|
||||
out, err := a.bmc(r.Context(), m, "chassis", "bootdev", req.Device)
|
||||
if errors.Is(err, errNoBMC) {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
out, err := a.ipmi(r.Context(), c, "chassis", "bootdev", req.Device)
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "bootdev "+req.Device, out)
|
||||
writeJSON(w, 200, obj{"id": m.ID, "device": req.Device, "ok": err == nil, "output": out})
|
||||
}
|
||||
@@ -614,7 +611,13 @@ func serve(cfgPath string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("database: %w", err)
|
||||
}
|
||||
app := &App{cfg: cfg, store: store, sealer: sealer}
|
||||
app := &App{cfg: cfg, store: store, sealer: sealer, hub: newRackHub()}
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
err = app.startRacks(ctx2)
|
||||
cancel2()
|
||||
if err != nil {
|
||||
return fmt.Errorf("rack controllers: %w", err)
|
||||
}
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: app.routes(), ReadHeaderTimeout: 10 * time.Second}
|
||||
log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "")
|
||||
return srv.ListenAndServe()
|
||||
|
||||
Reference in new issue
Block a user