Network model, IPAM, interface layouts and the region/rack split (MAAS-style)
CI / agent (push) Successful in 3s
CI / site (push) Successful in 2s
CI / go (push) Successful in 32s

- fabrics, VLANs, subnets, IP ranges (dynamic/reserved/static), IPAM with adoption of
  addresses machines already have; rollcall fabric|vlan|subnet|range|ip
- interface layouts (physical/bond/vlan, NICs by selector: boot, name, mac, fastest:N, ib:N)
  per machine or per tag; deployment allocates addresses and writes cloud-init network-config
  plus udev name rules (RHEL cloud images boot with net.ifnames=0); release frees them;
  rollcall layout, rollcall net show|set
- region/rack: rollcall rack long-polls its desired state (per-MAC PXE files kept in the
  database, dnsmasq configuration and host files) and a BMC command queue, caches boot and OS
  images (kernels/initrds prefetched for TFTP), proxies the node API and tags requests with its
  name; the region runs an embedded rack controller; install-rack.sh, rollcall-rack.service
- dnsmasq configuration generated per rack from the VLANs it serves (split pool for two racks);
  reload script restarts only on change and restores the last working config if dnsmasq does
  not come back (drilled in the lab)
- web: Network page, per-machine network tab, layout action, tag layout; console: Network page,
  network section in the machine view, layout action
- tests for candidates, plans (default, bond+VLAN, pinned, DHCP fallback), network-config,
  udev rules and dnsmasq rendering
- lab: pve-vbmc address script finds the interface per BMC address

Verified in the lab: embedded rack (10.213.77.0/24) and a remote rack controller on a second
SDN segment (10.213.78.0/24): enlistment, commissioning, BMC through the rack's queue, image
cache, deployment and phone-home through the rack; Debian 13, Ubuntu 24.04 and Rocky 9 with a
static boot NIC and with an active-backup bond carrying VLAN 200.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-27 23:58:40 +01:00
1 parent e3d9373201
commit 736a7e38a3
31 files changed
+4357 -176

No files matched your search

+5 -2
View File
@@ -225,6 +225,8 @@ func runClient(cmd string, args []string) error {
return c.tagCmd(cmd, args)
case "location":
return c.locationCmd(args)
case "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp":
return c.netCmd(cmd, args)
default:
return c.statelessCmd(cmd, args)
}
@@ -318,7 +320,7 @@ func (c *client) tagCmd(cmd string, args []string) error {
}
for _, t := range defs {
if t.Name == args[1] {
fmt.Printf("tag %s (%d machines)\ncomment: %s\nmatch: %s\noverlays: %v\n--- cloud_config ---\n%s\n", t.Name, t.Machines, t.Comment, t.Match, t.Overlays, t.CloudConfig)
fmt.Printf("tag %s (%d machines)\ncomment: %s\nmatch: %s\noverlays: %v\nlayout: %s\n--- cloud_config ---\n%s\n", t.Name, t.Machines, t.Comment, t.Match, t.Overlays, orDash(t.NetLayout), t.CloudConfig)
return nil
}
}
@@ -331,9 +333,10 @@ func (c *client) tagCmd(cmd string, args []string) error {
match := fs.String("match", "", "Go template over .Node that renders true, e.g. '{{ gt .Node.GPUs 0 }}'")
ov := fs.String("overlays", "", "comma-separated overlays for stateless nodes with this tag")
cc := fs.String("cloud-config", "", "file with a #cloud-config or #!/bin/sh fragment applied at deployment")
layout := fs.String("layout", "", "network layout for machines with this tag")
comment := fs.String("comment", "", "comment")
_ = fs.Parse(args[2:])
t := TagDef{Name: args[1], Match: *match, Comment: *comment}
t := TagDef{Name: args[1], Match: *match, Comment: *comment, NetLayout: *layout}
if *ov != "" {
t.Overlays = strings.Split(*ov, ",")
}
+8
View File
@@ -32,6 +32,11 @@ type Config struct {
// disk-based deployment
OSDir string `json:"os_dir"` // <os_dir>/<name>/<version>/{disk.raw.zst,meta.json}, served under /os/
OSSources string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync`
// region/rack: this server runs an embedded rack controller named RackName;
// remote rack controllers (rollcall rack) authenticate with RackSecret
RackName string `json:"rack_name"`
RackSecret string `json:"rack_secret"`
}
// BMCHint maps a machine MAC to a known BMC (virtual lab nodes).
@@ -78,5 +83,8 @@ func loadConfig(path string) (*Config, error) {
if c.OSSources == "" {
c.OSSources = "/etc/rollcall/os-sources.json"
}
if c.RackName == "" {
c.RackName, _ = os.Hostname()
}
return c, nil
}
+59 -17
View File
@@ -49,6 +49,8 @@ type DeploySpec struct {
Packages []string `json:"packages,omitempty"`
UserData string `json:"user_data,omitempty"` // full #cloud-config override
Wipe string `json:"wipe,omitempty"` // release: quick | full
NetworkConfig string `json:"network_config,omitempty"`
LinkRules string `json:"link_rules,omitempty"`
}
type seedFile struct {
@@ -165,7 +167,11 @@ func (s *Store) SetDeployed(ctx context.Context, id, os, version string) error {
}
func (s *Store) SetReleased(ctx context.Context, id string) error {
_, err := s.pool.Exec(ctx, `UPDATE machines SET state='ready', os=NULL, os_version=NULL, deployed_at=NULL, booted_at=NULL, last_seen=now() WHERE id=$1`, id)
if _, err := s.pool.Exec(ctx, `DELETE FROM ip_addresses WHERE machine_id=$1 AND kind='auto'`, id); err != nil {
return err
}
_, err := s.pool.Exec(ctx, `UPDATE machines SET state='ready', os=NULL, os_version=NULL, deployed_at=NULL, booted_at=NULL, last_seen=now(),
mgmt_ip=(SELECT ip FROM ip_addresses a WHERE a.machine_id=machines.id ORDER BY ip LIMIT 1), ib_ip=NULL WHERE id=$1`, id)
return err
}
@@ -253,6 +259,11 @@ func yamlList(l []string) string {
}
func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile {
base := a.cfg.ServerURL
m, merr := a.store.Get(context.Background(), j.MachineID)
if merr == nil {
base = a.urlFor(m)
}
meta := fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", j.ID, sp.Hostname)
ud := sp.UserData
if ud == "" {
@@ -274,20 +285,27 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile {
if len(sp.Packages) > 0 {
fmt.Fprintf(&b, "packages: %s\n", yamlList(sp.Packages))
}
fmt.Fprintf(&b, "phone_home:\n url: %s/api/v1/phone-home/%s\n post: [instance_id, hostname, fqdn, pub_key_ed25519]\n tries: 10\n", a.cfg.ServerURL, token)
fmt.Fprintf(&b, "phone_home:\n url: %s/api/v1/phone-home/%s\n post: [instance_id, hostname, fqdn, pub_key_ed25519]\n tries: 10\n", base, token)
b.WriteString("final_message: \"rollcall: deployed $hostname up after $UPTIME s\"\n")
ud = b.String()
}
if m, err := a.store.Get(context.Background(), j.MachineID); err == nil {
if merr == nil {
if frags, err := a.tagFragments(context.Background(), m); err == nil && len(frags) > 0 {
ud = multipartUserData(ud, frags)
}
}
return []seedFile{
files := []seedFile{
{Path: "var/lib/cloud/seed/nocloud/meta-data", Mode: "0600", Content: meta},
{Path: "var/lib/cloud/seed/nocloud/user-data", Mode: "0600", Content: ud},
{Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\n"},
}
if sp.NetworkConfig != "" {
files = append(files, seedFile{Path: "var/lib/cloud/seed/nocloud/network-config", Mode: "0600", Content: sp.NetworkConfig})
}
if sp.LinkRules != "" {
files = append(files, seedFile{Path: "etc/udev/rules.d/70-rollcall-net.rules", Mode: "0644", Content: sp.LinkRules})
}
return files
}
// ---------------------------------------------------------------- PXE
@@ -337,9 +355,9 @@ LABEL recommission
`
// deployAppend derives the deploy-mode cmdline from commission_append (same image, no agent token).
func (a *App) deployAppend(token string) string {
func (a *App) deployAppend(base, token string) string {
var parts []string
for _, p := range strings.Fields(a.cfg.CommissionAppend) {
for _, p := range strings.Fields(a.appendFor(base)) {
if strings.HasPrefix(p, "rollcall.token=") || strings.HasPrefix(p, "rollcall.nopoweroff") {
continue
}
@@ -351,13 +369,13 @@ func (a *App) deployAppend(token string) string {
func (a *App) writePXEDeploy(m *Machine, j *Job, sp DeploySpec, token string) error {
body := fmt.Sprintf(deployTmpl, time.Now().Format(time.RFC3339), m.ID, j.ID, j.Kind, sp.OS, sp.Version, sp.Disk,
m.ID, j.Kind, sp.OS, j.ID, a.deployAppend(token), a.cfg.CommissionAppend)
m.ID, j.Kind, sp.OS, j.ID, a.deployAppend(a.urlFor(m), token), a.appendFor(a.urlFor(m)))
return a.writePXEBody(m, body)
}
func (a *App) writePXEDeployed(m *Machine) error {
body := fmt.Sprintf(deployedTmpl, time.Now().Format(time.RFC3339), m.ID, deref(m.OS), deref(m.OSVersion),
deref(m.NodeName), deref(m.OS), m.ID, deref(m.OS), a.cfg.CommissionAppend)
deref(m.NodeName), deref(m.OS), m.ID, deref(m.OS), a.appendFor(a.urlFor(m)))
return a.writePXEBody(m, body)
}
@@ -443,40 +461,64 @@ func (a *App) handleDeploy(w http.ResponseWriter, r *http.Request) {
fail(w, 409, err)
return
}
// node name + management IP: keep an existing one, otherwise allocate like a stateless node
// node name: keep the existing one unless another is asked for
name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP)
if req.Hostname != "" && req.Hostname != name {
name = ""
}
if name == "" {
if name, mgmt, ib, err = a.allocate(r.Context(), str(a.cfg.ClusterVars["name_format"]), req.Hostname); err != nil {
if name, _, _, err = a.allocatePools(r.Context(), str(a.cfg.ClusterVars["name_format"]), req.Hostname, false); err != nil {
fail(w, 409, err)
return
}
if err := a.store.SetNodeAddr(r.Context(), m.ID, name, mgmt, ib); err != nil {
fail(w, 500, err)
m.NodeName = &name
}
// addresses: the network plan (layout, IPAM); without a network model, the legacy pools
netcfg, links := "", ""
var warnings []string
if len(a.subnetsSnapshot()) > 0 {
plan, err := a.netPlan(r.Context(), m, true)
if err != nil {
fail(w, 409, err)
return
}
m, _ = a.store.Get(r.Context(), m.ID)
if err := a.exportDnsmasq(r.Context()); err != nil {
log.Printf("dnsmasq export: %v", err)
warnings = plan.Warnings
if pm, pib := plan.primaryIPs(); pm != "" {
mgmt, ib = pm, pib
}
nb, _ := json.MarshalIndent(networkConfig(plan), "", " ")
netcfg = string(nb) + "\n"
links = linkRules(plan)
} else if mgmt == "" {
if _, mgmt, ib, err = a.allocatePools(r.Context(), "", name, true); err != nil {
fail(w, 409, err)
return
}
}
if err := a.store.SetNodeAddr(r.Context(), m.ID, name, mgmt, ib); err != nil {
fail(w, 500, err)
return
}
m, _ = a.store.Get(r.Context(), m.ID)
keys := req.SSHKeys
if keys == nil {
keys = a.configSSHKeys()
}
sp := DeploySpec{OS: meta.Name, Version: meta.Version,
ImageURL: a.cfg.ServerURL + "/os/" + meta.Name + "/" + meta.Version + "/disk.raw.zst",
ImageURL: a.urlFor(m) + "/os/" + meta.Name + "/" + meta.Version + "/disk.raw.zst",
SHA256: meta.SHA256, Size: meta.Size, RawSize: meta.RawSize, Disk: disk.Path, DiskSerial: disk.Serial,
Hostname: name, Domain: str(a.cfg.ClusterVars["domain"]), SSHKeys: keys, PackageUpgrade: req.PackageUpgrade,
Packages: req.Packages, UserData: req.UserData}
Packages: req.Packages, UserData: req.UserData, NetworkConfig: netcfg, LinkRules: links}
out, err := a.startJob(r.Context(), m, "deploy", sp, req.Boot)
if err != nil {
fail(w, 409, err)
return
}
out["mgmt_ip"] = mgmt
out["rack"] = a.rackOf(m)
if len(warnings) > 0 {
out["warnings"] = warnings
}
writeJSON(w, 200, out)
}
-19
View File
@@ -1,14 +1,8 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"os/exec"
"strconv"
"strings"
"time"
)
// bmcCreds is what the server needs to talk to a machine's BMC.
@@ -54,16 +48,3 @@ func (a *App) credsFor(m *Machine) (*bmcCreds, error) {
}
return &bmcCreds{Address: b.Address, Port: b.Port, User: b.User, Password: pw}, nil
}
// ipmi runs ipmitool server-side; the password never leaves this process.
func (a *App) ipmi(ctx context.Context, c *bmcCreds, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
full := append([]string{"-I", "lanplus", "-H", c.Address, "-p", strconv.Itoa(c.Port), "-U", c.User, "-E"}, args...)
cmd := exec.CommandContext(ctx, a.cfg.IPMITool, full...)
cmd.Env = []string{"IPMI_PASSWORD=" + c.Password, "PATH=/usr/sbin:/usr/bin:/sbin:/bin"}
var out bytes.Buffer
cmd.Stdout, cmd.Stderr = &out, &out
err := cmd.Run()
return strings.TrimSpace(out.String()), err
}
+16 -2
View File
@@ -26,7 +26,8 @@ func usage() {
_, _ = os.Stderr.WriteString(`usage:
rollcall in a terminal: the full-screen console
rollcall tui [-dump WxH -page P -machine ID -plain]
rollcall serve [-config PATH]
rollcall serve [-config PATH] region: API, database, web UI, embedded rack controller
rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines
rollcall version
rollcall keygen
rollcall list
@@ -52,6 +53,14 @@ func usage() {
rollcall location import FILE.csv [-dry] columns: id|serial|mac|node_name, building, room, rack, unit, asset_tag
rollcall location racks | rack NAME -building B -room R [-height 42]
rollcall list [-tag T] [-building B] [-room R] [-rack K]
rollcall fabric list | add NAME [-kind ethernet|infiniband] | rm NAME
rollcall vlan list | add FABRIC VID [-dhcp] [-rack R] [-secondary R] [-mtu N] [-name N] | rm FABRIC VID
rollcall subnet list | show CIDR | add CIDR -fabric F [-vid N] [-purpose P] [-gateway G] [-dns a,b] [-domain D] | rm CIDR [-force]
rollcall range add CIDR dynamic|reserved|static START END | rm ID
rollcall ip list [-subnet CIDR] | reserve IP [-name N] | release IP
rollcall layout list | show NAME | add NAME FILE.json | rm NAME
rollcall net show ID | set ID [-layout L] [-rack R] [-static IFACE=IP] [-unstatic IP]
rollcall controllers | controller rm NAME | dhcp CONTROLLER
(deploy, power and location take tag:NAME instead of ID to act on every machine with that tag)
env: ROLLCALL_URL (default from config listen), ROLLCALL_TOKEN, ROLLCALL_CONFIG;
away from the server the console saves URL and token in ~/.config/rollcall/client.json
@@ -76,6 +85,10 @@ func main() {
}
cmd, args := os.Args[1], os.Args[2:]
switch cmd {
case "rack":
if err := runRack(args); err != nil {
log.Fatal(err)
}
case "tui", "console":
if err := tuiDump(args); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)
@@ -94,7 +107,8 @@ func main() {
fmt.Println(randomToken(24))
case "list", "show", "events", "power", "bootdev", "recommission", "delete",
"images", "profile", "assign", "unassign", "overlay", "render",
"deploy", "release", "jobs", "job", "tag", "tags", "location":
"deploy", "release", "jobs", "job", "tag", "tags", "location",
"fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp":
if err := runClient(cmd, args); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)
os.Exit(1)
+848
View File
@@ -0,0 +1,848 @@
package main
// HTTP API and CLI for the network model and rack controllers.
import (
"encoding/json"
"errors"
"fmt"
"net"
"net/http"
"os"
"strconv"
"strings"
)
// ---------------------------------------------------------------- HTTP
func (a *App) netChanged(r *http.Request) {
a.reloadNet(r.Context())
a.hub.bump()
}
func (a *App) handleFabrics(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
fs, err := a.store.Fabrics(r.Context())
if err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, fs)
return
}
var f Fabric
if err := json.NewDecoder(r.Body).Decode(&f); err != nil || !tagNameRe.MatchString(f.Name) {
fail(w, 400, errors.New("name is required: lowercase letters, digits, . _ -"))
return
}
if f.Kind != "" && f.Kind != "ethernet" && f.Kind != "infiniband" {
fail(w, 400, errors.New("kind is ethernet or infiniband"))
return
}
if err := a.store.PutFabric(r.Context(), f); err != nil {
fail(w, 500, err)
return
}
a.netChanged(r)
writeJSON(w, 200, f)
}
func (a *App) handleFabricDelete(w http.ResponseWriter, r *http.Request) {
if err := a.store.DeleteFabric(r.Context(), r.PathValue("name")); err != nil {
fail(w, 500, err)
return
}
a.netChanged(r)
writeJSON(w, 200, obj{"deleted": r.PathValue("name")})
}
func (a *App) handleVLANs(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
vs, err := a.store.VLANs(r.Context())
if err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, vs)
return
}
var v VLAN
if err := json.NewDecoder(r.Body).Decode(&v); err != nil || v.Fabric == "" || v.VID < 0 || v.VID > 4094 && v.VID < 0x8000 {
fail(w, 400, errors.New("fabric and vid (0 untagged, 1-4094, or an IB pkey) are required"))
return
}
if v.DHCP && v.PrimaryRack == "" {
v.PrimaryRack = a.cfg.RackName
}
racks := a.racksSnapshot()
for _, rk := range []string{v.PrimaryRack, v.SecondaryRack} {
if _, ok := racks[rk]; rk != "" && !ok {
fail(w, 400, fmt.Errorf("rack controller %s is not registered", rk))
return
}
}
id, err := a.store.PutVLAN(r.Context(), v)
if err != nil {
fail(w, 500, err)
return
}
v.ID = id
a.netChanged(r)
writeJSON(w, 200, v)
}
func (a *App) handleVLANDelete(w http.ResponseWriter, r *http.Request) {
vid, _ := strconv.Atoi(r.PathValue("vid"))
if err := a.store.DeleteVLAN(r.Context(), r.PathValue("fabric"), vid); err != nil {
fail(w, 500, err)
return
}
a.netChanged(r)
writeJSON(w, 200, obj{"deleted": fmt.Sprintf("%s/%d", r.PathValue("fabric"), vid)})
}
func (a *App) handleSubnets(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
ss, err := a.store.Subnets(r.Context())
if err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, ss)
return
}
var sn Subnet
if err := json.NewDecoder(r.Body).Decode(&sn); err != nil {
fail(w, 400, err)
return
}
ip, n, err := net.ParseCIDR(sn.CIDR)
if err != nil || ip.To4() == nil {
fail(w, 400, errors.New("cidr must be an IPv4 network, e.g. 10.0.0.0/24"))
return
}
sn.CIDR = n.String()
if sn.Gateway != "" && !sn.contains(sn.Gateway) {
fail(w, 400, fmt.Errorf("gateway %s is outside %s", sn.Gateway, sn.CIDR))
return
}
switch sn.Purpose {
case "", "provisioning", "mgmt", "bmc", "storage", "ib", "other":
default:
fail(w, 400, errors.New("purpose is provisioning, mgmt, bmc, storage, ib or other"))
return
}
if sn.Fabric != "" && sn.VLANID == 0 {
vs, _ := a.store.VLANs(r.Context())
for _, v := range vs {
if v.Fabric == sn.Fabric && v.VID == sn.VID {
sn.VLANID = v.ID
}
}
if sn.VLANID == 0 { // a subnet on a VLAN not declared yet: create it
fs, _ := a.store.Fabrics(r.Context())
known := false
for _, f := range fs {
known = known || f.Name == sn.Fabric
}
if !known {
fail(w, 400, fmt.Errorf("fabric %s does not exist", sn.Fabric))
return
}
if sn.VLANID, err = a.store.PutVLAN(r.Context(), VLAN{Fabric: sn.Fabric, VID: sn.VID}); err != nil {
fail(w, 500, err)
return
}
}
}
for _, other := range a.subnetsSnapshot() {
if other.CIDR == sn.CIDR {
continue
}
if _, on, err := net.ParseCIDR(other.CIDR); err == nil && (on.Contains(n.IP) || n.Contains(on.IP)) {
fail(w, 409, fmt.Errorf("%s overlaps %s", sn.CIDR, other.CIDR))
return
}
}
if err := a.store.PutSubnet(r.Context(), sn); err != nil {
fail(w, 500, err)
return
}
a.adoptIPs(r.Context())
a.netChanged(r)
writeJSON(w, 200, sn)
}
func (a *App) handleSubnetDelete(w http.ResponseWriter, r *http.Request) {
cidr := r.PathValue("cidr")
if ips, _ := a.store.IPs(r.Context(), cidr, ""); len(ips) > 0 && r.URL.Query().Get("force") != "1" {
fail(w, 409, fmt.Errorf("%s still has %d address(es) handed out (force=1 drops them)", cidr, len(ips)))
return
}
if err := a.store.DeleteSubnet(r.Context(), cidr); err != nil {
fail(w, 500, err)
return
}
a.netChanged(r)
writeJSON(w, 200, obj{"deleted": cidr})
}
func (a *App) handleRanges(w http.ResponseWriter, r *http.Request) {
var rg IPRange
if err := json.NewDecoder(r.Body).Decode(&rg); err != nil {
fail(w, 400, err)
return
}
if rg.Kind != "dynamic" && rg.Kind != "reserved" && rg.Kind != "static" {
fail(w, 400, errors.New("kind is dynamic, reserved or static"))
return
}
sn := subnetByRef(a.subnetsSnapshot(), rg.Subnet)
if sn == nil {
fail(w, 404, fmt.Errorf("subnet %s: %w", rg.Subnet, errSubnetUnknown))
return
}
lo, ok1 := ip4(rg.Start)
hi, ok2 := ip4(rg.End)
if !ok1 || !ok2 || hi < lo || !sn.contains(rg.Start) || !sn.contains(rg.End) {
fail(w, 400, fmt.Errorf("range %s-%s is not inside %s", rg.Start, rg.End, sn.CIDR))
return
}
for _, o := range sn.Ranges {
ol, _ := ip4(o.Start)
oh, _ := ip4(o.End)
if lo <= oh && ol <= hi {
fail(w, 409, fmt.Errorf("overlaps the %s range %s-%s", o.Kind, o.Start, o.End))
return
}
}
rg.Subnet = sn.CIDR
id, err := a.store.AddRange(r.Context(), rg)
if err != nil {
fail(w, 500, err)
return
}
rg.ID = id
a.netChanged(r)
writeJSON(w, 200, rg)
}
func (a *App) handleRangeDelete(w http.ResponseWriter, r *http.Request) {
id, _ := strconv.ParseInt(r.PathValue("id"), 10, 64)
if err := a.store.DeleteRange(r.Context(), id); err != nil {
fail(w, 500, err)
return
}
a.netChanged(r)
writeJSON(w, 200, obj{"deleted": id})
}
func (a *App) handleIPs(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodGet:
ips, err := a.store.IPs(r.Context(), r.URL.Query().Get("subnet"), r.URL.Query().Get("machine"))
if err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, ips)
case http.MethodPost: // reserve one address (a switch, a PDU, a VIP)
var x IPAddr
if err := json.NewDecoder(r.Body).Decode(&x); err != nil {
fail(w, 400, err)
return
}
sn := subnetFor(a.subnetsSnapshot(), x.IP)
if sn == nil {
fail(w, 404, fmt.Errorf("%s is in no subnet", x.IP))
return
}
ok, err := a.store.ClaimIP(r.Context(), IPAddr{IP: x.IP, Subnet: sn.CIDR, Kind: "reserved", Hostname: x.Hostname})
if err != nil {
fail(w, 500, err)
return
}
if !ok {
fail(w, 409, fmt.Errorf("%s is taken", x.IP))
return
}
a.netChanged(r)
writeJSON(w, 200, obj{"reserved": x.IP, "subnet": sn.CIDR})
}
}
func (a *App) handleIPDelete(w http.ResponseWriter, r *http.Request) {
if err := a.store.ReleaseIP(r.Context(), r.PathValue("ip")); err != nil {
fail(w, 500, err)
return
}
a.netChanged(r)
writeJSON(w, 200, obj{"released": r.PathValue("ip")})
}
func validLayout(l NetLayout) error {
if !tagNameRe.MatchString(l.Name) {
return errors.New("name is required: lowercase letters, digits, . _ -")
}
if len(l.Interfaces) == 0 {
return errors.New("a layout needs at least one interface")
}
names := map[string]bool{}
for _, i := range l.Interfaces {
switch i.Type {
case "", "physical":
case "bond":
if len(i.Members) == 0 {
return fmt.Errorf("bond %s has no members", i.Name)
}
case "vlan":
if !names[i.Parent] || i.VID < 1 || i.VID > 4094 {
return fmt.Errorf("vlan %q needs a parent defined above it and a vid 1-4094", i.Name)
}
default:
return fmt.Errorf("interface type %q: physical, bond or vlan", i.Type)
}
for _, lk := range i.Links {
switch lk.Mode {
case "", "auto", "static", "dhcp", "link_up":
default:
return fmt.Errorf("link mode %q: auto, static, dhcp or link_up", lk.Mode)
}
}
n := i.Name
if n == "" && i.Type == "vlan" {
n = fmt.Sprintf("%s.%d", i.Parent, i.VID)
}
names[n] = true
}
return nil
}
func (a *App) handleLayouts(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodGet {
ls, err := a.store.Layouts(r.Context())
if err != nil {
fail(w, 500, err)
return
}
ms, _ := a.store.List(r.Context())
for i := range ls {
for _, m := range ms {
if l, _ := a.layoutFor(r.Context(), m); l != nil && strings.HasPrefix(l.Name, ls[i].Name) {
ls[i].Machines++
}
}
}
writeJSON(w, 200, ls)
return
}
var l NetLayout
if err := json.NewDecoder(r.Body).Decode(&l); err != nil {
fail(w, 400, err)
return
}
if err := validLayout(l); err != nil {
fail(w, 400, err)
return
}
if err := a.store.PutLayout(r.Context(), l); err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, l)
}
func (a *App) handleLayoutDelete(w http.ResponseWriter, r *http.Request) {
if err := a.store.DeleteLayout(r.Context(), r.PathValue("name")); err != nil {
fail(w, 409, err)
return
}
writeJSON(w, 200, obj{"deleted": r.PathValue("name")})
}
// handleMachineNetwork: GET shows the plan and the network-config it renders
// (without handing out addresses); POST sets the layout and pinned addresses.
func (a *App) handleMachineNetwork(w http.ResponseWriter, r *http.Request) {
m, err := a.store.Get(r.Context(), r.PathValue("id"))
if err != nil {
fail(w, 500, err)
return
}
if r.Method == http.MethodPost {
var req struct {
Layout *string `json:"layout"`
Rack *string `json:"rack_controller"`
Static []struct{ Iface, IP string }
Unstatic []string `json:"unstatic"`
}
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
fail(w, 400, err)
return
}
if req.Layout != nil {
if *req.Layout != "" {
ls, _ := a.store.Layouts(r.Context())
found := false
for _, l := range ls {
found = found || l.Name == *req.Layout
}
if !found {
fail(w, 404, fmt.Errorf("layout %s does not exist", *req.Layout))
return
}
}
if err := a.store.SetMachineLayout(r.Context(), m.ID, *req.Layout); err != nil {
fail(w, 500, err)
return
}
}
if req.Rack != nil {
if _, ok := a.racksSnapshot()[*req.Rack]; *req.Rack != "" && !ok {
fail(w, 404, fmt.Errorf("rack controller %s is not registered", *req.Rack))
return
}
if err := a.store.SetRackController(r.Context(), m.ID, *req.Rack); err != nil {
fail(w, 500, err)
return
}
m, _ = a.store.Get(r.Context(), m.ID)
_ = a.writePXEAny(r.Context(), m) // its PXE file moves to the new rack
}
for _, ip := range req.Unstatic {
have, _ := a.store.IPs(r.Context(), "", m.ID)
for _, x := range have {
if x.IP == ip && x.Kind == "static" {
_ = a.store.ReleaseIP(r.Context(), ip)
}
}
}
for _, st := range req.Static {
sn := subnetFor(a.subnetsSnapshot(), st.IP)
if sn == nil {
fail(w, 400, fmt.Errorf("%s is in no subnet", st.IP))
return
}
have, _ := a.store.IPs(r.Context(), sn.CIDR, m.ID)
for _, x := range have { // one pinned address per interface and subnet
if x.Iface == st.Iface && x.IP != st.IP {
_ = a.store.ReleaseIP(r.Context(), x.IP)
}
}
if ok, err := a.store.ClaimIP(r.Context(), IPAddr{IP: st.IP, Subnet: sn.CIDR, MachineID: m.ID, Iface: st.Iface, Kind: "static", Hostname: deref(m.NodeName)}); err != nil {
fail(w, 500, err)
return
} else if !ok {
cur, _ := a.store.IPs(r.Context(), sn.CIDR, m.ID)
mine := false
for _, x := range cur {
mine = mine || x.IP == st.IP
}
if !mine {
fail(w, 409, fmt.Errorf("%s is taken", st.IP))
return
}
_, _ = a.store.pool.Exec(r.Context(), `UPDATE ip_addresses SET kind='static', iface=$2 WHERE ip=$1`, st.IP, st.Iface)
}
}
_ = a.store.AddEvent(r.Context(), m.ID, "network", fmt.Sprintf("layout=%v static=%v unstatic=%v", deref(req.Layout), req.Static, req.Unstatic))
a.netChanged(r)
m, _ = a.store.Get(r.Context(), m.ID)
}
plan, err := a.netPlan(r.Context(), m, false)
if err != nil {
fail(w, 409, err)
return
}
ips, _ := a.store.IPs(r.Context(), "", m.ID)
writeJSON(w, 200, obj{"id": m.ID, "layout": plan.Layout, "plan": plan, "network_config": networkConfig(plan), "addresses": ips,
"rack_controller": a.rackOf(m), "modeled": len(a.subnetsSnapshot()) > 0})
}
// ---------------------------------------------------------------- CLI
func (c *client) netCmd(cmd string, args []string) error {
need := func(n int) {
if len(args) < n {
usage()
}
}
sub := ""
if len(args) > 0 {
sub = args[0]
}
switch cmd {
case "fabric":
switch sub {
case "", "list":
var fs []Fabric
if err := c.do("GET", "/api/v1/fabrics", nil, &fs); err != nil {
return err
}
fmt.Printf("%-16s %-11s %s\n", "FABRIC", "KIND", "COMMENT")
for _, f := range fs {
fmt.Printf("%-16s %-11s %s\n", f.Name, f.Kind, f.Comment)
}
case "add":
need(2)
fs := newFlags("fabric add")
kind := fs.String("kind", "ethernet", "ethernet | infiniband")
comment := fs.String("comment", "", "comment")
_ = fs.Parse(args[2:])
return c.do("POST", "/api/v1/fabrics", Fabric{Name: args[1], Kind: *kind, Comment: *comment}, nil)
case "rm":
need(2)
return c.do("DELETE", "/api/v1/fabrics/"+args[1], nil, nil)
default:
usage()
}
case "vlan":
switch sub {
case "", "list":
var vs []VLAN
if err := c.do("GET", "/api/v1/vlans", nil, &vs); err != nil {
return err
}
fmt.Printf("%-14s %5s %-14s %5s %-5s %-14s %s\n", "FABRIC", "VID", "NAME", "MTU", "DHCP", "RACK", "SECONDARY")
for _, v := range vs {
fmt.Printf("%-14s %5d %-14s %5d %-5v %-14s %s\n", v.Fabric, v.VID, v.Name, v.MTU, v.DHCP, v.PrimaryRack, v.SecondaryRack)
}
case "add", "set":
need(3)
fs := newFlags("vlan add")
name := fs.String("name", "", "name")
mtu := fs.Int("mtu", 1500, "MTU")
dhcp := fs.Bool("dhcp", false, "serve DHCP/PXE on this VLAN")
rack := fs.String("rack", "", "rack controller serving it (default: the region's embedded one)")
sec := fs.String("secondary", "", "second rack controller (the dynamic range is split)")
_ = fs.Parse(args[3:])
vid, _ := strconv.Atoi(args[2])
var v VLAN
if err := c.do("POST", "/api/v1/vlans", VLAN{Fabric: args[1], VID: vid, Name: *name, MTU: *mtu, DHCP: *dhcp, PrimaryRack: *rack, SecondaryRack: *sec}, &v); err != nil {
return err
}
fmt.Printf("vlan %s/%d: dhcp=%v rack=%s\n", v.Fabric, v.VID, v.DHCP, v.PrimaryRack)
case "rm":
need(3)
return c.do("DELETE", "/api/v1/vlans/"+args[1]+"/"+args[2], nil, nil)
default:
usage()
}
case "subnet":
switch sub {
case "", "list":
var ss []Subnet
if err := c.do("GET", "/api/v1/subnets", nil, &ss); err != nil {
return err
}
fmt.Printf("%-18s %-12s %-14s %-12s %-15s %-12s %9s %s\n", "SUBNET", "PURPOSE", "FABRIC/VLAN", "DHCP RACK", "GATEWAY", "DOMAIN", "USED", "RANGES")
for _, s := range ss {
dhcp := "-"
if s.DHCP {
dhcp = s.PrimaryRack
if s.SecondaryRack != "" {
dhcp += "+" + s.SecondaryRack
}
}
var rs []string
for _, r := range s.Ranges {
rs = append(rs, fmt.Sprintf("%s:%s-%s", r.Kind, r.Start, lastOctets(r.End)))
}
fmt.Printf("%-18s %-12s %-14s %-12s %-15s %-12s %4d/%-4d %s\n", s.CIDR, s.Purpose, fmt.Sprintf("%s/%d", orDash(s.Fabric), s.VID), dhcp, orDash(s.Gateway), orDash(s.Domain), s.Used, s.Size, strings.Join(rs, " "))
}
case "show":
need(2)
var ss []Subnet
if err := c.do("GET", "/api/v1/subnets", nil, &ss); err != nil {
return err
}
for _, s := range ss {
if s.CIDR == args[1] || s.Name == args[1] {
fmt.Printf("%s %s fabric %s vlan %d mtu %d dhcp=%v rack=%s %s\n gateway %s dns %v domain %s\n", s.CIDR, s.Purpose, s.Fabric, s.VID, s.MTU, s.DHCP, s.PrimaryRack, s.SecondaryRack, orDash(s.Gateway), s.DNS, orDash(s.Domain))
for _, r := range s.Ranges {
fmt.Printf(" range %-4d %-8s %s - %s %s\n", r.ID, r.Kind, r.Start, r.End, r.Comment)
}
var ips []IPAddr
if err := c.do("GET", "/api/v1/ips?subnet="+s.CIDR, nil, &ips); err != nil {
return err
}
fmt.Printf(" %d of %d addresses in use\n", len(ips), s.Size)
for _, x := range ips {
fmt.Printf(" %-15s %-8s %-16s %-10s %s\n", x.IP, x.Kind, orDash(x.MachineID), x.Iface, x.Hostname)
}
return nil
}
}
return errNotFound
case "add", "set":
need(2)
fs := newFlags("subnet add")
fabric := fs.String("fabric", "", "fabric (the VLAN is created when missing)")
vid := fs.Int("vid", 0, "VLAN id (0 = untagged)")
purpose := fs.String("purpose", "provisioning", "provisioning | mgmt | bmc | storage | ib | other")
gw := fs.String("gateway", "", "default gateway")
dns := fs.String("dns", "", "comma-separated DNS servers")
domain := fs.String("domain", "", "DNS domain")
name := fs.String("name", "", "name")
comment := fs.String("comment", "", "comment")
_ = fs.Parse(args[2:])
sn := Subnet{CIDR: args[1], Fabric: *fabric, VID: *vid, Purpose: *purpose, Gateway: *gw, Domain: *domain, Name: *name, Comment: *comment, DNS: []string{}}
if *dns != "" {
sn.DNS = strings.Split(*dns, ",")
}
var out Subnet
if err := c.do("POST", "/api/v1/subnets", sn, &out); err != nil {
return err
}
fmt.Printf("subnet %s saved\n", out.CIDR)
case "rm":
need(2)
q := ""
if len(args) > 2 && args[2] == "-force" {
q = "?force=1"
}
return c.do("DELETE", "/api/v1/subnets/"+args[1]+q, nil, nil)
default:
usage()
}
case "range":
switch sub {
case "add":
need(5)
fs := newFlags("range add")
comment := fs.String("comment", "", "comment")
_ = fs.Parse(args[5:])
var rg IPRange
if err := c.do("POST", "/api/v1/ranges", IPRange{Subnet: args[1], Kind: args[2], Start: args[3], End: args[4], Comment: *comment}, &rg); err != nil {
return err
}
fmt.Printf("range %d: %s %s-%s on %s\n", rg.ID, rg.Kind, rg.Start, rg.End, rg.Subnet)
case "rm":
need(2)
return c.do("DELETE", "/api/v1/ranges/"+args[1], nil, nil)
default:
usage()
}
case "ip":
switch sub {
case "", "list":
fs := newFlags("ip list")
subnet := fs.String("subnet", "", "only this subnet")
_ = fs.Parse(args[min(1, len(args)):])
var ips []IPAddr
if err := c.do("GET", "/api/v1/ips?subnet="+*subnet, nil, &ips); err != nil {
return err
}
fmt.Printf("%-15s %-18s %-8s %-16s %-10s %s\n", "IP", "SUBNET", "KIND", "MACHINE", "IFACE", "HOSTNAME")
for _, x := range ips {
fmt.Printf("%-15s %-18s %-8s %-16s %-10s %s\n", x.IP, x.Subnet, x.Kind, orDash(x.MachineID), x.Iface, x.Hostname)
}
case "reserve":
need(2)
fs := newFlags("ip reserve")
host := fs.String("name", "", "what it is (a switch, a VIP)")
_ = fs.Parse(args[2:])
return c.do("POST", "/api/v1/ips", IPAddr{IP: args[1], Hostname: *host}, nil)
case "release":
need(2)
return c.do("DELETE", "/api/v1/ips/"+args[1], nil, nil)
default:
usage()
}
case "layout":
switch sub {
case "", "list":
var ls []NetLayout
if err := c.do("GET", "/api/v1/layouts", nil, &ls); err != nil {
return err
}
fmt.Printf("%-18s %8s %-50s %s\n", "LAYOUT", "MACHINES", "INTERFACES", "COMMENT")
for _, l := range ls {
fmt.Printf("%-18s %8d %-50s %s\n", l.Name, l.Machines, trunc(layoutSummary(l), 50), l.Comment)
}
case "show":
need(2)
var ls []NetLayout
if err := c.do("GET", "/api/v1/layouts", nil, &ls); err != nil {
return err
}
for _, l := range ls {
if l.Name == args[1] {
b, _ := json.MarshalIndent(l, "", " ")
fmt.Println(string(b))
return nil
}
}
return errNotFound
case "add":
need(3)
fs := newFlags("layout add")
comment := fs.String("comment", "", "comment")
_ = fs.Parse(args[3:])
b, err := readFileArg(args[2])
if err != nil {
return err
}
var l NetLayout
if err := json.Unmarshal(b, &l); err != nil {
return fmt.Errorf("%s: %w", args[2], err)
}
l.Name = args[1]
if *comment != "" {
l.Comment = *comment
}
return c.do("POST", "/api/v1/layouts", l, nil)
case "rm":
need(2)
return c.do("DELETE", "/api/v1/layouts/"+args[1], nil, nil)
default:
usage()
}
case "net":
need(2)
switch sub {
case "show":
var d obj
if err := c.do("GET", "/api/v1/machines/"+args[1]+"/network", nil, &d); err != nil {
return err
}
printNet(d)
case "set":
fs := newFlags("net set")
layout := fs.String("layout", "\x00", "layout name (\"\" = from tags / default)")
rack := fs.String("rack", "\x00", "rack controller serving it (a machine moved to another segment)")
var statics, unstatic multiFlag
fs.Var(&statics, "static", "IFACE=IP pins an address (repeatable)")
fs.Var(&unstatic, "unstatic", "IP drops a pinned address (repeatable)")
_ = fs.Parse(args[2:])
req := obj{"unstatic": []string(unstatic)}
if *layout != "\x00" {
req["layout"] = *layout
}
if *rack != "\x00" {
req["rack_controller"] = *rack
}
var st []obj
for _, s := range statics {
i, ip, ok := strings.Cut(s, "=")
if !ok {
return fmt.Errorf("-static wants IFACE=IP")
}
st = append(st, obj{"iface": i, "ip": ip})
}
req["static"] = st
var d obj
if err := c.do("POST", "/api/v1/machines/"+args[1]+"/network", req, &d); err != nil {
return err
}
printNet(d)
default:
usage()
}
case "controllers", "controller":
switch {
case cmd == "controller" && sub == "rm":
need(2)
return c.do("DELETE", "/api/v1/controllers/"+args[1], nil, nil)
default:
var rs []RackController
if err := c.do("GET", "/api/v1/controllers", nil, &rs); err != nil {
return err
}
fmt.Printf("%-14s %-9s %-7s %-24s %8s %-20s %s\n", "CONTROLLER", "KIND", "ONLINE", "URL", "MACHINES", "LAST SEEN", "INTERFACES")
for _, r := range rs {
var ifs []string
for _, i := range r.Interfaces {
ifs = append(ifs, i.Name+"="+strings.Join(i.Addrs, ","))
}
fmt.Printf("%-14s %-9s %-7v %-24s %8d %-20s %s\n", r.Name, r.Kind, r.Online, r.URL, r.Machines, timeStr(r.LastSeen), strings.Join(ifs, " "))
}
}
case "dhcp":
need(1)
data, err := c.raw("GET", "/api/v1/controllers/"+args[0]+"/dhcp")
if err != nil {
return err
}
os.Stdout.Write(data)
}
return nil
}
type multiFlag []string
func (m *multiFlag) String() string { return strings.Join(*m, ",") }
func (m *multiFlag) Set(v string) error { *m = append(*m, v); return nil }
func lastOctets(ip string) string {
if i := strings.LastIndex(ip, "."); i >= 0 {
return ip[i+1:]
}
return ip
}
func layoutSummary(l NetLayout) string {
var p []string
for _, i := range l.Interfaces {
var links []string
for _, lk := range i.Links {
links = append(links, lk.Subnet+":"+orDash(lk.Mode))
}
switch i.Type {
case "bond":
p = append(p, fmt.Sprintf("%s(%s)[%s]", orDash(i.Name), strings.Join(i.Members, "+"), strings.Join(links, ",")))
case "vlan":
p = append(p, fmt.Sprintf("%s.%d[%s]", i.Parent, i.VID, strings.Join(links, ",")))
default:
p = append(p, fmt.Sprintf("%s[%s]", orDash(i.Select), strings.Join(links, ",")))
}
}
return strings.Join(p, " ")
}
func printNet(d obj) {
fmt.Printf("%s layout %s rack controller %s\n", str(d["id"]), str(d["layout"]), str(d["rack_controller"]))
if d["modeled"] == false {
fmt.Println(" no subnets are defined: deployed machines use DHCP (rollcall subnet add …)")
}
plan, _ := d["plan"].(obj)
for _, iv := range toList(plan["interfaces"]) {
i, _ := iv.(obj)
desc := str(i["type"])
switch desc {
case "physical":
desc = str(i["mac"])
case "bond":
var ms []string
for _, x := range toList(i["members"]) {
ms = append(ms, str(x))
}
desc = "bond " + str(i["bond_mode"]) + " of " + strings.Join(ms, ",")
case "vlan":
desc = fmt.Sprintf("vlan %s on %s", str(i["vid"]), str(i["parent"]))
}
mtu := ""
if num(i["mtu"]) > 0 {
mtu = fmt.Sprintf(" mtu %d", int(num(i["mtu"])))
}
fmt.Printf(" %-12s %s%s\n", str(i["name"]), desc, mtu)
for _, lv := range toList(i["links"]) {
l, _ := lv.(obj)
ip := str(l["ip"])
if ip == "" && str(l["mode"]) == "auto" {
ip = "(allocated at deploy)"
} else if ip != "" {
ip = fmt.Sprintf("%s/%d", ip, int(num(l["prefix"])))
}
gw := ""
if l["default_gateway"] == true {
gw = " default via " + str(l["gateway"])
}
fmt.Printf(" %-8s %-18s %s%s\n", str(l["mode"]), orDash(str(l["subnet"])), ip, gw)
}
}
for _, w := range toList(plan["warnings"]) {
fmt.Printf(" ! %s\n", str(w))
}
b, _ := json.MarshalIndent(d["network_config"], " ", " ")
fmt.Printf(" network-config:\n %s\n", b)
}
File diff suppressed because it is too large. Load diff
+165
View File
@@ -0,0 +1,165 @@
package main
import (
"encoding/json"
"strings"
"testing"
)
func labSubnet() Subnet {
return Subnet{CIDR: "10.0.0.0/24", Gateway: "10.0.0.1", DNS: []string{"10.0.0.1"}, Domain: "lab.local", Fabric: "f0", MTU: 1500,
DHCP: true, PrimaryRack: "r1", Purpose: "provisioning",
Ranges: []IPRange{{Kind: "reserved", Start: "10.0.0.2", End: "10.0.0.20"}, {Kind: "dynamic", Start: "10.0.0.100", End: "10.0.0.200"}}}
}
func TestFreeCandidates(t *testing.T) {
sn := labSubnet()
got := freeCandidates(sn, map[string]bool{"10.0.0.21": true}, []string{"10.0.0.22/24"}, 3)
if strings.Join(got, " ") != "10.0.0.23 10.0.0.24 10.0.0.25" {
t.Errorf("candidates: %v", got)
}
for _, ip := range freeCandidates(sn, nil, nil, 0) {
if ip == "10.0.0.1" || ip == "10.0.0.0" || ip == "10.0.0.255" || (ipLess("10.0.0.99", ip) && ipLess(ip, "10.0.0.201")) {
t.Fatalf("handed out %s", ip)
}
}
sn.Ranges = append(sn.Ranges, IPRange{Kind: "static", Start: "10.0.0.50", End: "10.0.0.52"})
if got := freeCandidates(sn, nil, nil, 0); strings.Join(got, " ") != "10.0.0.50 10.0.0.51 10.0.0.52" {
t.Errorf("static range: %v", got)
}
}
func allocFrom(next ...string) func(Subnet, string) (string, error) {
return func(Subnet, string) (string, error) {
ip := next[0]
next = next[1:]
return ip, nil
}
}
func TestDefaultPlanAndNetworkConfig(t *testing.T) {
nics := []invNIC{{Name: "eno1", MAC: "aa:bb:cc:00:00:01", Speed: 1000, Link: true, IPv4: []string{"10.0.0.150/24"}}}
p := resolvePlan(nil, nics, &nics[0], []Subnet{labSubnet()}, nil, allocFrom("10.0.0.21"))
if len(p.Warnings) > 0 || len(p.Ifaces) != 1 {
t.Fatalf("plan: %+v", p)
}
if mgmt, _ := p.primaryIPs(); mgmt != "10.0.0.21" {
t.Errorf("mgmt %q", mgmt)
}
b, _ := json.Marshal(networkConfig(p))
s := string(b)
for _, want := range []string{`"macaddress":"aa:bb:cc:00:00:01"`, `"set-name":"eno1"`, `"addresses":["10.0.0.21/24"]`,
`"routes":[{"to":"0.0.0.0/0","via":"10.0.0.1"}]`, `"search":["lab.local"]`, `"dhcp4":false`} {
if !strings.Contains(s, want) {
t.Errorf("network-config lacks %s:\n%s", want, s)
}
}
// a pinned address wins over allocation
p = resolvePlan(nil, nics, &nics[0], []Subnet{labSubnet()}, []IPAddr{{IP: "10.0.0.60", Subnet: "10.0.0.0/24", Iface: "eno1", Kind: "static"}}, allocFrom("10.0.0.21"))
if p.Ifaces[0].Links[0].IP != "10.0.0.60" || p.Ifaces[0].Links[0].Mode != "static" {
t.Errorf("static pin ignored: %+v", p.Ifaces[0].Links[0])
}
// no subnet model for the boot network: DHCP, as before
p = resolvePlan(nil, nics, &nics[0], nil, nil, nil)
if p.Ifaces[0].Links[0].Mode != "dhcp" {
t.Errorf("without subnets the default is dhcp, got %+v", p.Ifaces[0].Links)
}
}
func TestBondVLANLayout(t *testing.T) {
nics := []invNIC{
{Name: "eno1", MAC: "aa:00:00:00:00:01", Speed: 1000, Link: true, IPv4: []string{"10.0.0.150/24"}},
{Name: "ens1f0", MAC: "aa:00:00:00:00:02", Speed: 100000, Link: true},
{Name: "ens1f1", MAC: "aa:00:00:00:00:03", Speed: 100000, Link: true},
{Name: "ibp1s0", MAC: "80:00:00:00", IB: true, Link: true},
}
storage := Subnet{CIDR: "10.20.0.0/24", Fabric: "f0", VID: 200, MTU: 9000}
ib := Subnet{CIDR: "10.30.0.0/24", Fabric: "ib0"}
l := &NetLayout{Name: "gpu", Interfaces: []LayoutIface{
{Type: "physical", Select: "boot", Links: []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}},
{Type: "bond", Name: "bond0", Members: []string{"fastest:2"}, Links: []LayoutLink{{Mode: "link_up"}}},
{Type: "vlan", Parent: "bond0", VID: 200, Links: []LayoutLink{{Subnet: "10.20.0.0/24", Mode: "auto"}}},
{Type: "physical", Select: "ib:0", Links: []LayoutLink{{Subnet: "10.30.0.0/24", Mode: "auto"}}},
}}
p := resolvePlan(l, nics, &nics[0], []Subnet{labSubnet(), storage, ib}, nil, allocFrom("10.0.0.21", "10.20.0.5", "10.30.0.5"))
if len(p.Warnings) > 0 {
t.Fatalf("warnings: %v", p.Warnings)
}
if got := p.Ifaces[1].Members; strings.Join(got, ",") != "ens1f0,ens1f1" {
t.Errorf("bond members %v (the boot NIC must not be taken)", got)
}
if p.Ifaces[2].Name != "bond0.200" || p.Ifaces[2].MTU != 9000 {
t.Errorf("vlan iface %+v", p.Ifaces[2])
}
if mgmt, ibip := p.primaryIPs(); mgmt != "10.0.0.21" || ibip != "10.30.0.5" {
t.Errorf("primary %s ib %s", mgmt, ibip)
}
b, _ := json.Marshal(networkConfig(p))
s := string(b)
for _, want := range []string{`"bonds":{"bond0":`, `"interfaces":["ens1f0","ens1f1"]`, `"mode":"802.3ad"`, `"vlans":{"bond0.200":`,
`"link":"bond0"`, `"id":200`, `"10.20.0.5/24"`, `"mtu":9000`, `"ibp1s0":{"addresses":["10.30.0.5/24"]`} {
if !strings.Contains(s, want) {
t.Errorf("network-config lacks %s:\n%s", want, s)
}
}
if strings.Count(s, "0.0.0.0/0") != 1 {
t.Errorf("want exactly one default route:\n%s", s)
}
rules := linkRules(p)
for _, want := range []string{`ATTR{address}=="aa:00:00:00:00:01", NAME="eno1"`, `ATTR{address}=="aa:00:00:00:00:02", NAME="ens1f0"`, `ATTR{address}=="aa:00:00:00:00:03", NAME="ens1f1"`} {
if !strings.Contains(rules, want) {
t.Errorf("udev rules lack %s:\n%s", want, rules)
}
}
if strings.Contains(rules, "ibp1s0") {
t.Errorf("InfiniBand ports keep their names:\n%s", rules)
}
}
// NICs of one speed (VMs, a box of identical 25G ports): the bond must not count the boot NIC.
func TestBondSkipsUsedNICs(t *testing.T) {
nics := []invNIC{
{Name: "ens18", MAC: "bc:00:00:00:00:01", Link: true, IPv4: []string{"10.0.0.150/24"}},
{Name: "ens19", MAC: "bc:00:00:00:00:02", Link: true},
{Name: "ens20", MAC: "bc:00:00:00:00:03", Link: true},
}
l := &NetLayout{Interfaces: []LayoutIface{
{Type: "physical", Select: "boot", Links: []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}},
{Type: "bond", Name: "bond0", Members: []string{"fastest:2"}, BondMode: "active-backup", Links: []LayoutLink{{Mode: "link_up"}}},
}}
p := resolvePlan(l, nics, &nics[0], []Subnet{labSubnet()}, nil, allocFrom("10.0.0.21"))
if got := strings.Join(p.Ifaces[1].Members, ","); got != "ens19,ens20" {
t.Errorf("bond members %q, want ens19,ens20", got)
}
if lk := p.Ifaces[1].Links[0]; lk.Subnet != "" || lk.Mode != "link_up" {
t.Errorf("a link_up link without a subnet stays without one: %+v", lk)
}
}
func TestDnsmasqConfig(t *testing.T) {
rc := RackController{Name: "r1", TFTPRoot: "/srv/tftp", Interfaces: []RackIface{{Name: "eth0", Addrs: []string{"192.168.1.5/24"}}, {Name: "eth1", Addrs: []string{"10.0.0.1/24"}}}}
conf, ok := dnsmasqConfig(rc, []Subnet{labSubnet()}, "/var/lib/rollcall/dnsmasq")
if !ok {
t.Fatal("r1 serves 10.0.0.0/24")
}
for _, want := range []string{"interface=eth1\n", "dhcp-range=set:s0,10.0.0.100,10.0.0.200,255.255.255.0,12h", "dhcp-option=tag:s0,option:router,10.0.0.1",
"dhcp-option=tag:s0,option:dns-server,10.0.0.1", "domain=lab.local,10.0.0.0/24", "dhcp-authoritative", "tftp-root=/srv/tftp"} {
if !strings.Contains(conf, want) {
t.Errorf("dnsmasq conf lacks %q:\n%s", want, conf)
}
}
if strings.Contains(conf, "interface=eth0") {
t.Errorf("must not serve the management interface:\n%s", conf)
}
// two racks: each takes half of the pool, nobody is authoritative
sn := labSubnet()
sn.SecondaryRack = "r2"
c1, _ := dnsmasqConfig(rc, []Subnet{sn}, "/d")
c2, _ := dnsmasqConfig(RackController{Name: "r2", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"10.0.0.2/24"}}}}, []Subnet{sn}, "/d")
if !strings.Contains(c1, "10.0.0.100,10.0.0.150,") || !strings.Contains(c2, "10.0.0.151,10.0.0.200,") || strings.Contains(c1, "dhcp-authoritative") {
t.Errorf("split pool:\n%s\n%s", c1, c2)
}
if _, ok := dnsmasqConfig(RackController{Name: "other"}, []Subnet{labSubnet()}, "/d"); ok {
t.Errorf("a rack that serves nothing must not get a config")
}
}
+867
View File
@@ -0,0 +1,867 @@
package main
// Region side of the region/rack split, after MAAS. The region (rollcall
// serve) owns the database and computes everything; rack controllers sit next
// to the machines and do what has to happen on the machines' networks: DHCP
// and TFTP (dnsmasq), the per-MAC PXE files, BMC commands on a BMC network the
// region may not reach, and serving boot and OS images from a local cache.
//
// A rack controller pulls: it long-polls its desired state (PXE files,
// dnsmasq configuration, host files) and a queue of commands (ipmitool), so
// only rack → region connectivity is needed. The region host itself runs an
// embedded rack controller that applies the same state in-process, which is
// what a single-server installation uses.
import (
"bytes"
"context"
"crypto/subtle"
"encoding/json"
"errors"
"fmt"
"log"
"net"
"net/http"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
)
type RackIface struct {
Name string `json:"name"`
MAC string `json:"mac"`
Addrs []string `json:"addrs"`
}
type RackController struct {
Name string `json:"name"`
URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1
Kind string `json:"kind"` // embedded | remote
Version string `json:"version"`
Interfaces []RackIface `json:"interfaces"`
DHCPInterfaces []string `json:"dhcp_interfaces"`
TFTPRoot string `json:"tftp_root"`
LastSeen *time.Time `json:"last_seen"`
Registered time.Time `json:"registered"`
Online bool `json:"online"`
Machines int `json:"machines"`
}
// rackState is everything a rack controller writes to disk.
type rackState struct {
Gen int64 `json:"gen"`
DefaultPXE string `json:"default_pxe"`
PXE map[string]string `json:"pxe"` // pxelinux.cfg file name (01-aa-bb-…) → content
DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here
HostsDHCP string `json:"hosts_dhcp"`
Hosts string `json:"hosts"`
BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP
}
type rackJob struct {
ID string `json:"id"`
Kind string `json:"kind"` // ipmi
Address string `json:"address"`
Port int `json:"port"`
User string `json:"user"`
Password string `json:"password"`
Args []string `json:"args"`
done chan rackResult
}
type rackResult struct {
ID string `json:"id"`
Output string `json:"output"`
Error string `json:"error"`
}
// ---------------------------------------------------------------- store
func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
if err != nil {
return nil, err
}
defer rows.Close()
out := []RackController{}
for rows.Next() {
var r RackController
var ifs, dif []byte
if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
return nil, err
}
_ = json.Unmarshal(ifs, &r.Interfaces)
_ = json.Unmarshal(dif, &r.DHCPInterfaces)
out = append(out, r)
}
return out, rows.Err()
}
func (s *Store) PutRackController(ctx context.Context, r RackController) error {
if r.Interfaces == nil {
r.Interfaces = []RackIface{}
}
if r.DHCPInterfaces == nil {
r.DHCPInterfaces = []string{}
}
ifs, _ := json.Marshal(r.Interfaces)
dif, _ := json.Marshal(r.DHCPInterfaces)
_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen) VALUES ($1,$2,$3,$4,$5,$6,$7,now())
ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now()`,
r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot)
return err
}
func (s *Store) TouchRackController(ctx context.Context, name string) error {
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET last_seen=now() WHERE name=$1`, name)
return err
}
func (s *Store) DeleteRackController(ctx context.Context, name string) error {
_, err := s.pool.Exec(ctx, `DELETE FROM rack_controllers WHERE name=$1`, name)
return err
}
func (s *Store) SetRackController(ctx context.Context, id, rack string) error {
_, err := s.pool.Exec(ctx, `UPDATE machines SET rack_controller=nullif($2,'') WHERE id=$1`, id, rack)
return err
}
func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error {
_, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4)
ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now()`, mac, machineID, rack, content)
return err
}
func (s *Store) DeletePXE(ctx context.Context, mac string) error {
_, err := s.pool.Exec(ctx, `DELETE FROM pxe_files WHERE mac=$1`, mac)
return err
}
func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, error) {
rows, err := s.pool.Query(ctx, `SELECT mac, content FROM pxe_files WHERE rack=$1`, rack)
if err != nil {
return nil, err
}
defer rows.Close()
out := map[string]string{}
for rows.Next() {
var mac, content string
if err := rows.Scan(&mac, &content); err != nil {
return nil, err
}
out[pxeFileName(mac)] = content
}
return out, rows.Err()
}
func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") }
// ---------------------------------------------------------------- hub: state generations and the command queue
type rackHub struct {
mu sync.Mutex
gen int64
changed chan struct{}
work map[string][]*rackJob
wake map[string]chan struct{}
flight map[string]*rackJob
kick chan struct{} // the embedded rack applies on this
}
func newRackHub() *rackHub {
return &rackHub{gen: time.Now().UnixNano(), changed: make(chan struct{}), work: map[string][]*rackJob{},
wake: map[string]chan struct{}{}, flight: map[string]*rackJob{}, kick: make(chan struct{}, 1)}
}
func (h *rackHub) current() int64 {
h.mu.Lock()
defer h.mu.Unlock()
return h.gen
}
// bump says the desired state of some rack changed.
func (h *rackHub) bump() {
h.mu.Lock()
h.gen++
close(h.changed)
h.changed = make(chan struct{})
h.mu.Unlock()
select {
case h.kick <- struct{}{}:
default:
}
}
// waitChange blocks until the generation moves past since, or the wait is over.
func (h *rackHub) waitChange(ctx context.Context, since int64, wait time.Duration) int64 {
t := time.NewTimer(wait)
defer t.Stop()
for {
h.mu.Lock()
g, ch := h.gen, h.changed
h.mu.Unlock()
if g != since {
return g
}
select {
case <-ch:
case <-t.C:
return g
case <-ctx.Done():
return g
}
}
}
func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string, error) {
j.ID = randomToken(8)
j.done = make(chan rackResult, 1)
h.mu.Lock()
h.work[rack] = append(h.work[rack], j)
if w, ok := h.wake[rack]; ok {
close(w)
delete(h.wake, rack)
}
h.mu.Unlock()
t := time.NewTimer(90 * time.Second)
defer t.Stop()
select {
case r := <-j.done:
if r.Error != "" {
return r.Output, errors.New(r.Error)
}
return r.Output, nil
case <-t.C:
case <-ctx.Done():
}
h.mu.Lock()
delete(h.flight, j.ID)
q := h.work[rack][:0]
for _, x := range h.work[rack] {
if x != j {
q = append(q, x)
}
}
h.work[rack] = q
h.mu.Unlock()
return "", fmt.Errorf("rack controller %s did not answer in time", rack)
}
func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*rackJob {
deadline := time.Now().Add(wait)
for {
h.mu.Lock()
if q := h.work[rack]; len(q) > 0 {
h.work[rack] = nil
for _, j := range q {
h.flight[j.ID] = j
}
h.mu.Unlock()
return q
}
w, ok := h.wake[rack]
if !ok {
w = make(chan struct{})
h.wake[rack] = w
}
h.mu.Unlock()
left := time.Until(deadline)
if left <= 0 {
return nil
}
t := time.NewTimer(left)
select {
case <-w:
case <-t.C:
case <-ctx.Done():
t.Stop()
return nil
}
t.Stop()
}
}
func (h *rackHub) finish(r rackResult) bool {
h.mu.Lock()
j, ok := h.flight[r.ID]
delete(h.flight, r.ID)
h.mu.Unlock()
if ok {
j.done <- r
}
return ok
}
// ---------------------------------------------------------------- routing machines to racks
func (a *App) racksSnapshot() map[string]RackController {
a.netMu.RLock()
defer a.netMu.RUnlock()
out := make(map[string]RackController, len(a.racks))
for k, v := range a.racks {
out[k] = v
}
return out
}
func (a *App) subnetsSnapshot() []Subnet {
a.netMu.RLock()
defer a.netMu.RUnlock()
return a.subnets
}
// reloadNet refreshes the cached subnets and rack controllers after a change.
func (a *App) reloadNet(ctx context.Context) {
if sn, err := a.store.Subnets(ctx); err == nil {
a.netMu.Lock()
a.subnets = sn
a.netMu.Unlock()
}
a.reloadRacks(ctx)
}
func (a *App) reloadRacks(ctx context.Context) {
rs, err := a.store.RackControllers(ctx)
if err != nil {
log.Printf("racks: %v", err)
return
}
m := map[string]RackController{}
for _, r := range rs {
m[r.Name] = r
}
a.netMu.Lock()
a.racks = m
a.netMu.Unlock()
}
func rackOnline(r RackController, local string) bool {
return r.Name == local || (r.LastSeen != nil && time.Since(*r.LastSeen) < 90*time.Second)
}
// rackOf names the rack controller serving a machine: the one it last came
// through, else the one serving the VLAN of its management or boot address,
// else the embedded one.
func (a *App) rackOf(m *Machine) string {
racks := a.racksSnapshot()
if m.RackController != "" {
if _, ok := racks[m.RackController]; ok {
return m.RackController
}
}
if subnets := a.subnetsSnapshot(); len(subnets) > 0 {
ips := []string{deref(m.MgmtIP)}
if b := bootNIC(machineNICs(m), m.MACs); b != nil {
ips = append(ips, b.IPv4...)
}
for _, ip := range ips {
if sn := subnetFor(subnets, ip); sn != nil && sn.PrimaryRack != "" {
if _, ok := racks[sn.PrimaryRack]; ok {
return sn.PrimaryRack
}
}
}
}
return a.cfg.RackName
}
func (a *App) rackURL(name string) string {
if name == "" || name == a.cfg.RackName {
return a.cfg.ServerURL
}
if r, ok := a.racksSnapshot()[name]; ok && r.URL != "" {
return strings.TrimRight(r.URL, "/")
}
return a.cfg.ServerURL
}
// urlFor is the base URL a machine uses for images and the node API.
func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) }
// appendFor is commission_append with the server URL of the machine's rack.
func (a *App) appendFor(base string) string {
if base == a.cfg.ServerURL {
return a.cfg.CommissionAppend
}
return strings.ReplaceAll(a.cfg.CommissionAppend, a.cfg.ServerURL, base)
}
// rackAddresses are the rack controllers' own addresses, never handed to machines.
func (a *App) rackAddresses() []string {
var out []string
for _, r := range a.racksSnapshot() {
for _, i := range r.Interfaces {
out = append(out, i.Addrs...)
}
}
return out
}
// ---------------------------------------------------------------- BMC through the machine's rack
var errNoBMC = errors.New("no BMC credentials stored")
func ipmiExec(ctx context.Context, tool string, c *bmcCreds, args ...string) (string, error) {
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
defer cancel()
if tool == "" {
tool = "ipmitool"
}
full := append([]string{"-I", "lanplus", "-H", c.Address, "-p", strconv.Itoa(c.Port), "-U", c.User, "-E"}, args...)
cmd := exec.CommandContext(ctx, tool, full...)
cmd.Env = []string{"IPMI_PASSWORD=" + c.Password, "PATH=/usr/sbin:/usr/bin:/sbin:/bin"}
var out bytes.Buffer
cmd.Stdout, cmd.Stderr = &out, &out
err := cmd.Run()
return strings.TrimSpace(out.String()), err
}
// bmc runs an ipmitool command for a machine, on the rack controller that can reach its BMC.
func (a *App) bmc(ctx context.Context, m *Machine, args ...string) (string, error) {
c, err := a.credsFor(m)
if err != nil {
return "", fmt.Errorf("%w: %v", errNoBMC, err)
}
rack := a.rackOf(m)
if rack == a.cfg.RackName {
return ipmiExec(ctx, a.cfg.IPMITool, c, args...)
}
r, ok := a.racksSnapshot()[rack]
if !ok || !rackOnline(r, a.cfg.RackName) {
return "", fmt.Errorf("rack controller %s is offline", rack)
}
return a.hub.dispatch(ctx, rack, &rackJob{Kind: "ipmi", Address: c.Address, Port: c.Port, User: c.User, Password: c.Password, Args: args})
}
// ---------------------------------------------------------------- desired state
const defaultPXETmpl = `# generated by rollcall for rack controller %s
# unknown machines land here; known ones get a per-MAC file (01-xx-xx-…)
DEFAULT menu.c32
PROMPT 0
TIMEOUT 80
MENU TITLE rollcall · %s · unknown machine: commissioning in 8s
LABEL commission
MENU LABEL Commission this machine (inventory + BMC account)
MENU DEFAULT
KERNEL rollcall/vmlinuz
INITRD rollcall/initrd.img
APPEND %s
LABEL commission-debug
MENU LABEL Commission, but stay up afterwards (ssh root@<ip>)
KERNEL rollcall/vmlinuz
INITRD rollcall/initrd.img
APPEND %s rollcall.nopoweroff
LABEL local
MENU LABEL Boot from local disk
LOCALBOOT 0
`
func (a *App) rackState(ctx context.Context, name string) (*rackState, error) {
gen := a.hub.current()
st := &rackState{Gen: gen}
base := a.rackURL(name)
app := a.appendFor(base)
st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app)
var err error
if st.PXE, err = a.store.PXEFiles(ctx, name); err != nil {
return nil, err
}
subnets, err := a.store.Subnets(ctx)
if err != nil {
return nil, err
}
rc := a.racksSnapshot()[name]
rc.Name = name
dnsDir := a.cfg.DnsmasqDir
if dnsDir == "" {
dnsDir = "/var/lib/rollcall/dnsmasq"
}
if conf, ok := dnsmasqConfig(rc, subnets, dnsDir); ok {
st.DnsmasqConf = conf
}
if st.HostsDHCP, st.Hosts, err = a.hostsFiles(ctx, name, subnets); err != nil {
return nil, err
}
st.BootFiles = []string{"rollcall/vmlinuz", "rollcall/initrd.img"}
if ents, err := os.ReadDir(a.cfg.ImagesDir); err == nil {
for _, e := range ents {
if e.IsDir() {
st.BootFiles = append(st.BootFiles, "images/"+e.Name()+"/vmlinuz", "images/"+e.Name()+"/initrd.img")
}
}
}
return st, nil
}
// hostsFiles: static DHCP leases for the machines this rack serves, and names for every machine.
func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) {
ms, err := a.store.List(ctx)
if err != nil {
return "", "", err
}
var dhcp, hosts strings.Builder
dhcp.WriteString("# generated by rollcall\n")
hosts.WriteString("# generated by rollcall\n")
domainOf := func(ip string) string {
if sn := subnetFor(subnets, ip); sn != nil && sn.Domain != "" {
return sn.Domain
}
return str(a.cfg.ClusterVars["domain"])
}
sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) })
for _, m := range ms {
if m.NodeName == nil || len(m.MACs) == 0 {
continue
}
name := *m.NodeName
if ip := deref(m.MgmtIP); ip != "" {
if a.rackOf(m) == rack {
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], ip, name)
}
if d := domainOf(ip); d != "" {
fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name)
} else {
fmt.Fprintf(&hosts, "%s %s\n", ip, name)
}
}
if ip := deref(m.IBIP); ip != "" {
fmt.Fprintf(&hosts, "%s %s-ib\n", ip, name)
}
}
return dhcp.String(), hosts.String(), nil
}
// writeIfChanged replaces a file only when its content differs (path units watch these).
func writeIfChanged(p, content string, mode os.FileMode) (bool, error) {
if old, err := os.ReadFile(p); err == nil && string(old) == content {
return false, nil
}
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
return false, err
}
if err := os.WriteFile(p+".tmp", []byte(content), mode); err != nil {
return false, err
}
return true, os.Rename(p+".tmp", p)
}
// applyRackState writes a rack's state: PXE files under tftp/pxelinux.cfg
// (stale per-MAC files removed), the dnsmasq configuration and host files.
func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err error) {
dir := filepath.Join(tftp, "pxelinux.cfg")
note := func(p string, ch bool, e error) {
if e != nil && err == nil {
err = e
}
if ch {
changed = append(changed, p)
}
}
if st.DefaultPXE != "" {
ch, e := writeIfChanged(filepath.Join(dir, "default"), st.DefaultPXE, 0o644)
note("default", ch, e)
}
for name, body := range st.PXE {
ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644)
note(name, ch, e)
}
if olds, e := filepath.Glob(filepath.Join(dir, "01-*")); e == nil {
for _, p := range olds {
if _, keep := st.PXE[filepath.Base(p)]; !keep && !strings.HasSuffix(p, ".tmp") {
note("-"+filepath.Base(p), true, os.Remove(p))
}
}
}
if dnsDir != "" {
ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644)
note("hosts.dhcp", ch, e)
ch, e = writeIfChanged(filepath.Join(dnsDir, "hosts"), st.Hosts, 0o644)
note("hosts", ch, e)
conf := filepath.Join(dnsDir, "conf.d", "rollcall.conf")
if st.DnsmasqConf != "" {
ch, e = writeIfChanged(conf, st.DnsmasqConf, 0o644)
note("dnsmasq.conf", ch, e)
} else if _, e := os.Stat(conf); e == nil {
note("-dnsmasq.conf", true, os.Remove(conf))
}
}
return changed, err
}
// localRackLoop is the embedded rack controller: it applies this host's state on every change.
func (a *App) localRackLoop() {
apply := func() {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
st, err := a.rackState(ctx, a.cfg.RackName)
if err != nil {
log.Printf("embedded rack: %v", err)
return
}
if ch, err := applyRackState(st, a.cfg.TFTPRoot, a.cfg.DnsmasqDir); err != nil {
log.Printf("embedded rack: %v", err)
} else if len(ch) > 0 {
log.Printf("embedded rack %s: wrote %s", a.cfg.RackName, strings.Join(ch, " "))
}
_ = a.store.TouchRackController(ctx, a.cfg.RackName)
}
apply()
tick := time.NewTicker(60 * time.Second)
for {
select {
case <-a.hub.kick:
time.Sleep(150 * time.Millisecond) // let a burst of changes settle
select {
case <-a.hub.kick:
default:
}
case <-tick.C:
}
apply()
}
}
func localInterfaces() []RackIface {
var out []RackIface
ifs, _ := net.Interfaces()
for _, i := range ifs {
if i.Flags&net.FlagLoopback != 0 {
continue
}
ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()}
addrs, _ := i.Addrs()
for _, ad := range addrs {
if ipn, ok := ad.(*net.IPNet); ok && ipn.IP.To4() != nil {
ri.Addrs = append(ri.Addrs, ipn.String())
}
}
if len(ri.Addrs) > 0 {
out = append(out, ri)
}
}
return out
}
// startRacks registers the embedded rack controller, carries over what an
// older version kept on disk or in config, and starts the local apply loop.
func (a *App) startRacks(ctx context.Context) error {
if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version,
Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil {
return err
}
a.reloadNet(ctx)
// per-MAC files written before pxe_files existed
var n int
_ = a.store.pool.QueryRow(ctx, `SELECT count(*) FROM pxe_files`).Scan(&n)
if n == 0 {
files, _ := filepath.Glob(filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-*"))
ms, _ := a.store.List(ctx)
for _, f := range files {
b, err := os.ReadFile(f)
if err != nil {
continue
}
mac := strings.ReplaceAll(strings.TrimPrefix(filepath.Base(f), "01-"), "-", ":")
owner := ""
for _, m := range ms {
if hasTag(m.MACs, mac) {
owner = m.ID
}
}
_ = a.store.PutPXE(ctx, mac, owner, a.cfg.RackName, string(b))
}
if len(files) > 0 {
log.Printf("imported %d existing PXE files", len(files))
}
}
a.adoptIPs(ctx)
go a.localRackLoop()
return nil
}
// adoptIPs records addresses machines already have (node pools, before a
// subnet was modeled) so IPAM never hands them out twice.
func (a *App) adoptIPs(ctx context.Context) {
subnets, err := a.store.Subnets(ctx)
if err != nil || len(subnets) == 0 {
return
}
ms, _ := a.store.List(ctx)
for _, m := range ms {
for _, ip := range []string{deref(m.MgmtIP), deref(m.IBIP)} {
if sn := subnetFor(subnets, ip); sn != nil {
iface := ""
if b := bootNIC(machineNICs(m), m.MACs); b != nil && ip == deref(m.MgmtIP) {
iface = b.Name
}
_, _ = a.store.ClaimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: m.ID, Iface: iface, Kind: "auto", Hostname: deref(m.NodeName)})
}
}
}
}
// ---------------------------------------------------------------- PXE files through the rack
// writePXEBody records a machine's PXE file for its rack controller.
func (a *App) writePXEBody(m *Machine, body string) error {
rack := a.rackOf(m)
for _, mac := range m.MACs {
if err := a.store.PutPXE(context.Background(), normMAC(mac), m.ID, rack, body); err != nil {
return err
}
}
a.hub.bump()
return nil
}
func (a *App) removePXE(m *Machine) {
for _, mac := range m.MACs {
_ = a.store.DeletePXE(context.Background(), normMAC(mac))
}
a.hub.bump()
}
// exportDnsmasq: host files and dnsmasq configuration follow from the state now.
func (a *App) exportDnsmasq(ctx context.Context) error {
a.hub.bump()
return nil
}
// ---------------------------------------------------------------- HTTP: rack controllers
func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" {
fail(w, 401, errors.New("rack controller secret required"))
return
}
next(w, r, name)
}
}
// fromRack names the rack controller a node request came through (it proxies them).
func (a *App) fromRack(r *http.Request) string {
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 {
return name
}
return a.cfg.RackName
}
func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack string) {
var rc RackController
if err := json.NewDecoder(r.Body).Decode(&rc); err != nil {
fail(w, 400, err)
return
}
rc.Name, rc.Kind = rack, "remote"
if rack == a.cfg.RackName {
fail(w, 409, fmt.Errorf("%s is the embedded rack controller's name", rack))
return
}
if rc.URL == "" {
fail(w, 400, errors.New("url is required"))
return
}
if err := a.store.PutRackController(r.Context(), rc); err != nil {
fail(w, 500, err)
return
}
a.reloadRacks(r.Context())
a.hub.bump()
log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces))
writeJSON(w, 200, obj{"ok": true, "gen": a.hub.current()})
}
func (a *App) handleRackState(w http.ResponseWriter, r *http.Request, rack string) {
if _, ok := a.racksSnapshot()[rack]; !ok {
fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack))
return
}
since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
_ = a.store.TouchRackController(r.Context(), rack)
a.hub.waitChange(r.Context(), since, 25*time.Second)
_ = a.store.TouchRackController(r.Context(), rack)
go a.reloadRacks(context.Background()) // keep last_seen fresh for online checks
st, err := a.rackState(r.Context(), rack)
if err != nil {
fail(w, 500, err)
return
}
writeJSON(w, 200, st)
}
func (a *App) handleRackWork(w http.ResponseWriter, r *http.Request, rack string) {
jobs := a.hub.take(r.Context(), rack, 25*time.Second)
if jobs == nil {
jobs = []*rackJob{}
}
writeJSON(w, 200, jobs)
}
func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack string) {
var res rackResult
if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
fail(w, 400, err)
return
}
writeJSON(w, 200, obj{"accepted": a.hub.finish(res)})
}
func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) {
rs, err := a.store.RackControllers(r.Context())
if err != nil {
fail(w, 500, err)
return
}
ms, _ := a.store.List(r.Context())
for i := range rs {
rs[i].Online = rackOnline(rs[i], a.cfg.RackName)
for _, m := range ms {
if a.rackOf(m) == rs[i].Name {
rs[i].Machines++
}
}
}
writeJSON(w, 200, rs)
}
func (a *App) handleControllerDelete(w http.ResponseWriter, r *http.Request) {
name := r.PathValue("name")
if name == a.cfg.RackName {
fail(w, 409, errors.New("the embedded rack controller cannot be removed"))
return
}
if err := a.store.DeleteRackController(r.Context(), name); err != nil {
fail(w, 500, err)
return
}
a.reloadRacks(r.Context())
a.hub.bump()
writeJSON(w, 200, obj{"deleted": name})
}
// handleRackDHCP shows a rack's generated dnsmasq configuration (admin).
func (a *App) handleRackDHCP(w http.ResponseWriter, r *http.Request) {
st, err := a.rackState(r.Context(), r.PathValue("name"))
if err != nil {
fail(w, 500, err)
return
}
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
if st.DnsmasqConf == "" {
fmt.Fprintf(w, "# rack controller %s serves no VLAN with dhcp on: rollcall leaves dnsmasq's configuration alone\n", r.PathValue("name"))
}
fmt.Fprint(w, st.DnsmasqConf, "\n# --- hosts.dhcp\n", st.HostsDHCP)
}
+414
View File
@@ -0,0 +1,414 @@
package main
// `rollcall rack`: a rack controller next to the machines. It registers with
// the region, long-polls its desired state and writes it (PXE files, dnsmasq
// configuration, host files), runs BMC commands the region queues for it,
// serves boot and OS images from a local cache filled from the region on
// first use, and proxies the node API (commissioning, deploy jobs, overlays,
// phone-home) to the region, marking requests with its name.
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log"
"net/http"
"net/http/httputil"
"net/url"
"os"
"path"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
)
type RackConfig struct {
Name string `json:"name"` // default: hostname
Region string `json:"region"` // the region's URL, e.g. http://10.0.0.234
Secret string `json:"secret"` // the region's rack_secret
URL string `json:"url"` // how machines reach this rack, e.g. http://10.213.78.1
Listen string `json:"listen"` // default ":80"
TFTPRoot string `json:"tftp_root"` // default /srv/tftp
DnsmasqDir string `json:"dnsmasq_dir"` // default /var/lib/rollcall/dnsmasq
CacheDir string `json:"cache_dir"` // default /var/cache/rollcall
IPMITool string `json:"ipmitool"` // default ipmitool
DHCPInterfaces []string `json:"dhcp_interfaces"` // default: the interfaces with an address in a served subnet
}
type rackd struct {
cfg RackConfig
region *url.URL
hc *http.Client // long polls
fast *http.Client
gen int64
headMu sync.Mutex
heads map[string]headInfo
fetch sync.Mutex // one boot-file prefetch at a time
}
type headInfo struct {
size int64
mod time.Time
at time.Time
}
func loadRackConfig(p string) (RackConfig, error) {
var c RackConfig
b, err := os.ReadFile(p)
if err != nil {
return c, err
}
if err := json.Unmarshal(b, &c); err != nil {
return c, fmt.Errorf("%s: %w", p, err)
}
if c.Name == "" {
c.Name, _ = os.Hostname()
}
if c.Listen == "" {
c.Listen = ":80"
}
if c.TFTPRoot == "" {
c.TFTPRoot = "/srv/tftp"
}
if c.DnsmasqDir == "" {
c.DnsmasqDir = "/var/lib/rollcall/dnsmasq"
}
if c.CacheDir == "" {
c.CacheDir = "/var/cache/rollcall"
}
if c.IPMITool == "" {
c.IPMITool = "ipmitool"
}
c.Region, c.URL = strings.TrimRight(c.Region, "/"), strings.TrimRight(c.URL, "/")
for _, k := range []struct{ v, n string }{{c.Region, "region"}, {c.Secret, "secret"}, {c.URL, "url"}} {
if k.v == "" {
return c, fmt.Errorf("%s: %s is required", p, k.n)
}
}
return c, nil
}
func runRack(args []string) error {
fs := newFlags("rack")
cfgPath := fs.String("config", "/etc/rollcall/rack.json", "rack controller configuration")
_ = fs.Parse(args)
cfg, err := loadRackConfig(*cfgPath)
if err != nil {
return err
}
ru, err := url.Parse(cfg.Region)
if err != nil {
return fmt.Errorf("region: %w", err)
}
r := &rackd{cfg: cfg, region: ru, hc: &http.Client{Timeout: 45 * time.Second}, fast: &http.Client{Timeout: 30 * time.Second}, heads: map[string]headInfo{}}
for _, d := range []string{filepath.Join(cfg.TFTPRoot, "pxelinux.cfg"), cfg.CacheDir, filepath.Join(cfg.DnsmasqDir, "conf.d")} {
if err := os.MkdirAll(d, 0o755); err != nil {
return err
}
}
go r.loop()
mux := http.NewServeMux()
for _, p := range []string{"/rollcall/", "/images/", "/os/"} {
mux.HandleFunc("GET "+p, r.serveCached)
mux.HandleFunc("HEAD "+p, r.serveCached)
}
proxy := &httputil.ReverseProxy{
Rewrite: func(pr *httputil.ProxyRequest) {
pr.SetURL(ru)
pr.SetXForwarded()
pr.Out.Header.Set("X-Rollcall-Rack", cfg.Name)
pr.Out.Header.Set("X-Rollcall-Rack-Secret", cfg.Secret)
},
FlushInterval: 100 * time.Millisecond,
}
for _, p := range []string{"/api/v1/commission", "/api/v1/bmc-hint", "/api/v1/jobs/self", "/api/v1/jobs/self/progress",
"/api/v1/nodes/self/overlay", "/api/v1/phone-home/"} {
mux.Handle(p, proxy)
}
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
writeJSON(w, 200, obj{"rack": cfg.Name, "gen": r.gen, "version": version})
})
srv := &http.Server{Addr: cfg.Listen, Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second}
log.Printf("rack controller %s: region %s, serving %s on %s (tftp %s, cache %s)", cfg.Name, cfg.Region, cfg.URL, cfg.Listen, cfg.TFTPRoot, cfg.CacheDir)
return srv.ListenAndServe()
}
func (r *rackd) req(ctx context.Context, method, p string, body any) (*http.Request, error) {
var rd io.Reader
if body != nil {
b, _ := json.Marshal(body)
rd = bytes.NewReader(b)
}
req, err := http.NewRequestWithContext(ctx, method, r.cfg.Region+p, rd)
if err != nil {
return nil, err
}
req.Header.Set("X-Rollcall-Rack", r.cfg.Name)
req.Header.Set("X-Rollcall-Rack-Secret", r.cfg.Secret)
req.Header.Set("Content-Type", "application/json")
return req, nil
}
func (r *rackd) call(c *http.Client, method, p string, body, out any) (int, error) {
req, err := r.req(context.Background(), method, p, body)
if err != nil {
return 0, err
}
resp, err := c.Do(req)
if err != nil {
return 0, err
}
defer resp.Body.Close()
data, _ := io.ReadAll(resp.Body)
if resp.StatusCode >= 300 {
var e apiError
_ = json.Unmarshal(data, &e)
if e.Error == "" {
e.Error = strings.TrimSpace(string(data))
}
return resp.StatusCode, fmt.Errorf("%s %s: %s", method, p, e.Error)
}
if out != nil {
return resp.StatusCode, json.Unmarshal(data, out)
}
return resp.StatusCode, nil
}
func (r *rackd) register() error {
_, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version,
Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil)
return err
}
func (r *rackd) loop() {
backoff := time.Second
for {
if err := r.register(); err != nil {
log.Printf("register: %v (retry in %s)", err, backoff)
time.Sleep(backoff)
backoff = min(backoff*2, 30*time.Second)
continue
}
break
}
log.Printf("registered with %s as %s", r.cfg.Region, r.cfg.Name)
go r.workLoop()
go func() { // interfaces change; registration also refreshes last_seen
for range time.Tick(5 * time.Minute) {
if err := r.register(); err != nil {
log.Printf("re-register: %v", err)
}
}
}()
for {
var st rackState
code, err := r.call(r.hc, "GET", "/api/v1/rack/state?since="+strconv.FormatInt(r.gen, 10), nil, &st)
if err != nil {
log.Printf("state: %v", err)
if code == 409 || code == 401 {
_ = r.register()
}
time.Sleep(3 * time.Second)
continue
}
if st.Gen == r.gen {
continue
}
ch, err := applyRackState(&st, r.cfg.TFTPRoot, r.cfg.DnsmasqDir)
if err != nil {
log.Printf("apply: %v", err)
} else if len(ch) > 0 {
log.Printf("state %d: wrote %s", st.Gen, strings.Join(ch, " "))
}
r.gen = st.Gen
go r.prefetch(st.BootFiles)
}
}
func (r *rackd) workLoop() {
for {
var jobs []rackJob
if _, err := r.call(r.hc, "GET", "/api/v1/rack/work", nil, &jobs); err != nil {
log.Printf("work: %v", err)
time.Sleep(3 * time.Second)
continue
}
for _, j := range jobs {
go func(j rackJob) {
res := rackResult{ID: j.ID}
switch j.Kind {
case "ipmi":
out, err := ipmiExec(context.Background(), r.cfg.IPMITool, &bmcCreds{Address: j.Address, Port: j.Port, User: j.User, Password: j.Password}, j.Args...)
res.Output = out
if err != nil {
res.Error = err.Error()
if out != "" {
res.Error = out
}
}
default:
res.Error = "unknown job kind " + j.Kind
}
log.Printf("job %s %s %s: %s", j.ID, j.Kind, strings.Join(j.Args, " "), oneLine(res.Output+" "+res.Error))
if _, err := r.call(r.fast, "POST", "/api/v1/rack/result", res, nil); err != nil {
log.Printf("result %s: %v", j.ID, err)
}
}(j)
}
}
}
// ---------------------------------------------------------------- image cache
// head asks the region for a file's size and modification time (cached briefly).
func (r *rackd) head(rel string) (headInfo, error) {
r.headMu.Lock()
if h, ok := r.heads[rel]; ok && time.Since(h.at) < 30*time.Second {
r.headMu.Unlock()
return h, nil
}
r.headMu.Unlock()
req, _ := http.NewRequest("HEAD", r.cfg.Region+rel, nil)
resp, err := r.fast.Do(req)
if err != nil {
return headInfo{}, err
}
resp.Body.Close()
if resp.StatusCode != 200 {
return headInfo{}, fmt.Errorf("%s: %s", rel, resp.Status)
}
mod, _ := http.ParseTime(resp.Header.Get("Last-Modified"))
h := headInfo{size: resp.ContentLength, mod: mod, at: time.Now()}
r.headMu.Lock()
r.heads[rel] = h
r.headMu.Unlock()
return h, nil
}
// fresh says whether the cached copy matches the region's file. OS images live
// under versioned paths and never change; the rest is checked by size and time.
func (r *rackd) fresh(rel string, fi os.FileInfo) bool {
if strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/") {
return true
}
h, err := r.head(rel)
if err != nil {
return true // region unreachable: serve what we have
}
return h.size == fi.Size() && h.mod.Unix() == fi.ModTime().Unix()
}
func (r *rackd) localPath(rel string) (string, bool) {
clean := path.Clean("/" + rel)
if clean == "/" || strings.Contains(clean, "..") {
return "", false
}
return filepath.Join(r.cfg.CacheDir, filepath.FromSlash(clean)), true
}
// fill downloads rel from the region into the cache, copying to w on the way when given.
func (r *rackd) fill(rel string, w http.ResponseWriter) error {
local, ok := r.localPath(rel)
if !ok {
return errors.New("bad path")
}
resp, err := http.Get(r.cfg.Region + rel) // no timeout: images are large
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
if w != nil {
w.WriteHeader(resp.StatusCode)
}
return fmt.Errorf("%s: %s", rel, resp.Status)
}
if err := os.MkdirAll(filepath.Dir(local), 0o755); err != nil {
return err
}
tmp, err := os.CreateTemp(filepath.Dir(local), ".fill-*")
if err != nil {
return err
}
defer os.Remove(tmp.Name())
var dst io.Writer = tmp
if w != nil {
for _, h := range []string{"Content-Length", "Content-Type", "Last-Modified"} {
if v := resp.Header.Get(h); v != "" {
w.Header().Set(h, v)
}
}
w.WriteHeader(200)
dst = io.MultiWriter(tmp, w)
}
n, err := io.Copy(dst, resp.Body)
if cerr := tmp.Close(); err == nil {
err = cerr
}
if err != nil {
return err
}
if resp.ContentLength >= 0 && n != resp.ContentLength {
return fmt.Errorf("%s: short read %d/%d", rel, n, resp.ContentLength)
}
if mod, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil {
_ = os.Chtimes(tmp.Name(), mod, mod)
}
_ = os.Chmod(tmp.Name(), 0o644)
if err := os.Rename(tmp.Name(), local); err != nil {
return err
}
log.Printf("cached %s (%s)", rel, human(float64(n)))
return nil
}
func (r *rackd) serveCached(w http.ResponseWriter, req *http.Request) {
rel := path.Clean(req.URL.Path)
local, ok := r.localPath(rel)
if !ok {
http.NotFound(w, req)
return
}
if fi, err := os.Stat(local); err == nil && !fi.IsDir() && r.fresh(rel, fi) {
http.ServeFile(w, req, local)
return
}
if req.Method == http.MethodHead {
h, err := r.head(rel)
if err != nil {
http.NotFound(w, req)
return
}
w.Header().Set("Content-Length", strconv.FormatInt(h.size, 10))
w.Header().Set("Last-Modified", h.mod.UTC().Format(http.TimeFormat))
return
}
if err := r.fill(rel, w); err != nil {
log.Printf("cache %s: %v", rel, err)
}
}
// prefetch keeps kernels and initrds cached: TFTP cannot fetch on demand.
func (r *rackd) prefetch(files []string) {
r.fetch.Lock()
defer r.fetch.Unlock()
for _, f := range files {
rel := "/" + strings.TrimPrefix(f, "/")
local, ok := r.localPath(rel)
if !ok {
continue
}
if fi, err := os.Stat(local); err == nil && r.fresh(rel, fi) {
continue
}
if err := r.fill(rel, nil); err != nil {
log.Printf("prefetch %s: %v", rel, err)
}
}
}
+51 -48
View File
@@ -11,10 +11,9 @@ import (
"io/fs"
"log"
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"sync"
"time"
)
@@ -25,6 +24,11 @@ type App struct {
cfg *Config
store *Store
sealer *Sealer
hub *rackHub
netMu sync.RWMutex
racks map[string]RackController
subnets []Subnet
}
type apiError struct {
@@ -146,6 +150,33 @@ func (a *App) routes() http.Handler {
}
writeJSON(w, 200, jobs)
}))
// network model and rack controllers
mux.HandleFunc("GET /api/v1/fabrics", a.adminAuth(a.handleFabrics))
mux.HandleFunc("POST /api/v1/fabrics", a.adminAuth(a.handleFabrics))
mux.HandleFunc("DELETE /api/v1/fabrics/{name}", a.adminAuth(a.handleFabricDelete))
mux.HandleFunc("GET /api/v1/vlans", a.adminAuth(a.handleVLANs))
mux.HandleFunc("POST /api/v1/vlans", a.adminAuth(a.handleVLANs))
mux.HandleFunc("DELETE /api/v1/vlans/{fabric}/{vid}", a.adminAuth(a.handleVLANDelete))
mux.HandleFunc("GET /api/v1/subnets", a.adminAuth(a.handleSubnets))
mux.HandleFunc("POST /api/v1/subnets", a.adminAuth(a.handleSubnets))
mux.HandleFunc("DELETE /api/v1/subnets/{cidr...}", a.adminAuth(a.handleSubnetDelete))
mux.HandleFunc("POST /api/v1/ranges", a.adminAuth(a.handleRanges))
mux.HandleFunc("DELETE /api/v1/ranges/{id}", a.adminAuth(a.handleRangeDelete))
mux.HandleFunc("GET /api/v1/ips", a.adminAuth(a.handleIPs))
mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs))
mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete))
mux.HandleFunc("GET /api/v1/layouts", a.adminAuth(a.handleLayouts))
mux.HandleFunc("POST /api/v1/layouts", a.adminAuth(a.handleLayouts))
mux.HandleFunc("DELETE /api/v1/layouts/{name}", a.adminAuth(a.handleLayoutDelete))
mux.HandleFunc("GET /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
mux.HandleFunc("POST /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
mux.HandleFunc("GET /api/v1/controllers", a.adminAuth(a.handleControllers))
mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete))
mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP))
mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister))
mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState))
mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork))
mux.HandleFunc("POST /api/v1/rack/result", a.rackAuth(a.handleRackResult))
// physical location
mux.HandleFunc("POST /api/v1/machines/{id}/location", a.adminAuth(a.handleLocation))
mux.HandleFunc("POST /api/v1/locations/import", a.adminAuth(a.handleLocationImport))
@@ -271,6 +302,8 @@ type Summary struct {
ManualTags []string `json:"manual_tags"`
AutoTags []string `json:"auto_tags"`
Location Location `json:"location"`
Rack string `json:"rack_controller"`
NetLayout string `json:"net_layout"`
}
func num(v any) float64 {
@@ -289,7 +322,8 @@ func summarize(m *Machine) Summary {
IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: nodeCtx(m).GPUs, // real accelerators only
Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt,
OS: m.OS, OSVersion: m.OSVersion, DeployedAt: m.DeployedAt,
Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location}
Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location,
Rack: m.RackController, NetLayout: m.NetLayout}
if m.BMC != nil {
var b obj
if json.Unmarshal(m.BMC, &b) == nil {
@@ -325,10 +359,6 @@ LABEL local
LOCALBOOT 0
`
func (a *App) pxePath(mac string) string {
return filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-"+strings.ReplaceAll(normMAC(mac), ":", "-"))
}
func deref(p *string) string {
if p == nil {
return ""
@@ -339,32 +369,10 @@ func deref(p *string) string {
func (a *App) writePXE(m *Machine) error {
man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial)
body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser,
m.ID, man, prod, ser, a.cfg.CommissionAppend)
m.ID, man, prod, ser, a.appendFor(a.urlFor(m)))
return a.writePXEBody(m, body)
}
func (a *App) writePXEBody(m *Machine, body string) error {
for _, mac := range m.MACs {
p := a.pxePath(mac)
if err := os.WriteFile(p+".tmp", []byte(body), 0o644); err != nil {
return err
}
if err := os.Rename(p+".tmp", p); err != nil {
return err
}
log.Printf("wrote %s", p)
}
return nil
}
func (a *App) removePXE(m *Machine) {
for _, mac := range m.MACs {
if err := os.Remove(a.pxePath(mac)); err == nil {
log.Printf("removed %s", a.pxePath(mac))
}
}
}
// ---------------------------------------------------------------- handlers
func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
@@ -427,6 +435,7 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
return
}
_ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail)
_ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r))
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
m = full // carries profile / node token when this is a re-commission of a stateless node
}
@@ -527,17 +536,7 @@ func (a *App) handleRecommission(w http.ResponseWriter, r *http.Request) {
// optional: power-cycle it right away when the caller asks (?now=1)
out := obj{"id": m.ID, "state": "recommission"}
if r.URL.Query().Get("now") == "1" {
if c, err := a.credsFor(m); err == nil {
o, ierr := a.ipmi(r.Context(), c, "chassis", "bootdev", "pxe")
o2, ierr2 := a.ipmi(r.Context(), c, "chassis", "power", "cycle")
if ierr2 != nil && strings.Contains(o2, "off") { // cycle fails when off; power on instead
o2, ierr2 = a.ipmi(r.Context(), c, "chassis", "power", "on")
}
out["ipmi"] = strings.TrimSpace(o + "\n" + o2)
out["ipmi_ok"] = ierr == nil && ierr2 == nil
} else {
out["ipmi"] = err.Error()
}
out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m)
}
writeJSON(w, 200, out)
}
@@ -564,12 +563,11 @@ func (a *App) handlePower(w http.ResponseWriter, r *http.Request) {
fail(w, 400, fmt.Errorf("unknown power action %q", action))
return
}
c, err := a.credsFor(m)
if err != nil {
out, err := a.bmc(r.Context(), m, args...)
if errors.Is(err, errNoBMC) {
fail(w, 409, err)
return
}
out, err := a.ipmi(r.Context(), c, args...)
if action != "status" {
_ = a.store.AddEvent(r.Context(), m.ID, "power "+action, out)
}
@@ -589,12 +587,11 @@ func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) {
fail(w, 400, fmt.Errorf("device must be one of pxe, disk, cdrom, bios, none, safe"))
return
}
c, err := a.credsFor(m)
if err != nil {
out, err := a.bmc(r.Context(), m, "chassis", "bootdev", req.Device)
if errors.Is(err, errNoBMC) {
fail(w, 409, err)
return
}
out, err := a.ipmi(r.Context(), c, "chassis", "bootdev", req.Device)
_ = a.store.AddEvent(r.Context(), m.ID, "bootdev "+req.Device, out)
writeJSON(w, 200, obj{"id": m.ID, "device": req.Device, "ok": err == nil, "output": out})
}
@@ -614,7 +611,13 @@ func serve(cfgPath string) error {
if err != nil {
return fmt.Errorf("database: %w", err)
}
app := &App{cfg: cfg, store: store, sealer: sealer}
app := &App{cfg: cfg, store: store, sealer: sealer, hub: newRackHub()}
ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
err = app.startRacks(ctx2)
cancel2()
if err != nil {
return fmt.Errorf("rack controllers: %w", err)
}
srv := &http.Server{Addr: cfg.Listen, Handler: app.routes(), ReadHeaderTimeout: 10 * time.Second}
log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "")
return srv.ListenAndServe()
+40 -59
View File
@@ -186,7 +186,29 @@ func parsePool(spec string) ([]string, error) {
return out, nil
}
func (a *App) allocate(ctx context.Context, nameFormat, wantName string) (name, mgmt, ib string, err error) {
// allocate picks a node name and addresses: from IPAM when the machine's boot
// NIC sits in a modeled subnet, else from the legacy node_pools.
func (a *App) allocate(ctx context.Context, m *Machine, nameFormat, wantName string) (name, mgmt, ib string, err error) {
if name, _, _, err = a.allocatePools(ctx, nameFormat, wantName, false); err != nil {
return
}
if b := bootNIC(machineNICs(m), m.MACs); b != nil {
for _, ip := range b.IPv4 {
if sn := subnetFor(a.subnetsSnapshot(), ip); sn != nil {
have, _ := a.store.IPs(ctx, sn.CIDR, m.ID)
for _, x := range have {
return name, x.IP, "", nil
}
mgmt, err = a.allocIP(ctx, *sn, m.ID, b.Name, name)
return
}
}
}
_, mgmt, ib, err = a.allocatePools(ctx, nameFormat, name, true)
return
}
func (a *App) allocatePools(ctx context.Context, nameFormat, wantName string, pools bool) (name, mgmt, ib string, err error) {
ms, err := a.store.List(ctx)
if err != nil {
return
@@ -215,10 +237,13 @@ func (a *App) allocate(ctx context.Context, nameFormat, wantName string) (name,
break
}
}
} else if usedName[name] {
} else if usedName[name] && !pools {
err = fmt.Errorf("node name %s already in use", name)
return
}
if !pools {
return
}
pick := func(pool string) (string, error) {
spec := a.cfg.NodePools[pool]
if spec == "" {
@@ -441,7 +466,7 @@ func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) {
}
}
}
return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, Vars: vars, Server: a.cfg.ServerURL,
return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, Vars: vars, Server: a.urlFor(m),
Now: time.Now().Format(time.RFC3339)}, nil
}
@@ -614,11 +639,12 @@ LABEL local
`
func (a *App) statelessAppend(m *Machine, p *Profile, token string) string {
base := a.urlFor(m)
parts := []string{"boot=live", "components",
"fetch=" + a.cfg.ServerURL + "/images/" + p.Image + "/filesystem.squashfs",
"fetch=" + base + "/images/" + p.Image + "/filesystem.squashfs",
"hostname=" + deref(m.NodeName),
"live-config.nocomponents=user-setup,sudo",
"rollcall.server=" + a.cfg.ServerURL, "rollcall.node=" + token}
"rollcall.server=" + base, "rollcall.node=" + token}
if p.Append != "" {
parts = append(parts, p.Append)
}
@@ -628,7 +654,7 @@ func (a *App) statelessAppend(m *Machine, p *Profile, token string) string {
func (a *App) writePXEStateless(m *Machine, p *Profile, token string) error {
body := fmt.Sprintf(statelessTmpl, time.Now().Format(time.RFC3339), deref(m.NodeName), m.ID, p.Name,
deref(m.NodeName), p.Name, deref(m.NodeName), p.Name, p.Image, p.Image, a.statelessAppend(m, p, token),
a.cfg.CommissionAppend)
a.appendFor(a.urlFor(m)))
return a.writePXEBody(m, body)
}
@@ -651,51 +677,6 @@ func (a *App) writePXEAny(ctx context.Context, m *Machine) error {
return a.writePXE(m)
}
// exportDnsmasq writes dhcp-hostsfile and addn-hosts files; a systemd path unit HUPs dnsmasq.
func (a *App) exportDnsmasq(ctx context.Context) error {
if a.cfg.DnsmasqDir == "" {
return nil
}
ms, err := a.store.List(ctx)
if err != nil {
return err
}
domain := str(a.cfg.ClusterVars["domain"])
var dhcp, hosts bytes.Buffer
fmt.Fprintf(&dhcp, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339))
fmt.Fprintf(&hosts, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339))
for _, m := range ms {
if m.NodeName == nil || len(m.MACs) == 0 {
continue
}
name := *m.NodeName
if m.MgmtIP != nil {
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], *m.MgmtIP, name)
if domain != "" {
fmt.Fprintf(&hosts, "%s %s.%s %s\n", *m.MgmtIP, name, domain, name)
} else {
fmt.Fprintf(&hosts, "%s %s\n", *m.MgmtIP, name)
}
}
if m.IBIP != nil {
fmt.Fprintf(&hosts, "%s %s-ib\n", *m.IBIP, name)
}
}
if err := os.MkdirAll(a.cfg.DnsmasqDir, 0o755); err != nil {
return err
}
for fn, b := range map[string]*bytes.Buffer{"hosts.dhcp": &dhcp, "hosts": &hosts} {
p := filepath.Join(a.cfg.DnsmasqDir, fn)
if err := os.WriteFile(p+".tmp", b.Bytes(), 0o644); err != nil {
return err
}
if err := os.Rename(p+".tmp", p); err != nil {
return err
}
}
return nil
}
// ---------------------------------------------------------------- handlers
func (a *App) handleProfiles(w http.ResponseWriter, r *http.Request) {
@@ -769,7 +750,7 @@ func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) {
// keep the existing name/IPs when re-assigning an already named node
name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP)
if name == "" || (req.Name != "" && req.Name != name) {
if name, mgmt, ib, err = a.allocate(r.Context(), p.NameFormat, req.Name); err != nil {
if name, mgmt, ib, err = a.allocate(r.Context(), m, p.NameFormat, req.Name); err != nil {
fail(w, 409, err)
return
}
@@ -805,18 +786,17 @@ func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) {
// pxeBoot sets bootdev pxe and power-cycles (or powers on) through the BMC.
func (a *App) pxeBoot(ctx context.Context, m *Machine) (string, bool) {
c, err := a.credsFor(m)
if err != nil {
return err.Error(), false
o1, e1 := a.bmc(ctx, m, "chassis", "bootdev", "pxe")
if errors.Is(e1, errNoBMC) {
return e1.Error(), false
}
o1, e1 := a.ipmi(ctx, c, "chassis", "bootdev", "pxe")
st, _ := a.ipmi(ctx, c, "chassis", "power", "status")
st, _ := a.bmc(ctx, m, "chassis", "power", "status")
var o2 string
var e2 error
if strings.Contains(st, "off") {
o2, e2 = a.ipmi(ctx, c, "chassis", "power", "on")
o2, e2 = a.bmc(ctx, m, "chassis", "power", "on")
} else {
o2, e2 = a.ipmi(ctx, c, "chassis", "power", "cycle")
o2, e2 = a.bmc(ctx, m, "chassis", "power", "cycle")
}
_ = a.store.AddEvent(ctx, m.ID, "pxe boot", strings.TrimSpace(o1+"\n"+o2))
return strings.TrimSpace(o1 + "\n" + o2), e1 == nil && e2 == nil
@@ -828,6 +808,7 @@ func (a *App) handleUnassign(w http.ResponseWriter, r *http.Request) {
fail(w, 500, err)
return
}
_ = a.store.FreeMachineIPs(r.Context(), m.ID, "auto")
if err := a.store.Unassign(r.Context(), m.ID); err != nil {
fail(w, 500, err)
return
+83 -3
View File
@@ -100,6 +100,76 @@ CREATE TABLE IF NOT EXISTS racks (
comment text,
PRIMARY KEY (building, room, name)
);
ALTER TABLE machines ADD COLUMN IF NOT EXISTS rack_controller text;
ALTER TABLE machines ADD COLUMN IF NOT EXISTS net_layout text;
ALTER TABLE tags ADD COLUMN IF NOT EXISTS net_layout text;
CREATE TABLE IF NOT EXISTS fabrics (
name text PRIMARY KEY,
kind text NOT NULL DEFAULT 'ethernet',
comment text
);
CREATE TABLE IF NOT EXISTS vlans (
id bigserial PRIMARY KEY,
fabric text NOT NULL REFERENCES fabrics(name) ON DELETE CASCADE,
vid int NOT NULL DEFAULT 0,
name text,
mtu int NOT NULL DEFAULT 1500,
dhcp boolean NOT NULL DEFAULT false,
primary_rack text,
secondary_rack text,
UNIQUE (fabric, vid)
);
CREATE TABLE IF NOT EXISTS subnets (
cidr text PRIMARY KEY,
name text,
vlan_id bigint REFERENCES vlans(id) ON DELETE SET NULL,
purpose text NOT NULL DEFAULT 'provisioning',
gateway text,
dns jsonb NOT NULL DEFAULT '[]',
domain text,
comment text
);
CREATE TABLE IF NOT EXISTS ip_ranges (
id bigserial PRIMARY KEY,
subnet text NOT NULL REFERENCES subnets(cidr) ON DELETE CASCADE,
kind text NOT NULL,
start_ip text NOT NULL,
end_ip text NOT NULL,
comment text
);
CREATE TABLE IF NOT EXISTS ip_addresses (
ip text PRIMARY KEY,
subnet text NOT NULL REFERENCES subnets(cidr) ON DELETE CASCADE,
machine_id text,
iface text,
kind text NOT NULL,
hostname text,
created timestamptz NOT NULL DEFAULT now()
);
CREATE INDEX IF NOT EXISTS ip_addresses_machine ON ip_addresses(machine_id);
CREATE TABLE IF NOT EXISTS net_layouts (
name text PRIMARY KEY,
spec jsonb NOT NULL,
comment text
);
CREATE TABLE IF NOT EXISTS rack_controllers (
name text PRIMARY KEY,
url text,
kind text NOT NULL DEFAULT 'remote',
version text,
interfaces jsonb NOT NULL DEFAULT '[]',
dhcp_interfaces jsonb NOT NULL DEFAULT '[]',
tftp_root text,
last_seen timestamptz,
registered timestamptz NOT NULL DEFAULT now()
);
CREATE TABLE IF NOT EXISTS pxe_files (
mac text PRIMARY KEY,
machine_id text,
rack text,
content text NOT NULL,
updated timestamptz NOT NULL DEFAULT now()
);
`
var errNotFound = errors.New("not found")
@@ -131,6 +201,8 @@ type Machine struct {
Tags []string `json:"tags"` // manual
AutoTags []string `json:"auto_tags"` // from tag match rules, recomputed at commissioning
Location Location `json:"location"`
RackController string `json:"rack_controller"`
NetLayout string `json:"net_layout"`
bmcSecret []byte
nodeToken []byte
hostKey []byte
@@ -156,7 +228,8 @@ func openStore(ctx context.Context, url string) (*Store, error) {
const machineCols = `id, uuid, serial, hostname, manufacturer, product, state, first_seen, last_seen,
commissioned_at, macs, bmc, bmc_secret, inventory, profile, node_name, mgmt_ip, ib_ip, node_token, host_key, booted_at,
os, os_version, deployed_at, tags, auto_tags,
coalesce(building,''), coalesce(room,''), coalesce(rack,''), coalesce(rack_unit,0), coalesce(asset_tag,'')`
coalesce(building,''), coalesce(room,''), coalesce(rack,''), coalesce(rack_unit,0), coalesce(asset_tag,''),
coalesce(rack_controller,''), coalesce(net_layout,'')`
func scanMachine(row pgx.Row, full bool) (*Machine, error) {
m := &Machine{}
@@ -165,7 +238,8 @@ func scanMachine(row pgx.Row, full bool) (*Machine, error) {
&m.FirstSeen, &m.LastSeen, &m.CommissionedAt, &macs, &m.BMC, &m.bmcSecret, &m.Inventory,
&m.Profile, &m.NodeName, &m.MgmtIP, &m.IBIP, &m.nodeToken, &m.hostKey, &m.BootedAt,
&m.OS, &m.OSVersion, &m.DeployedAt, &tags, &auto,
&m.Location.Building, &m.Location.Room, &m.Location.Rack, &m.Location.Unit, &m.Location.AssetTag)
&m.Location.Building, &m.Location.Room, &m.Location.Rack, &m.Location.Unit, &m.Location.AssetTag,
&m.RackController, &m.NetLayout)
if errors.Is(err, pgx.ErrNoRows) {
return nil, errNotFound
}
@@ -276,7 +350,7 @@ func (s *Store) UpsertCommissioned(ctx context.Context, m *Machine, secret []byt
}
// a node that already has a profile goes back to 'stateless', everything else to 'ready'
_, err = tx.Exec(ctx, `UPDATE machines SET uuid=$2, serial=$3, hostname=$4, manufacturer=$5, product=$6,
state=CASE WHEN profile IS NULL THEN 'ready' ELSE 'stateless' END, booted_at=NULL,
state=CASE WHEN profile IS NULL THEN 'ready' ELSE 'stateless' END, booted_at=NULL, os=NULL, os_version=NULL, deployed_at=NULL,
last_seen=now(), commissioned_at=now(), macs=$7, bmc=$8, bmc_secret=$9, inventory=$10 WHERE id=$1`,
m.ID, m.UUID, m.Serial, m.Hostname, m.Manufacturer, m.Product, macs, m.BMC, secret, m.Inventory)
if err != nil {
@@ -308,6 +382,12 @@ func (s *Store) Delete(ctx context.Context, id string) error {
if _, err := tx.Exec(ctx, `DELETE FROM jobs WHERE machine_id=$1`, id); err != nil {
return err
}
if _, err := tx.Exec(ctx, `DELETE FROM ip_addresses WHERE machine_id=$1`, id); err != nil {
return err
}
if _, err := tx.Exec(ctx, `DELETE FROM pxe_files WHERE machine_id=$1`, id); err != nil {
return err
}
tag, err := tx.Exec(ctx, `DELETE FROM machines WHERE id=$1`, id)
if err != nil {
return err
+6 -5
View File
@@ -31,6 +31,7 @@ type TagDef struct {
Match string `json:"match"`
CloudConfig string `json:"cloud_config"`
Overlays []string `json:"overlays"`
NetLayout string `json:"net_layout"`
Created time.Time `json:"created"`
Machines int `json:"machines,omitempty"` // filled by the list handler
}
@@ -40,7 +41,7 @@ var tagNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,63}$`)
// ---------------------------------------------------------------- store
func (s *Store) Tags(ctx context.Context) ([]TagDef, error) {
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(comment,''), coalesce(match,''), coalesce(cloud_config,''), overlays, created FROM tags ORDER BY name`)
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(comment,''), coalesce(match,''), coalesce(cloud_config,''), overlays, coalesce(net_layout,''), created FROM tags ORDER BY name`)
if err != nil {
return nil, err
}
@@ -49,7 +50,7 @@ func (s *Store) Tags(ctx context.Context) ([]TagDef, error) {
for rows.Next() {
var t TagDef
var ov []byte
if err := rows.Scan(&t.Name, &t.Comment, &t.Match, &t.CloudConfig, &ov, &t.Created); err != nil {
if err := rows.Scan(&t.Name, &t.Comment, &t.Match, &t.CloudConfig, &ov, &t.NetLayout, &t.Created); err != nil {
return nil, err
}
_ = json.Unmarshal(ov, &t.Overlays)
@@ -66,9 +67,9 @@ func (s *Store) PutTag(ctx context.Context, t TagDef) error {
t.Overlays = []string{}
}
ov, _ := json.Marshal(t.Overlays)
_, err := s.pool.Exec(ctx, `INSERT INTO tags(name, comment, match, cloud_config, overlays) VALUES ($1,$2,$3,$4,$5)
ON CONFLICT (name) DO UPDATE SET comment=$2, match=$3, cloud_config=$4, overlays=$5`,
t.Name, t.Comment, t.Match, t.CloudConfig, ov)
_, err := s.pool.Exec(ctx, `INSERT INTO tags(name, comment, match, cloud_config, overlays, net_layout) VALUES ($1,$2,$3,$4,$5,nullif($6,''))
ON CONFLICT (name) DO UPDATE SET comment=$2, match=$3, cloud_config=$4, overlays=$5, net_layout=nullif($6,'')`,
t.Name, t.Comment, t.Match, t.CloudConfig, ov, t.NetLayout)
return err
}
+143 -2
View File
@@ -69,6 +69,7 @@ const (
pOverview = iota
pMachines
pRacks
pNetwork
pJobs
pEvents
pTags
@@ -77,8 +78,8 @@ const (
pCount
)
var pageLabels = [pCount]string{"Overview", "Machines", "Racks", "Jobs", "Events", "Tags", "Profiles", "OS images"}
var pageNames = [pCount]string{"overview", "machines", "racks", "jobs", "events", "tags", "profiles", "images"}
var pageLabels = [pCount]string{"Overview", "Machines", "Racks", "Network", "Jobs", "Events", "Tags", "Profiles", "OS images"}
var pageNames = [pCount]string{"overview", "machines", "racks", "network", "jobs", "events", "tags", "profiles", "images"}
type tuiData struct {
machines []Summary
@@ -88,6 +89,9 @@ type tuiData struct {
tags []TagDef
profiles []Profile
os []OSImage
subnets []Subnet
ctls []RackController
layouts []NetLayout
at time.Time
}
@@ -98,6 +102,7 @@ type machineDetail struct {
Inventory obj `json:"inventory"`
Events []Event `json:"events"`
Jobs []*Job `json:"jobs"`
Net obj `json:"-"`
}
type (
@@ -339,12 +344,17 @@ func (m tuiModel) fetch() tea.Cmd {
get("/api/v1/tags", &d.tags)
get("/api/v1/profiles", &d.profiles)
get("/api/v1/os", &d.os)
get("/api/v1/subnets", &d.subnets)
get("/api/v1/controllers", &d.ctls)
get("/api/v1/layouts", &d.layouts)
d.at = time.Now()
var det *machineDetail
if id != "" && err == nil {
det = &machineDetail{}
if e := c.do("GET", "/api/v1/machines/"+id, nil, det); e != nil {
det = nil
} else {
_ = c.do("GET", "/api/v1/machines/"+id+"/network", nil, &det.Net)
}
}
return tuiDataMsg{d: d, detail: det, detailID: id, err: err}
@@ -650,6 +660,18 @@ func (m tuiModel) enter() (tea.Model, tea.Cmd) {
if m.row < len(m.racks()) {
m.rackOpen, m.rackRow = m.row, 0
}
case pNetwork:
if m.row < len(m.d.subnets) {
s := m.d.subnets[m.row]
var b strings.Builder
fmt.Fprintf(&b, "%s · %s · fabric %s vlan %d · mtu %d\ngateway %s · dns %s · domain %s\ndhcp: %s\n\n", orDash(s.Name), s.Purpose, orDash(s.Fabric), s.VID, s.MTU,
orDash(s.Gateway), orDash(strings.Join(s.DNS, ",")), orDash(s.Domain), pick(s.DHCP, "on, rack controller "+s.PrimaryRack+pick(s.SecondaryRack != "", " + "+s.SecondaryRack, ""), "off"))
for _, r := range s.Ranges {
fmt.Fprintf(&b, "%-9s %s – %s %s\n", r.Kind, r.Start, r.End, sDim.Render(r.Comment))
}
fmt.Fprintf(&b, "\n%d of %d addresses in use", s.Used, s.Size)
m.dlg = &tuiDialog{title: "Subnet " + s.CIDR, info: b.String()}
}
case pJobs:
if m.row < len(m.d.jobs) {
return m.open(m.d.jobs[m.row].MachineID)
@@ -878,6 +900,7 @@ func (m tuiModel) actionMenu(ids []string) *tuiDialog {
}},
{label: "Tags…", sep: true, run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.tagDialog(ids) }},
{label: "Location…", run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.locationDialog(ids) }},
{label: "Network layout…", run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.layoutDialog(ids) }},
{label: "Re-commission on next boot", sep: true, run: func(m tuiModel) (tuiModel, tea.Cmd) {
return m.run("recommission", ids, apiPost("/recommission", nil, func(d obj) string { return "state " + str(d["state"]) }))
}},
@@ -1043,6 +1066,22 @@ func (m tuiModel) locationDialog(ids []string) (tuiModel, tea.Cmd) {
return m, nil
}
func (m tuiModel) layoutDialog(ids []string) (tuiModel, tea.Cmd) {
opts, vals := []string{"(from tags, else the boot NIC on its subnet)"}, []string{""}
for _, l := range m.d.layouts {
opts = append(opts, l.Name+" "+sDim.Render(trunc(layoutSummary(l), 40)))
vals = append(vals, l.Name)
}
m.dlg = &tuiDialog{title: "Network layout · " + m.who(ids), note: "Applied at the next deployment; addresses come from the subnets the layout names.",
fields: []dlgField{{label: "Layout", kind: fkChoice, opts: opts, vals: vals}}, okLabel: "save",
onOK: func(m tuiModel, f []dlgField) (tuiModel, tea.Cmd) {
req := obj{"layout": field(f, "Layout")}
return m.run("network layout", ids, apiPost("/network", req, func(d obj) string { return str(d["layout"]) }))
}}
m.dlg.setFocus(0)
return m, nil
}
func (m tuiModel) releaseDialog(ids []string) (tuiModel, tea.Cmd) {
m.dlg = &tuiDialog{title: "Release · " + m.who(ids), note: "The OS disk is wiped and the machine returns to ready.",
fields: []dlgField{
@@ -1215,6 +1254,13 @@ func (m tuiModel) panel() (string, string, string, int) {
k = side
}
return "Racks", m.racksBody(pw), k, m.sel0(3)
case pNetwork:
body, head := m.networkBody(pw)
sel := -1
if m.inPanel {
sel = m.row + head
}
return "Network", body, pick(m.inPanel, "↑↓ subnet · Enter details · Esc back · r refresh · q quit", side), sel
case pJobs:
return "Jobs", m.jobsBody(pw), pick(m.inPanel, list, side), m.sel0(3)
case pEvents:
@@ -1249,6 +1295,8 @@ func (m tuiModel) rowCount() int {
return len(m.filtered())
case pRacks:
return len(m.racks())
case pNetwork:
return len(m.d.subnets)
case pJobs:
return len(m.d.jobs)
case pEvents:
@@ -1581,6 +1629,9 @@ func (m tuiModel) overviewBody(pw int) string {
failed++
}
}
for _, c := range m.d.ctls {
note(!c.Online, sBad.Render("rack controller "+c.Name+" is offline (last seen "+agoP(c.LastSeen)+" ago)"))
}
note(nobmc > 0, sBad.Render(fmt.Sprintf("%d machine(s) without a configured BMC", nobmc)))
note(silent > 0, sWarn.Render(fmt.Sprintf("%d stateless node(s) silent for more than 15 minutes", silent)))
note(failed > 0, sBad.Render(fmt.Sprintf("%d failed job(s) among the last 100", failed)))
@@ -1718,6 +1769,46 @@ func (m tuiModel) detailPanel(pw int) (string, string, string, int) {
kv("BMC", sBad.Render("none"))
}
if d.Net != nil {
sec("Network · layout " + str(d.Net["layout"]) + " · rack controller " + str(d.Net["rack_controller"]))
plan, _ := d.Net["plan"].(obj)
for _, iv := range toList(plan["interfaces"]) {
i, _ := iv.(obj)
what := str(i["mac"])
switch str(i["type"]) {
case "bond":
var ms []string
for _, x := range toList(i["members"]) {
ms = append(ms, str(x))
}
what = "bond " + str(i["bond_mode"]) + " of " + strings.Join(ms, ",")
case "vlan":
what = "vlan " + str(i["vid"]) + " on " + str(i["parent"])
}
for _, lv := range toList(i["links"]) {
l, _ := lv.(obj)
ip := str(l["ip"])
if ip == "" && str(l["mode"]) == "auto" {
ip = sDim.Render("allocated at deploy")
} else if ip != "" {
ip = fmt.Sprintf("%s/%d", ip, int(num(l["prefix"])))
}
gw := ""
if l["default_gateway"] == true {
gw = sDim.Render(" default via " + str(l["gateway"]))
}
w(" %-12s %-28s %-8s %-18s %s%s", str(i["name"]), truncateWidth(what, 28), str(l["mode"]), orDash(str(l["subnet"])), ip, gw)
what = ""
}
}
for _, x := range toList(plan["warnings"]) {
w(" %s", sWarn.Render("! "+str(x)))
}
if d.Net["modeled"] == false {
w(" %s", sDim.Render("no subnets defined: deployed machines use DHCP"))
}
}
job := s.Job
if job == nil && len(d.Jobs) > 0 {
job = d.Jobs[0]
@@ -2013,3 +2104,53 @@ func tuiDump(args []string) error {
fmt.Println(out)
return nil
}
// networkBody: rack controllers, then the subnets (the selectable rows), then layouts.
// It returns the body and the line of the first subnet row.
func (m tuiModel) networkBody(pw int) (string, int) {
var b strings.Builder
b.WriteString(" " + sTitle.Render("Rack controllers") + "\n")
var rows [][]string
for _, c := range m.d.ctls {
st := sOK.Render("● online")
if !c.Online {
st = sBad.Render("● offline")
}
var ifs []string
for _, i := range c.Interfaces {
ifs = append(ifs, i.Name+" "+strings.Join(i.Addrs, ","))
}
rows = append(rows, []string{c.Name, c.Kind, st, c.URL, fmt.Sprint(c.Machines), agoP(c.LastSeen), sDim.Render(strings.Join(ifs, " · "))})
}
b.WriteString(table([]string{"Controller", "Kind", "State", "URL", "Machines", "Seen", "Interfaces"}, rows, -1, pw-2, false))
b.WriteString("\n " + sTitle.Render("Subnets") + "\n")
head := strings.Count(b.String(), "\n") + 1
rows = nil
for _, s := range m.d.subnets {
dhcp := sDim.Render("off")
if s.DHCP {
dhcp = sOK.Render("on") + " " + s.PrimaryRack
if s.SecondaryRack != "" {
dhcp += "+" + s.SecondaryRack
}
}
var rs []string
for _, r := range s.Ranges {
rs = append(rs, r.Kind[:1]+":"+lastOctets(r.Start)+"-"+lastOctets(r.End))
}
pct := 0.0
if s.Size > 0 {
pct = float64(s.Used) * 100 / float64(s.Size)
}
rows = append(rows, []string{s.CIDR, s.Purpose, fmt.Sprintf("%s/%d", orDash(s.Fabric), s.VID), dhcp, orDash(s.Gateway),
fmt.Sprintf("%s %d/%d", bar(pct, 8), s.Used, s.Size), strings.Join(rs, " ")})
}
b.WriteString(table([]string{"Subnet", "Purpose", "Fabric/VLAN", "DHCP", "Gateway", "Used", "Ranges"}, rows, m.row, pw-2, m.inPanel))
b.WriteString("\n " + sTitle.Render("Interface layouts") + "\n")
rows = nil
for _, l := range m.d.layouts {
rows = append(rows, []string{l.Name, fmt.Sprint(l.Machines), layoutSummary(l), l.Comment})
}
b.WriteString(table([]string{"Layout", "Machines", "Interfaces", "Comment"}, rows, -1, pw-2, false))
return b.String(), head
}
+146 -8
View File
@@ -14,7 +14,7 @@ let token = '';
try { token = localStorage.getItem('rollcall.token') || ''; } catch (e) { /* storage unavailable */ }
// ?token=… on the URL signs in once and is removed from the address bar (for bookmarks/kiosks)
{ const u = new URL(location.href); const t = u.searchParams.get('token'); if (t) { token = t; try { localStorage.setItem('rollcall.token', t); } catch (e) { /* ignore */ } u.searchParams.delete('token'); history.replaceState(null, '', u.pathname + u.search + u.hash); } }
const S = { machines: [], tags: [], profiles: [], os: [], jobs: [], events: [], racks: [], sel: new Set(), q: '', states: new Set(), tagf: new Set(), loc: { building: '', room: '', rack: '' }, page: 0, per: 50, sort: 'name', dir: 1, loaded: false };
const S = { machines: [], tags: [], profiles: [], os: [], jobs: [], events: [], racks: [], subnets: [], controllers: [], layouts: [], fabrics: [], sel: new Set(), q: '', states: new Set(), tagf: new Set(), loc: { building: '', room: '', rack: '' }, page: 0, per: 50, sort: 'name', dir: 1, loaded: false };
const locStr = l => { if (!l) return ''; const p = [l.building, l.room, l.rack].filter(Boolean).join(' / '); return p + (l.unit ? ` U${l.unit}` : ''); };
let timer = null;
@@ -59,9 +59,12 @@ function dialog(title, fields, note, okLabel = 'OK') {
async function refresh(full) {
if (!token) return askToken();
try {
const [machines, tags, profiles, os, racks] = await Promise.all([api('GET', '/api/v1/machines'), api('GET', '/api/v1/tags'), api('GET', '/api/v1/profiles'), api('GET', '/api/v1/os'), api('GET', '/api/v1/racks')]);
Object.assign(S, { machines, tags, profiles, os, racks, loaded: true });
const [machines, tags, profiles, os, racks, subnets, controllers, layouts, fabrics] = await Promise.all([api('GET', '/api/v1/machines'), api('GET', '/api/v1/tags'), api('GET', '/api/v1/profiles'), api('GET', '/api/v1/os'), api('GET', '/api/v1/racks'),
api('GET', '/api/v1/subnets'), api('GET', '/api/v1/controllers'), api('GET', '/api/v1/layouts'), api('GET', '/api/v1/fabrics')]);
Object.assign(S, { machines, tags, profiles, os, racks, subnets, controllers, layouts, fabrics, loaded: true });
$('#n-racks').textContent = racks.length || '';
const off = controllers.filter(c => !c.online).length;
$('#n-net').textContent = off ? `${off} offline` : (subnets.length || '');
const page = route().page;
if (full || page === 'overview' || page === 'jobs') S.jobs = await api('GET', '/api/v1/jobs');
if (full || page === 'overview') S.events = await api('GET', '/api/v1/events');
@@ -107,6 +110,7 @@ function render() {
case 'machine': if (!detailCache[r.id]) { main.innerHTML = '<p class="muted">loading…</p>'; loadDetail(r.id); } else { main.innerHTML = pageMachine(detailCache[r.id], r.tab); wireMachine(detailCache[r.id]); } break;
case 'jobs': main.innerHTML = pageJobs(); wireJobs(); break;
case 'racks': main.innerHTML = pageRacks(); wireRacks(); break;
case 'network': main.innerHTML = pageNetwork(); wireNetwork(); break;
case 'tags': main.innerHTML = pageTags(); wireTags(); break;
case 'profiles': main.innerHTML = pageProfiles(); wireProfiles(); break;
case 'images': main.innerHTML = pageImages(); break;
@@ -133,6 +137,7 @@ function pageOverview() {
<div class="grid">
<div class="card wide"><h3>Active jobs</h3>${jobsTable(active, true)}</div>
<div class="card"><h3>Attention</h3>${kv([
['Rack controllers offline', (() => { const off = S.controllers.filter(c => !c.online); return off.length ? `<span class="bad">${off.map(c => esc(c.name)).join(', ')}</span> <a href="#/network">show</a>` : `<span class="ok">none</span> <span class="muted">(${S.controllers.length})</span>`; })()],
['BMC not configured', bmcBad ? `<span class="bad">${bmcBad}</span> <a href="#/machines">show</a>` : '<span class="ok">none</span>'],
['Stateless nodes silent &gt; 15 min', stale ? `<span class="warn">${stale}</span>` : '<span class="ok">none</span>'],
['Failed jobs (last 100)', (() => { const n = S.jobs.filter(j => j.state === 'failed').length; return n ? `<span class="bad">${n}</span> <a href="#/jobs">show</a>` : '<span class="ok">none</span>'; })()],
@@ -223,7 +228,7 @@ function bulkBar() {
<button class="btn" id="bassign">Assign profile…</button>
<button class="btn" id="btag">Tag…</button>
<button class="btn" id="bloc">Set location…</button>
${menuBtn('More', [['recommission', 'Re-commission (next boot)'], ['recommission-now', 'Re-commission now (BMC)'], ['release', 'Release (wipe disk)…'], ['unassign', 'Unassign profile'], ['hr'], ['bootdev-pxe', 'Boot device: PXE'], ['bootdev-disk', 'Boot device: disk']], 'bmore')}
${menuBtn('More', [['layout', 'Network layout…'], ['recommission', 'Re-commission (next boot)'], ['recommission-now', 'Re-commission now (BMC)'], ['release', 'Release (wipe disk)…'], ['unassign', 'Unassign profile'], ['hr'], ['bootdev-pxe', 'Boot device: PXE'], ['bootdev-disk', 'Boot device: disk']], 'bmore')}
<button class="btn ghost" id="bclear">Clear selection</button></div>`;
}
function menuBtn(label, items, id) { return `<span class="menu" id="${id}"><button class="btn" type="button" data-menu>${esc(label)} ▾</button><div class="list">${items.map(([v, l]) => v === 'hr' ? '<hr>' : `<button data-item="${esc(v)}"${/release|delete/.test(v) ? ' class="danger"' : ''}>${esc(l)}</button>`).join('')}</div></span>`; }
@@ -341,6 +346,12 @@ const act = {
case 'unassign': if (confirm(`Remove the stateless profile from ${n} machine(s)? Names and IPs are released.`)) await each(ids, 'unassign', async id => (await api('POST', `/api/v1/machines/${id}/unassign`)).state); break;
case 'bootdev-pxe': await act.bootdev(ids, 'pxe'); break;
case 'bootdev-disk': await act.bootdev(ids, 'disk'); break;
case 'layout': {
const one = ids.length === 1 ? S.machines.find(m => m.id === ids[0]) : null;
const v = await dialog(`Network layout of ${one ? nameOf(one) : n + ' machines'}`, [{ name: 'layout', label: 'Layout', type: 'select', value: one ? one.net_layout : '', options: [['', '(from tags, else the boot NIC on its subnet)'], ...S.layouts.map(l => [l.name, `${l.name} · ${l.comment || layoutSummary(l)}`])] }], 'Applied at the next deployment; addresses come from the subnets the layout names.', 'Save');
if (v) await each(ids, 'network layout', async id => { const r = await api('POST', `/api/v1/machines/${id}/network`, { layout: v.layout }); delete netCache[id]; return r.layout; });
break;
}
case 'cancel-job': await each(ids, 'cancel job', async id => `job ${(await api('POST', `/api/v1/machines/${id}/jobs/cancel`)).job} cancelled`); break;
case 'delete': if (confirm(`Delete ${n} machine record(s)? They are commissioned again as new machines on their next PXE boot.`)) { await each(ids, 'delete', async id => { await api('DELETE', `/api/v1/machines/${id}`); delete detailCache[id]; return 'deleted'; }); S.sel.clear(); nav('machines'); } break;
}
@@ -370,6 +381,7 @@ function pageMachine(d, tab) {
${card('Role', kv([
['State', pill(s.state)],
['Node name', esc(s.node_name)], ['Management IP', s.mgmt_ip ? `<span class="mono">${esc(s.mgmt_ip)}</span>` : ''], ['IB IP', s.ib_ip ? `<span class="mono">${esc(s.ib_ip)}</span>` : ''],
['Rack controller', s.rack_controller ? `${esc(s.rack_controller)} <a href="#/machine/${esc(s.id)}/network">network</a>` : ''],
['Stateless profile', s.profile ? `${esc(s.profile)} · booted ${esc(s.booted_at ? when(s.booted_at) : 'not yet')}` : ''],
['OS on disk', s.os ? `${esc(s.os)} <span class="muted mono">${esc(s.os_version || '')}</span> · deployed ${esc(when(s.deployed_at))}` : ''],
['Phoned home', s.deployed_at ? (s.booted_at ? `<span class="ok">${esc(when(s.booted_at))}</span>` : '<span class="muted">not yet</span>') : ''],
@@ -395,7 +407,7 @@ function pageMachine(d, tab) {
} else if (tab === 'network') {
const hcas = (inv.infiniband || {}).hcas || [];
const lldpRows = []; for (const [ifn, ents] of Object.entries(inv.lldp || {})) for (const e of (Array.isArray(ents) ? ents : [ents])) { const ch = e.chassis || {}; const name = Object.keys(ch)[0] || '?'; const c = ch[name] || {}; lldpRows.push([esc(ifn), esc(name), esc((c.descr || '').slice(0, 60)), esc((e.port || {}).id && (e.port.id.value || e.port.id)), esc((e.port || {}).descr), esc((e.vlan || {})['vlan-id'] || (e.vlan || {}).value)]); }
body = `<div class="grid">
body = netPlanCard(s.id) + `<div class="grid">
${card(`NICs · ${(inv.network || []).length}`, table(['If', 'MAC', 'Driver', 'FW', 'Speed', 'Link', 'PCI', 'NUMA', 'VFs', 'IPv4'], (inv.network || []).map(n => [esc(n.name), `<span class="mono">${esc(n.mac)}</span>`, esc(n.driver), esc(n.firmware_version), esc(n.speed), n.link === true ? '<span class="ok">up</span>' : n.link === false ? '<span class="muted">down</span>' : '', `<span class="mono">${esc(n.pci)}</span>`, esc(n.numa_node), esc(n.sriov_totalvfs), esc((n.ipv4 || []).join(' '))])), true)}
${card(`InfiniBand · ${hcas.length} HCAs`, hcas.length ? table(['HCA', 'Port', 'Layer', 'State', 'Rate', 'LID', 'Node GUID', 'FW', 'Board'], hcas.flatMap(h => (h.ports || []).map(p => [esc(h.name), esc(p.port), esc(p.link_layer), esc(p.state), esc(p.rate), esc(p.lid), `<span class="mono">${esc(h.node_guid)}</span>`, esc(h.fw_ver), esc(h.board_id)]))) : '<span class="muted">none</span>', true)}
${card('LLDP neighbours (switch ports)', table(['If', 'Switch', 'Descr', 'Port', 'Port descr', 'VLAN'], lldpRows), true)}
@@ -416,7 +428,7 @@ function pageMachine(d, tab) {
${menuBtn('Boot device', [['pxe', 'PXE'], ['disk', 'Disk'], ['bios', 'BIOS setup'], ['cdrom', 'CD-ROM']], 'dboot')}
<button class="btn primary" id="ddeploy">Deploy OS…</button>
<button class="btn" id="dassign">Assign profile…</button>
${menuBtn('More', [['recommission', 'Re-commission (next boot)'], ['recommission-now', 'Re-commission now (BMC)'], ['release', 'Release (wipe disk)…'], ['unassign', 'Unassign profile'], ...(hasJob ? [['cancel-job', 'Cancel active job']] : []), ['hr'], ['delete', 'Delete record']], 'dmore')}
${menuBtn('More', [['layout', 'Network layout…'], ['recommission', 'Re-commission (next boot)'], ['recommission-now', 'Re-commission now (BMC)'], ['release', 'Release (wipe disk)…'], ['unassign', 'Unassign profile'], ...(hasJob ? [['cancel-job', 'Cancel active job']] : []), ['hr'], ['delete', 'Delete record']], 'dmore')}
</div></div>
<div class="d-meta"><span>${esc([sys['system-manufacturer'], sys['system-product-name']].filter(Boolean).join(' '))}</span><span>S/N <span class="mono">${esc(s.serial || '')}</span></span><span>${hwSummary(m)}</span>${s.mgmt_ip ? `<span class="mono">${esc(s.mgmt_ip)}</span>` : ''}<span>seen ${ago(s.last_seen)} ago</span></div>
<div class="tabs">${tabs.map(t => `<a href="#/machine/${esc(s.id)}/${t}" class="${t === tab ? 'active' : ''}">${t === 'bmc' ? 'BMC' : t[0].toUpperCase() + t.slice(1)}</a>`).join('')}</div>
@@ -424,6 +436,7 @@ function pageMachine(d, tab) {
}
function wireMachine(d) {
const id = d.summary.id, ids = [id];
if (route().tab === 'network') wireNetPlan(id);
wireMenus();
$$('#dpower [data-item]').forEach(b => b.onclick = () => act.power(ids, b.dataset.item));
$$('#dboot [data-item]').forEach(b => b.onclick = () => act.bootdev(ids, b.dataset.item));
@@ -489,7 +502,7 @@ function wireRacks() {
// ---------------------------------------------------------------- tags
function pageTags() {
return `<div class="page-head"><h1>Tags</h1><span class="sub">single out machines for special treatment: auto rules from the inventory, cloud-init fragments at deployment, overlays for stateless nodes</span><div class="right"><button class="btn primary" id="newtag">New tag…</button></div></div>
${table(['Tag', 'Machines', 'Match rule', 'Cloud-config', 'Overlays', 'Comment', ''], S.tags.map(t => [`<span class="pill tag">${esc(t.name)}</span>`, `<a href="#/machines" data-filtertag="${esc(t.name)}">${t.machines || 0}</a>`, `<span class="mono">${esc(t.match || '')}</span>`, t.cloud_config ? `<span class="muted">${t.cloud_config.split('\n').length} lines</span>` : '', esc((t.overlays || []).join(', ')), esc(t.comment || ''), `<button class="btn sm" data-edit="${esc(t.name)}">edit</button> <button class="btn sm danger" data-rm="${esc(t.name)}">delete</button>`]))}
${table(['Tag', 'Machines', 'Match rule', 'Cloud-config', 'Overlays', 'Layout', 'Comment', ''], S.tags.map(t => [`<span class="pill tag">${esc(t.name)}</span>`, `<a href="#/machines" data-filtertag="${esc(t.name)}">${t.machines || 0}</a>`, `<span class="mono">${esc(t.match || '')}</span>`, t.cloud_config ? `<span class="muted">${t.cloud_config.split('\n').length} lines</span>` : '', esc((t.overlays || []).join(', ')), esc(t.net_layout || ''), esc(t.comment || ''), `<button class="btn sm" data-edit="${esc(t.name)}">edit</button> <button class="btn sm danger" data-rm="${esc(t.name)}">delete</button>`]))}
<p class="muted" style="margin-top:10px">Match rules are Go templates over <code>.Node</code> (GPUs, IBGUIDs, CPUs, Sockets, MemMB, Disks, NVMe, PCIVendors, Manufacturer, Product, OS, Tags), e.g. <code>{{ gt .Node.GPUs 0 }}</code>, <code>{{ has .Node.PCIVendors "15b3" }}</code>, <code>{{ matches "(?i)PowerEdge R7" .Node.Product }}</code>. Rules run at commissioning and whenever a tag is saved.</p>`;
}
async function editTag(t) {
@@ -498,10 +511,11 @@ async function editTag(t) {
{ name: 'comment', label: 'Comment', value: t ? t.comment : '' },
{ name: 'match', label: 'Match rule', value: t ? t.match : '', placeholder: '{{ gt .Node.GPUs 0 }} (empty = manual only)' },
{ name: 'overlays', label: 'Overlays', value: t ? (t.overlays || []).join(',') : '', placeholder: 'gpu-slurm', help: 'extra overlays for stateless nodes with this tag' },
{ name: 'net_layout', label: 'Network layout', type: 'select', value: t ? t.net_layout : '', options: [['', '(none)'], ...S.layouts.map(l => [l.name, l.name])], help: 'interfaces of deployed machines with this tag' },
{ name: 'cloud_config', label: 'Cloud-config', type: 'textarea', value: t ? t.cloud_config : '', placeholder: '#cloud-config\npackages: [nvidia-driver]\nruncmd:\n - [ sh, -c, "..." ]' },
], 'The cloud-config fragment is merged into the user-data of every deployment of a tagged machine. A fragment starting with #! runs as a script.', 'Save');
if (!v) return;
try { await api('POST', '/api/v1/tags', { name: v.name.trim(), comment: v.comment, match: v.match, overlays: v.overlays.split(',').map(s => s.trim()).filter(Boolean), cloud_config: v.cloud_config }); toast(`tag ${v.name} saved; auto tags re-evaluated`); refresh(true); } catch (e) { toast(e.message, true); }
try { await api('POST', '/api/v1/tags', { name: v.name.trim(), comment: v.comment, match: v.match, overlays: v.overlays.split(',').map(s => s.trim()).filter(Boolean), cloud_config: v.cloud_config, net_layout: v.net_layout }); toast(`tag ${v.name} saved; auto tags re-evaluated`); refresh(true); } catch (e) { toast(e.message, true); }
}
function wireTags() {
$('#newtag').onclick = () => editTag(null);
@@ -545,3 +559,127 @@ function pageImages() {
// ---------------------------------------------------------------- go
schedule();
refresh(true);
// ---------------------------------------------------------------- network: a machine's plan
const netCache = {};
function layoutSummary(l) {
return (l.interfaces || []).map(i => i.type === 'bond' ? `${i.name || 'bond0'}(${(i.members || []).join('+')})` : i.type === 'vlan' ? `${i.parent}.${i.vid}` : (i.select || 'boot')).join(' ');
}
function netPlanCard(id) {
const n = netCache[id];
if (!n) { api('GET', `/api/v1/machines/${encodeURIComponent(id)}/network`).then(r => { netCache[id] = r; if (route().page === 'machine' && route().id === id) render(); }).catch(e => toast(e.message, true)); return '<div class="card wide" style="margin-bottom:12px"><h3>Network configuration</h3><span class="muted">loading…</span></div>'; }
const p = n.plan || {};
const rows = [];
for (const i of (p.interfaces || [])) {
const what = i.type === 'bond' ? `bond ${esc(i.bond_mode)} of ${esc((i.members || []).join(', '))}` : i.type === 'vlan' ? `vlan ${i.vid} on ${esc(i.parent)}` : `<span class="mono">${esc(i.mac || '')}</span>${i.ib ? ' IB' : ''}`;
const links = i.links && i.links.length ? i.links : [{}];
links.forEach((l, k) => rows.push([k ? '' : `<b>${esc(i.name)}</b>`, k ? '' : what, k ? '' : esc(i.mtu || ''), `<span class="mono">${esc(l.subnet || '')}</span>`, esc(l.mode || '—'),
l.ip ? `<span class="mono">${esc(l.ip)}/${l.prefix}</span>` : (l.mode === 'auto' ? '<span class="muted">allocated at deploy</span>' : ''), l.default_gateway ? `default via <span class="mono">${esc(l.gateway)}</span>` : '']));
}
const pins = (n.addresses || []).map(a => `<span class="pill tag">${esc(a.ip)} ${esc(a.kind)}${a.iface ? ' · ' + esc(a.iface) : ''}${a.kind === 'static' ? ` <a href="#" data-unpin="${esc(a.ip)}" title="unpin">×</a>` : ''}</span>`).join(' ');
return `<div class="card wide" style="margin-bottom:12px"><h3>Network configuration</h3>
${n.modeled ? '' : '<p class="warn">No subnets are defined: deployed machines use DHCP. Define them on the Network page.</p>'}
${kv([['Layout', `${esc(p.layout)} <button class="btn sm" id="n-layout">change</button>`], ['Rack controller', `${esc(n.rack_controller)} <button class="btn sm" id="n-rack">change</button>`], ['Addresses', (pins || '<span class="muted">none</span>') + ' <button class="btn sm" id="n-pin">pin an address…</button>']])}
${table(['Interface', 'What', 'MTU', 'Subnet', 'Mode', 'Address', 'Route'], rows)}
${(p.warnings || []).length ? `<p class="bad">${p.warnings.map(esc).join('<br>')}</p>` : ''}
<details><summary>network-config (cloud-init v2, written into the image at deployment)</summary><pre class="json">${esc(JSON.stringify(n.network_config, null, 2))}</pre></details></div>`;
}
function wireNetPlan(id) {
const post = async (body, label) => { try { const r = await api('POST', `/api/v1/machines/${id}/network`, body); netCache[id] = r; toast(label); render(); } catch (e) { toast(e.message, true); } };
const l = $('#n-layout'); if (l) l.onclick = () => act.more([id], 'layout');
const rk = $('#n-rack'); if (rk) rk.onclick = async () => {
const v = await dialog('Rack controller', [{ name: 'rack', label: 'Serves this machine', type: 'select', value: (netCache[id] || {}).rack_controller, options: S.controllers.map(c => [c.name, `${c.name} · ${c.url}${c.online ? '' : ' (offline)'}`]) }], 'Normally set by where the machine last PXE-booted; change it when a machine moved to another segment.', 'Save');
if (v) await post({ rack_controller: v.rack }, 'rack controller ' + v.rack);
};
const pin = $('#n-pin'); if (pin) pin.onclick = async () => {
const ifs = (((netCache[id] || {}).plan || {}).interfaces || []).map(i => [i.name, i.name]);
const v = await dialog('Pin an address', [{ name: 'iface', label: 'Interface', type: 'select', options: ifs }, { name: 'ip', label: 'Address', placeholder: '10.0.0.60' }], 'A pinned address is used instead of one from the subnet\'s pool.', 'Pin');
if (v && v.ip) await post({ static: [{ iface: v.iface, ip: v.ip.trim() }] }, 'pinned ' + v.ip);
};
$$('[data-unpin]').forEach(a => a.onclick = ev => { ev.preventDefault(); post({ unstatic: [a.dataset.unpin] }, 'unpinned ' + a.dataset.unpin); });
}
// ---------------------------------------------------------------- network page
function pageNetwork() {
const ctl = table(['Controller', 'Kind', 'State', 'URL', 'Machines', 'Last seen', 'Interfaces', ''], S.controllers.map(c => [`<b>${esc(c.name)}</b>`, esc(c.kind), c.online ? '<span class="dot ok"></span>online' : '<span class="dot bad"></span><span class="bad">offline</span>', `<span class="mono">${esc(c.url)}</span>`, c.machines,
`<span class="muted" title="${esc(when(c.last_seen))}">${ago(c.last_seen)}</span>`, `<span class="mono muted">${esc((c.interfaces || []).map(i => i.name + ' ' + (i.addrs || []).join(',')).join(' · '))}</span>`, `<button class="btn sm" data-dhcp="${esc(c.name)}">dnsmasq</button>${c.kind === 'remote' ? ` <button class="btn sm danger" data-rmctl="${esc(c.name)}">remove</button>` : ''}`]));
const sub = table(['Subnet', 'Purpose', 'Fabric / VLAN', 'DHCP', 'Gateway', 'DNS', 'Usage', 'Ranges', ''], S.subnets.map(s => {
const pct = s.size ? Math.round(s.used * 100 / s.size) : 0;
const ranges = (s.ranges || []).map(r => `<span class="pill tag" title="${esc(r.comment || '')}">${esc(r.kind)} ${esc(r.start)}–${esc(r.end.split('.').pop())} <a href="#" data-rmrange="${r.id}" title="remove">×</a></span>`).join(' ');
return [`<b class="mono">${esc(s.cidr)}</b>${s.name ? `<br><span class="muted">${esc(s.name)}</span>` : ''}`, esc(s.purpose), `${esc(s.fabric || '—')} / ${s.vid || 'untagged'}${s.mtu && s.mtu !== 1500 ? ` · mtu ${s.mtu}` : ''}`,
s.dhcp ? `<span class="ok">on</span> · ${esc(s.primary_rack)}${s.secondary_rack ? ' + ' + esc(s.secondary_rack) : ''}` : '<span class="muted">off</span>', `<span class="mono">${esc(s.gateway || '—')}</span>`,
`<span class="mono">${esc((s.dns || []).join(', '))}</span>${s.domain ? `<br><span class="muted">${esc(s.domain)}</span>` : ''}`, `<div class="bar" title="${s.used}/${s.size}"><span style="width:${pct}%"></span></div><span class="muted">${s.used} / ${s.size}</span>`,
ranges + ` <button class="btn sm" data-addrange="${esc(s.cidr)}">+ range</button>`, `<button class="btn sm" data-ips="${esc(s.cidr)}">addresses</button> <button class="btn sm" data-editsubnet="${esc(s.cidr)}">edit</button> <button class="btn sm danger" data-rmsubnet="${esc(s.cidr)}">delete</button>`];
}));
const lay = table(['Layout', 'Machines', 'Interfaces', 'Comment', ''], S.layouts.map(l => [`<b>${esc(l.name)}</b>`, l.machines || 0, `<span class="mono">${esc(layoutSummary(l))}</span>`, esc(l.comment || ''), `<button class="btn sm" data-editlayout="${esc(l.name)}">edit</button> <button class="btn sm danger" data-rmlayout="${esc(l.name)}">delete</button>`]));
return `<div class="page-head"><h1>Network</h1><span class="sub">fabrics, VLANs and subnets (MAAS-style); rack controllers serve DHCP/PXE on their VLANs</span>
<div class="right"><button class="btn primary" id="new-subnet">New subnet…</button><button class="btn" id="new-layout">New layout…</button></div></div>
<div class="card wide" style="margin-bottom:12px"><h3>Rack controllers</h3><div class="tw">${ctl}</div><p class="muted" style="margin:8px 0 0">Add one next to a segment: <code>deploy/install-rack.sh</code> with <code>/etc/rollcall/rack.json</code> (region URL, rack_secret, its own URL). It registers itself.</p></div>
<div class="card wide" style="margin-bottom:12px"><h3>Subnets</h3><div class="tw">${sub}</div></div>
<div class="card wide"><h3>Interface layouts</h3>${lay}<p class="muted" style="margin:8px 0 0">A layout applies to a machine directly (More → Network layout…) or through a tag (Tags → layout). Without one, the boot NIC gets an address on its subnet.</p></div>`;
}
const layoutExample = JSON.stringify({ interfaces: [
{ type: 'physical', select: 'boot', links: [{ subnet: 'boot', mode: 'auto', default_gateway: true }] },
{ type: 'bond', name: 'bond0', members: ['fastest:2'], bond_mode: '802.3ad', mtu: 9000, links: [{ mode: 'link_up' }] },
{ type: 'vlan', parent: 'bond0', vid: 200, links: [{ subnet: '10.20.0.0/24', mode: 'auto' }] },
{ type: 'physical', select: 'ib:0', links: [{ subnet: '10.30.0.0/24', mode: 'auto' }] }] }, null, 2);
async function editSubnet(s) {
const fabrics = S.fabrics.map(f => [f.name, f.name]);
const v = await dialog(s ? `Subnet ${s.cidr}` : 'New subnet', [
{ name: 'cidr', label: 'CIDR', value: s ? s.cidr : '', placeholder: '10.50.0.0/24' }, { name: 'name', label: 'Name', value: s ? s.name : '' },
{ name: 'fabric', label: 'Fabric', value: s ? s.fabric : (S.fabrics[0] || {}).name, placeholder: 'fabric-0', help: 'a new name creates the fabric', list: 'dl-fabrics' },
{ name: 'vid', label: 'VLAN id', value: s ? String(s.vid) : '0', help: '0 = untagged' },
{ name: 'purpose', label: 'Purpose', type: 'select', value: s ? s.purpose : 'provisioning', options: ['provisioning', 'mgmt', 'bmc', 'storage', 'ib', 'other'].map(x => [x, x]) },
{ name: 'gateway', label: 'Gateway', value: s ? s.gateway : '' }, { name: 'dns', label: 'DNS servers', value: s ? (s.dns || []).join(',') : '', placeholder: 'comma separated' },
{ name: 'domain', label: 'Domain', value: s ? s.domain : '' }, { name: 'mtu', label: 'VLAN MTU', value: s ? String(s.mtu || 1500) : '1500' },
{ name: 'dhcp', label: 'Serve DHCP/PXE', type: 'checkbox', value: s ? s.dhcp : false },
{ name: 'rack', label: 'Rack controller', type: 'select', value: s ? s.primary_rack : '', options: S.controllers.map(c => [c.name, `${c.name}${c.online ? '' : ' (offline)'}`]) },
], 'Ranges are added afterwards: dynamic (PXE pool), reserved (never handed out), static (where machines get addresses).', 'Save');
if (!v) return;
try {
const fabric = v.fabric.trim();
if (fabric && !S.fabrics.some(f => f.name === fabric)) await api('POST', '/api/v1/fabrics', { name: fabric });
const vid = Number(v.vid) || 0;
await api('POST', '/api/v1/subnets', { cidr: v.cidr.trim(), name: v.name, fabric, vid, purpose: v.purpose, gateway: v.gateway.trim(), dns: v.dns.split(',').map(x => x.trim()).filter(Boolean), domain: v.domain.trim() });
if (fabric) await api('POST', '/api/v1/vlans', { fabric, vid, mtu: Number(v.mtu) || 1500, dhcp: v.dhcp, primary_rack: v.dhcp ? v.rack : '' });
toast(`subnet ${v.cidr} saved`); refresh(true);
} catch (e) { toast(e.message, true); }
}
async function editLayout(l) {
const v = await dialog(l ? `Layout ${l.name}` : 'New layout', [{ name: 'name', label: 'Name', value: l ? l.name : '', placeholder: 'gpu-bond' }, { name: 'comment', label: 'Comment', value: l ? l.comment : '' },
{ name: 'spec', label: 'Interfaces (JSON)', type: 'textarea', rows: 14, value: l ? JSON.stringify({ interfaces: l.interfaces }, null, 2) : layoutExample }],
'Selectors: boot, name:X, mac:X, fastest:N, ib:N, all. Link modes: auto (from the subnet), static (with ip), dhcp, link_up. Subnets by CIDR, name, or "boot".', 'Save');
if (!v) return;
let spec; try { spec = JSON.parse(v.spec); } catch (e) { return toast('interfaces: invalid JSON', true); }
try { await api('POST', '/api/v1/layouts', { name: v.name.trim(), comment: v.comment, interfaces: spec.interfaces || spec }); for (const k in netCache) delete netCache[k]; toast(`layout ${v.name} saved`); refresh(true); } catch (e) { toast(e.message, true); }
}
function wireNetwork() {
let dl = $('#dl-fabrics'); if (!dl) { dl = document.createElement('datalist'); dl.id = 'dl-fabrics'; document.body.appendChild(dl); }
dl.innerHTML = S.fabrics.map(f => `<option value="${esc(f.name)}">`).join('');
$('#new-subnet').onclick = () => editSubnet(null);
$('#new-layout').onclick = () => editLayout(null);
const del = async (path, label) => { try { await api('DELETE', path); toast(label); refresh(true); } catch (e) { toast(e.message, true); } };
$$('[data-editsubnet]').forEach(b => b.onclick = () => editSubnet(S.subnets.find(s => s.cidr === b.dataset.editsubnet)));
$$('[data-rmsubnet]').forEach(b => b.onclick = () => { if (confirm(`Delete subnet ${b.dataset.rmsubnet}?`)) del('/api/v1/subnets/' + b.dataset.rmsubnet, 'deleted ' + b.dataset.rmsubnet); });
$$('[data-rmrange]').forEach(a => a.onclick = ev => { ev.preventDefault(); if (confirm('Remove this range?')) del('/api/v1/ranges/' + a.dataset.rmrange, 'range removed'); });
$$('[data-addrange]').forEach(b => b.onclick = async () => {
const v = await dialog(`Range on ${b.dataset.addrange}`, [{ name: 'kind', label: 'Kind', type: 'select', options: [['static', 'static: machines get addresses here'], ['dynamic', 'dynamic: DHCP pool for PXE and commissioning'], ['reserved', 'reserved: never handed out']] }, { name: 'start', label: 'First address' }, { name: 'end', label: 'Last address' }, { name: 'comment', label: 'Comment' }], '', 'Add');
if (v) { try { await api('POST', '/api/v1/ranges', { subnet: b.dataset.addrange, kind: v.kind, start: v.start.trim(), end: v.end.trim(), comment: v.comment }); toast('range added'); refresh(true); } catch (e) { toast(e.message, true); } }
});
$$('[data-ips]').forEach(b => b.onclick = async () => {
try {
const ips = await api('GET', '/api/v1/ips?subnet=' + encodeURIComponent(b.dataset.ips));
const lines = ips.map(x => `${x.ip.padEnd(15)} ${x.kind.padEnd(8)} ${(S.machines.find(m => m.id === x.machine_id) || {}).node_name || x.machine_id || ''} ${x.iface || ''} ${x.hostname || ''}`);
await dialog(`Addresses in ${b.dataset.ips}`, [{ name: 'list', label: `${ips.length} in use`, type: 'textarea', rows: Math.min(18, Math.max(4, ips.length + 1)), value: lines.join('\n') || 'none' }], '', 'Close');
} catch (e) { toast(e.message, true); }
});
$$('[data-dhcp]').forEach(b => b.onclick = async () => {
try {
const r = await fetch('/api/v1/controllers/' + encodeURIComponent(b.dataset.dhcp) + '/dhcp', { headers: { Authorization: 'Bearer ' + token } });
await dialog(`dnsmasq on ${b.dataset.dhcp}`, [{ name: 'conf', label: 'generated', type: 'textarea', rows: 18, value: await r.text() }], '', 'Close');
} catch (e) { toast(e.message, true); }
});
$$('[data-rmctl]').forEach(b => b.onclick = () => { if (confirm(`Remove rack controller ${b.dataset.rmctl}? It registers again if it is still running.`)) del('/api/v1/controllers/' + b.dataset.rmctl, 'removed ' + b.dataset.rmctl); });
$$('[data-editlayout]').forEach(b => b.onclick = () => editLayout(S.layouts.find(l => l.name === b.dataset.editlayout)));
$$('[data-rmlayout]').forEach(b => b.onclick = () => { if (confirm(`Delete layout ${b.dataset.rmlayout}?`)) del('/api/v1/layouts/' + b.dataset.rmlayout, 'deleted ' + b.dataset.rmlayout); });
}
+1
View File
@@ -13,6 +13,7 @@
<a href="#/overview" data-nav="overview">Overview</a>
<a href="#/machines" data-nav="machines">Machines <span class="count" id="n-machines"></span></a>
<a href="#/racks" data-nav="racks">Racks <span class="count" id="n-racks"></span></a>
<a href="#/network" data-nav="network">Network <span class="count" id="n-net"></span></a>
<a href="#/jobs" data-nav="jobs">Jobs <span class="count" id="n-jobs"></span></a>
<a href="#/tags" data-nav="tags">Tags</a>
<a href="#/profiles" data-nav="profiles">Profiles</a>