Day-0 install without a network plan; rollcall net discover
CI / agent (push) Successful in 2s
CI / site (push) Successful in 2s
CI / go (push) Successful in 36s

bootstrap.sh no longer needs the provisioning network: it installs and
starts the region on the management address; --iface later models the
subnet, serves DHCP and moves server_url. rollcall net discover shows what
each interface sees (LLDP, DHCP servers via a parallel probe, neighbours)
and suggests the next step. commission_append follows server_url. Doctor
treats a host that serves no DHCP yet as day 0. Fixes found on a clean VM:
os-release clobbered the rack name, install.sh not executable, slow DHCP
offers missed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-28 18:28:37 +01:00
1 parent 01cc53120c
commit 0ff96d7e74
8 files changed
+488 -118

No files matched your search

+7 -3
View File
@@ -73,9 +73,13 @@ publish a tag that has no section here.
- Admin client certificates: `rollcall admin-cert request NAME` (key stays on the client, CSR only),
`list`, `revoke`; the region accepts them as admin over HTTPS; CLI/console use them from
`client.json`, the web UI works without a token when the browser presents one (`.p12` written).
- `deploy/bootstrap.sh`: a region on a fresh Debian 13 host in one command (packages, PostgreSQL,
TFTP tree, generated config, services, commissioning image downloaded and verified or built,
provisioning subnet with ranges and DHCP, optional NAT, first OS image).
- `deploy/bootstrap.sh`: a region on a fresh Debian 13 host. Day 0 needs nothing about the network:
packages, PostgreSQL, TFTP tree, config, services and a verified commissioning image, the region
reachable on its management address with no DHCP or subnets. Run again with `--iface` once the
provisioning network is known: server address updated, subnet with ranges, DHCP, optional NAT.
- `rollcall net discover`: per interface link, speed, addresses and implied subnets, switch/port/VLAN
from LLDP, DHCP servers that answer, neighbours, and what to do with it.
- `commission_append` follows `server_url` when left empty.
- `rollcall doctor`: checks configuration, database, listeners, certificates, TFTP files, image
checksums, dnsmasq, DNS as machines see it, clock, foreign DHCP servers on the provisioning
network (DHCPDISCOVER probe), rack controllers, OS images and IPAM, with what to do.
+5 -1
View File
@@ -18,7 +18,7 @@ type Config struct {
MasterKeyFile string `json:"master_key_file"` // 32-byte hex key for AES-GCM at-rest encryption
AdminToken string `json:"admin_token"` // bearer token for the management API / web UI
AgentToken string `json:"agent_token"` // X-Rollcall-Token for the agent endpoints ("" = open)
CommissionAppend string `json:"commission_append"` // kernel cmdline used in generated pxelinux files
CommissionAppend string `json:"commission_append"` // kernel cmdline used in generated pxelinux files ("" = derived from server_url)
IPMITool string `json:"ipmitool"` // path, default "ipmitool"
BMCHints []BMCHint `json:"bmc_hints"` // out-of-band BMC mapping for machines without in-band IPMI
@@ -83,6 +83,10 @@ func loadConfig(path string) (*Config, error) {
c.ServerURL = "http://" + c.Listen // host:port, IPv6 already bracketed ([::1]:80)
}
c.ServerURL = strings.TrimRight(c.ServerURL, "/")
if c.CommissionAppend == "" { // follows server_url (it often changes once the provisioning network is known)
c.CommissionAppend = fmt.Sprintf("boot=live components fetch=%s/rollcall/filesystem.squashfs rollcall.server=%s noautologin rollcall.token=%s console=tty0 console=ttyS0,115200n8",
c.ServerURL, c.ServerURL, c.AgentToken)
}
if c.ImagesDir == "" {
c.ImagesDir = filepath.Join(filepath.Dir(c.ImageDir), "images")
}
+324
View File
@@ -0,0 +1,324 @@
package main
// rollcall net discover: what this host sees on each interface, for the day
// when the provisioning network is not decided yet. Addresses (and the subnet
// they imply), the switch port and VLAN from LLDP (lldpd), DHCP servers that
// answer, neighbours; then a suggestion of what to do with each interface.
import (
"encoding/json"
"flag"
"fmt"
"net"
"os"
"os/exec"
"path/filepath"
"sort"
"strconv"
"strings"
"time"
)
type discoIface struct {
Name string `json:"name"`
Kind string `json:"kind"` // physical | vlan | bond | bridge | virtual
MAC string `json:"mac"`
Up bool `json:"up"`
SpeedMbps int `json:"speed_mbps,omitempty"`
Addrs []string `json:"addrs"`
Subnets []string `json:"subnets"`
Default bool `json:"default_route"`
Switch string `json:"switch,omitempty"`
Port string `json:"port,omitempty"`
VLAN string `json:"vlan,omitempty"`
DHCP []string `json:"dhcp_servers"`
DHCPProbe string `json:"dhcp_probe,omitempty"` // why it was not probed
Neighbours int `json:"neighbours"`
Suggestion string `json:"suggestion"`
}
func discoverCmd(args []string) error {
fs := flag.NewFlagSet("net discover", flag.ExitOnError)
asJSON := fs.Bool("json", false, "JSON output")
noProbe := fs.Bool("no-dhcp-probe", false, "do not send DHCPDISCOVERs")
_ = fs.Parse(args)
ifs := discoverInterfaces(!*noProbe)
if *asJSON {
enc := json.NewEncoder(os.Stdout)
enc.SetIndent("", " ")
return enc.Encode(ifs)
}
fmt.Printf("%-12s %-8s %-5s %6s %-22s %-28s %-14s %s\n", "INTERFACE", "KIND", "LINK", "SPEED", "ADDRESSES", "SWITCH / PORT / VLAN", "DHCP SERVERS", "NEIGH")
for _, i := range ifs {
link := "down"
if i.Up {
link = "up"
}
speed := "-"
if i.SpeedMbps > 0 {
speed = fmt.Sprintf("%dG", i.SpeedMbps/1000)
if i.SpeedMbps < 1000 {
speed = fmt.Sprintf("%dM", i.SpeedMbps)
}
}
sw := "-"
if i.Switch != "" || i.Port != "" {
sw = strings.Trim(i.Switch+" / "+i.Port+" / "+orDash(i.VLAN), " /")
}
dh := strings.Join(i.DHCP, ",")
if dh == "" {
dh = orDash(i.DHCPProbe)
}
name := i.Name
if i.Default {
name += "*"
}
fmt.Printf("%-12s %-8s %-5s %6s %-22s %-28s %-14s %d\n", name, i.Kind, link, speed, trunc(strings.Join(i.Addrs, " "), 22), trunc(sw, 28), trunc(dh, 14), i.Neighbours)
}
fmt.Println("(* carries the default route)")
fmt.Println()
for _, i := range ifs {
if i.Suggestion != "" {
fmt.Printf("%-12s %s\n", i.Name, i.Suggestion)
}
}
return nil
}
func discoverInterfaces(probe bool) []discoIface {
def := defaultRouteIface()
lldp := lldpNeighbours()
nets, _ := net.Interfaces()
var out []discoIface
type probeRes struct {
servers []string
err error
}
probes := map[string]chan probeRes{}
if probe { // in parallel: a DHCP server may take seconds to offer (it pings the address first)
for _, n := range nets {
if n.Flags&net.FlagLoopback != 0 || n.Flags&net.FlagUp == 0 || skipIface(n.Name) || ifaceKind(n.Name) == "bridge" || ifaceKind(n.Name) == "virtual" {
continue
}
ch := make(chan probeRes, 1)
probes[n.Name] = ch
go func(name string) { s, err := dhcpProbe(name, 6*time.Second); ch <- probeRes{s, err} }(n.Name)
}
}
for _, n := range nets {
if n.Flags&net.FlagLoopback != 0 || skipIface(n.Name) {
continue
}
d := discoIface{Name: n.Name, MAC: n.HardwareAddr.String(), Kind: ifaceKind(n.Name), Default: n.Name == def, Addrs: []string{}, Subnets: []string{}, DHCP: []string{}}
if b, err := os.ReadFile("/sys/class/net/" + n.Name + "/operstate"); err == nil {
d.Up = strings.TrimSpace(string(b)) == "up"
}
if b, err := os.ReadFile("/sys/class/net/" + n.Name + "/speed"); err == nil {
if v, err := strconv.Atoi(strings.TrimSpace(string(b))); err == nil && v > 0 {
d.SpeedMbps = v
}
}
addrs, _ := n.Addrs()
for _, a := range addrs {
ipn, ok := a.(*net.IPNet)
if !ok || ipn.IP.IsLinkLocalUnicast() {
continue
}
ones, _ := ipn.Mask.Size()
d.Addrs = append(d.Addrs, fmt.Sprintf("%s/%d", ipn.IP, ones))
d.Subnets = append(d.Subnets, (&net.IPNet{IP: ipn.IP.Mask(ipn.Mask), Mask: ipn.Mask}).String())
}
if l, ok := lldp[n.Name]; ok {
d.Switch, d.Port, d.VLAN = l[0], l[1], l[2]
}
d.Neighbours = neighbourCount(n.Name)
switch {
case !probe:
d.DHCPProbe = "not probed"
case !d.Up:
d.DHCPProbe = "link down"
case d.Kind == "bridge" || d.Kind == "virtual":
d.DHCPProbe = "skipped"
case probes[n.Name] == nil:
d.DHCPProbe = "skipped"
default:
r := <-probes[n.Name]
if s, err := r.servers, r.err; err != nil {
d.DHCPProbe = "needs root"
} else {
for _, x := range s {
if ifaceWithAddr(x) == "" {
d.DHCP = append(d.DHCP, x)
} else {
d.DHCP = append(d.DHCP, x+"(this host)")
}
}
if len(s) == 0 {
d.DHCPProbe = "none"
}
}
}
d.Suggestion = discoSuggest(d)
out = append(out, d)
}
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
return out
}
func discoSuggest(d discoIface) string {
foreign := 0
for _, s := range d.DHCP {
if !strings.HasSuffix(s, "(this host)") {
foreign++
}
}
v4 := ""
for _, s := range d.Subnets {
if !strings.Contains(s, ":") {
v4 = s
break
}
}
vlan := ""
if d.VLAN != "" {
vlan = " (switch says VLAN " + d.VLAN + ")"
}
switch {
case d.Kind == "bridge" || d.Kind == "virtual":
return ""
case !d.Up:
return "no link: cable it to the provisioning network" + vlan
case d.Default && foreign > 0:
return "uplink: management network with its own DHCP (" + strings.Join(d.DHCP, ",") + "); keep it for the region's access" + vlan
case d.Default:
return "uplink: the region is reached here" + vlan
case v4 == "":
return "link up, no IPv4 address" + vlan + ": give it its address on the provisioning network, then bootstrap.sh --iface " + d.Name
case foreign > 0:
return v4 + " already has DHCP (" + strings.Join(d.DHCP, ",") + "): either the network team turns it off for this VLAN, or keep it and point it at this host (next-server " + strings.Split(d.Addrs[0], "/")[0] + ", or a DHCP relay); then bootstrap.sh --iface " + d.Name + " --no-dhcp"
default:
return "candidate provisioning network " + v4 + vlan + ": bootstrap.sh --iface " + d.Name + " (or Network -> New subnet in the web UI)"
}
}
func skipIface(name string) bool {
for _, p := range []string{"docker", "veth", "virbr", "br-", "cni", "flannel", "tap", "vnet", "wg", "tun"} {
if strings.HasPrefix(name, p) {
return true
}
}
return false
}
func ifaceKind(name string) string {
base := "/sys/class/net/" + name
switch {
case exists(base + "/bridge"):
return "bridge"
case exists(base + "/bonding"):
return "bond"
case exists("/proc/net/vlan/" + name):
return "vlan"
case exists(base + "/device"):
return "physical"
}
return "virtual"
}
func exists(p string) bool { _, err := os.Stat(p); return err == nil }
func defaultRouteIface() string {
b, err := os.ReadFile("/proc/net/route")
if err != nil {
return ""
}
for _, l := range strings.Split(string(b), "\n")[1:] {
f := strings.Fields(l)
if len(f) > 2 && f[1] == "00000000" && f[7] == "00000000" {
return f[0]
}
}
return ""
}
func neighbourCount(iface string) int {
out, err := exec.Command("ip", "-j", "neigh", "show", "dev", iface).Output()
if err != nil {
return 0
}
var ns []map[string]any
_ = json.Unmarshal(out, &ns)
n := 0
for _, x := range ns {
if st, _ := x["state"].([]any); len(st) > 0 && st[0] != "FAILED" && st[0] != "INCOMPLETE" {
n++
}
}
return n
}
// lldpNeighbours: interface -> [switch name, port, vlan] from lldpd.
func lldpNeighbours() map[string][3]string {
out := map[string][3]string{}
bin, err := exec.LookPath("lldpcli")
if err != nil {
return out
}
b, err := exec.Command(bin, "-f", "json0", "show", "neighbors", "details").Output()
if err != nil {
return out
}
var doc struct {
LLDP []struct {
Interface []struct {
Name string `json:"name"`
Chassis []struct {
Name []struct {
Value string `json:"value"`
} `json:"name"`
} `json:"chassis"`
Port []struct {
ID []struct {
Value string `json:"value"`
} `json:"id"`
Descr []struct {
Value string `json:"value"`
} `json:"descr"`
} `json:"port"`
VLAN []struct {
VlanID string `json:"vlan-id"`
PVID bool `json:"pvid"`
} `json:"vlan"`
} `json:"interface"`
} `json:"lldp"`
}
if json.Unmarshal(b, &doc) != nil {
return out
}
for _, l := range doc.LLDP {
for _, i := range l.Interface {
var sw, port, vlan string
if len(i.Chassis) > 0 && len(i.Chassis[0].Name) > 0 {
sw = i.Chassis[0].Name[0].Value
}
if len(i.Port) > 0 {
if len(i.Port[0].Descr) > 0 {
port = i.Port[0].Descr[0].Value
} else if len(i.Port[0].ID) > 0 {
port = i.Port[0].ID[0].Value
}
}
var vs []string
for _, v := range i.VLAN {
s := v.VlanID
if v.PVID {
s += "(untagged)"
}
vs = append(vs, s)
}
vlan = strings.Join(vs, ",")
out[filepath.Base(i.Name)] = [3]string{sw, port, vlan}
}
}
return out
}
+8 -4
View File
@@ -98,8 +98,10 @@ func doctorCmd(args []string) error {
}
if ports[fmt.Sprintf("%s/%d", p.proto, p.port)] {
d.ok(fmt.Sprintf("%s/%d", p.proto, p.port), p.what)
} else if p.port == 69 && !ports["udp/67"] {
d.ok("udp/69", "TFTP starts with the first VLAN served")
} else if p.port == 67 {
d.warn("udp/67", "nothing listens for DHCP", "no VLAN with dhcp on this rack yet (rollcall vlan add FABRIC VID -dhcp -rack "+cfg.RackName+"), or dnsmasq is down")
d.warn("udp/67", "this host serves no DHCP yet", "no provisioning network decided: rollcall net discover, then deploy/bootstrap.sh --iface IFACE (or a VLAN with dhcp in the web UI)")
} else {
d.fail(fmt.Sprintf("%s/%d", p.proto, p.port), "nothing listens for "+p.what, "systemctl status rollcall dnsmasq")
}
@@ -189,7 +191,7 @@ func doctorCmd(args []string) error {
} else {
d.ok("dns (host)", "resolves outside names")
}
if srvHost != "" && ports["udp/53"] {
if srvHost != "" && ports["udp/53"] && ports["udp/67"] {
r := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "udp", net.JoinHostPort(srvHost, "53"))
}}
@@ -211,8 +213,10 @@ func doctorCmd(args []string) error {
}
// other DHCP servers on the provisioning network
if iface != "" && !*noProbe {
servers, err := dhcpProbe(iface, 4*time.Second)
if iface != "" && !*noProbe && !ports["udp/67"] {
d.ok("dhcp probe", "skipped: this host serves no DHCP yet")
} else if iface != "" && !*noProbe {
servers, err := dhcpProbe(iface, 6*time.Second)
switch {
case err != nil:
d.warn("dhcp probe", err.Error(), "run as root; or --no-dhcp-probe")
+9
View File
@@ -28,6 +28,7 @@ func usage() {
rollcall tui [-dump WxH -page P -machine ID -plain]
rollcall serve [-config PATH] region: API, database, web UI, embedded rack controller
rollcall doctor [-deep] [-no-dhcp-probe] check this installation and say what to fix
rollcall net discover [-json] what each interface of this host sees (LLDP, DHCP, neighbours)
rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines
rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate)
rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host)
@@ -79,6 +80,9 @@ func newFlags(name string) *flag.FlagSet { return flag.NewFlagSet(name, flag.Exi
func main() {
log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
if len(os.Args) > 2 && os.Args[1] == "net" && os.Args[2] == "discover" {
os.Args = append([]string{os.Args[0], "net-discover"}, os.Args[3:]...)
}
if len(os.Args) < 2 {
// A bare `rollcall` in a terminal opens the console; in a pipe or a
// script it prints its help, as it always did.
@@ -108,6 +112,11 @@ func main() {
fmt.Fprintln(os.Stderr, "error:", err)
os.Exit(1)
}
case "net-discover":
if err := discoverCmd(args); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)
os.Exit(1)
}
case "doctor":
if err := doctorCmd(args); err != nil {
fmt.Fprintln(os.Stderr, "error:", err)
+135 -110
View File
@@ -1,31 +1,39 @@
#!/bin/bash
# Rollcall region on a fresh Debian 13 host, in one go. Run as root from a
# checkout (deploy/) or an unpacked release (rollcall + deploy/ + agent/).
# Rollcall region on a fresh Debian 13 host. Run as root from a checkout
# (deploy/) or an unpacked release (rollcall + deploy/ + agent/).
#
# ./bootstrap.sh --iface eth1 [--name NAME] [--domain D] [--gateway IP | --nat UPLINK]
# [--image-url URL | --build-image] [--os debian-13] [--no-dhcp] [--yes]
# Day 0 - nothing about the network is decided yet:
# ./bootstrap.sh [--image-url URL | --build-image] [--name NAME]
# installs everything and starts the region on this host's management
# address: web UI and API work, no DHCP, no subnets. Then look around:
# rollcall net discover interfaces, switch ports (LLDP), DHCP servers, neighbours
#
# --iface interface on the provisioning network; it must already have
# its IPv4 address (this host's address there becomes the server URL)
# --name rack controller name of this host (default: short hostname)
# --domain DNS domain handed to machines (default: rollcall.internal)
# --gateway default router for machines on the provisioning network
# --nat UPLINK make this host the router: forward + masquerade out of UPLINK
# (default when --gateway is not given: the interface of the default route)
# --image-url directory with vmlinuz, initrd.img, filesystem.squashfs and SHA256SUMS
# Later - once the provisioning network is known (run again, nothing is lost):
# ./bootstrap.sh --iface IFACE [--gateway IP | --nat UPLINK] [--domain D] [--no-dhcp]
# IFACE must have its address on that network; this host's address there
# becomes the server URL machines use, the subnet is modelled with ranges
# and (unless --no-dhcp) this host serves DHCP/PXE on it.
# Or do the same from the web UI / CLI (rollcall subnet add, vlan add -dhcp),
# or import the prefixes from NetBox (rollcall ipam import).
#
# --name NAME rack controller name of this host (default: short hostname)
# --image-url directory with vmlinuz, initrd.img, filesystem.squashfs, SHA256SUMS
# --build-image build the commissioning image here with live-build (~20 min)
# --os NAME import one OS image now (e.g. debian-13); the nightly sync does the rest
# --no-dhcp model the network but do not serve DHCP (another DHCP server is in charge)
#
# Safe to run again: existing configuration, database and image are kept.
# --gateway IP default router for machines on the provisioning network
# --nat UPLINK this host routes for them: forward + masquerade out of UPLINK
# (default with --iface and no --gateway: the default route's interface)
# --domain D DNS domain handed to machines (default rollcall.internal)
# --no-dhcp model the network but leave DHCP to the existing server
# --os NAME import one OS image now (e.g. debian-13); a nightly timer syncs the catalog
# --yes do not ask
set -euo pipefail
HERE=$(cd "$(dirname "$0")" && pwd)
ROOT=$(cd "$HERE/.." && pwd)
IFACE="" NAME=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0
IFACE="" RACK=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0
while [ $# -gt 0 ]; do
case "$1" in
--iface) IFACE=$2; shift ;;
--name) NAME=$2; shift ;;
--name) RACK=$2; shift ;;
--domain) DOMAIN=$2; shift ;;
--gateway) GATEWAY=$2; shift ;;
--nat) NAT=$2; shift ;;
@@ -34,7 +42,7 @@ while [ $# -gt 0 ]; do
--os) OS=$2; shift ;;
--no-dhcp) DHCP=0 ;;
--yes|-y) YES=1 ;;
-h|--help) sed -n '2,24p' "$0"; exit 0 ;;
-h|--help) sed -n '2,31p' "$0"; exit 0 ;;
*) echo "unknown option $1 (--help)" >&2; exit 2 ;;
esac
shift
@@ -44,40 +52,29 @@ say() { printf '\n\033[1m== %s\033[0m\n' "$*"; }
ok() { printf ' \033[32mOK\033[0m %s\n' "$*"; }
die() { printf ' \033[31mERROR\033[0m %s\n' "$*" >&2; exit 1; }
[ "$(id -u)" = 0 ] || die "run as root"
. /etc/os-release
[ "$ID" = debian ] || echo "note: tested on Debian 13; this is $PRETTY_NAME" >&2
# ------------------------------------------------------------------ 0. what and where
say "0/9 provisioning network"
if [ -z "$IFACE" ]; then
def=$(ip -4 route show default | awk '{print $5; exit}')
cand=$(ip -4 -o addr show scope global | awk -v d="$def" '$2 != d {print $2}' | sort -u)
[ "$(echo "$cand" | grep -c .)" = 1 ] || die "say which interface faces the machines: --iface IFACE (candidates: $(echo $cand))"
IFACE=$cand
fi
CIDR_ADDR=$(ip -4 -o addr show dev "$IFACE" scope global | awk '{print $4; exit}')
[ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; configure it first (netplan / ifupdown)"
read -r ADDR NET PREFIX FIRST <<<"$(python3 - "$CIDR_ADDR" <<'P'
import ipaddress, sys
i = ipaddress.ip_interface(sys.argv[1])
print(i.ip, i.network, i.network.prefixlen, i.network.network_address)
P
)"
[ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; create the subnet and ranges by hand"
off() { python3 -c "import ipaddress,sys;print(ipaddress.ip_address('$FIRST')+int(sys.argv[1]))" "$1"; }
OSNAME=$(. /etc/os-release; echo "$ID $VERSION_ID")
case "$OSNAME" in debian*) ;; *) echo "note: tested on Debian 13; this is $OSNAME" >&2 ;; esac
CONF=/etc/rollcall/config.json
UPLINK=$(ip -4 route show default | awk '{print $5; exit}')
if [ -z "$GATEWAY" ]; then
GATEWAY=$ADDR
NAT=${NAT:-$UPLINK}
addr_of() { ip -4 -o addr show dev "$1" scope global | awk '{print $4; exit}'; }
# ------------------------------------------------------------------ 0. where
say "0/9 this host"
if [ -n "$IFACE" ]; then
CIDR_ADDR=$(addr_of "$IFACE")
[ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; give it its address on the provisioning network first"
read -r ADDR NET PREFIX FIRST <<<"$(python3 -c 'import ipaddress,sys;i=ipaddress.ip_interface(sys.argv[1]);print(i.ip,i.network,i.network.prefixlen,i.network.network_address)' "$CIDR_ADDR")"
[ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; model it by hand (rollcall subnet add / range add)"
if [ -z "$GATEWAY" ]; then GATEWAY=$ADDR; NAT=${NAT:-$UPLINK}; fi
[ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not $IFACE"
echo " provisioning $IFACE $CIDR_ADDR (subnet $NET), machines' gateway $GATEWAY${NAT:+ via NAT out of $NAT}, DHCP $([ $DHCP = 1 ] && echo on || echo off)"
else
[ -n "$UPLINK" ] || die "no default route: give this host network access first"
ADDR=$(addr_of "$UPLINK" | cut -d/ -f1)
echo " no provisioning network yet: the region starts on $UPLINK ($ADDR) without DHCP or subnets"
fi
[ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not the provisioning interface"
SERVER=http://$ADDR
cat <<EOF
interface $IFACE ($CIDR_ADDR), subnet $NET
server $SERVER (rack controller "$NAME")
machines gateway $GATEWAY${NAT:+ (NAT out of $NAT)}, DNS $ADDR, domain $DOMAIN, DHCP $([ $DHCP = 1 ] && echo on || echo off)
ranges reserved $(off 1)-$(off 20), static $(off 50)-$(off 99), dynamic $(off 100)-$(off 200)
EOF
echo " region $SERVER, rack controller \"$RACK\""
if [ $YES = 0 ] && [ -t 0 ]; then
read -r -p " continue? [y/N] " a; case "$a" in y|Y|yes) ;; *) exit 1 ;; esac
fi
@@ -85,30 +82,31 @@ fi
# ------------------------------------------------------------------ 1. packages
say "1/9 packages"
export DEBIAN_FRONTEND=noninteractive
PKGS="postgresql dnsmasq pxelinux syslinux-common syslinux-efi qemu-utils ipmitool jq zstd curl ca-certificates python3 nftables"
PKGS="postgresql dnsmasq pxelinux syslinux-common syslinux-efi qemu-utils ipmitool jq zstd curl ca-certificates python3 nftables lldpd"
[ $BUILD = 1 ] && PKGS="$PKGS live-build debootstrap squashfs-tools xorriso"
apt-get update -q >/dev/null
apt-get install -yq $PKGS >/dev/null
# until rollcall writes its configuration, dnsmasq must not grab port 53 on every address
install -d /etc/dnsmasq.d
[ -e /etc/dnsmasq.d/00-rollcall-base.conf ] || printf '# rollcall bootstrap: bind only where configured (systemd-resolved keeps 127.0.0.53)\nbind-dynamic\n' > /etc/dnsmasq.d/00-rollcall-base.conf
# until rollcall serves a VLAN, dnsmasq must not answer DNS/DHCP everywhere
# (rollcall's generated configuration adds the interfaces of the VLANs it serves)
printf '# rollcall bootstrap: answer only on loopback until rollcall serves a VLAN\n# (systemd-resolved keeps 127.0.0.53)\nbind-dynamic\ninterface=lo\n' > /etc/dnsmasq.d/00-rollcall-base.conf
systemctl restart dnsmasq || true
ok "$(echo $PKGS | wc -w) packages"
ok "installed (lldpd shows which switch port and VLAN each interface is on)"
# ------------------------------------------------------------------ 2. database
say "2/9 PostgreSQL"
systemctl enable --now postgresql >/dev/null
CONF=/etc/rollcall/config.json
PGPASS=""
if [ -f "$CONF" ]; then
ok "config exists, database settings kept"
ok "configured before, kept"
else
PGPASS=$(head -c 24 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28)
if [ "$(su postgres -c "psql -tAc \"select 1 from pg_roles where rolname='rollcall'\"")" = 1 ]; then
su postgres -c "psql -q -c \"ALTER ROLE rollcall PASSWORD '$PGPASS'\""
PGPASS=$(head -c 32 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28)
if [ "$(runuser -u postgres -- psql -tAc "select 1 from pg_roles where rolname='rollcall'")" = 1 ]; then
runuser -u postgres -- psql -q -c "ALTER ROLE rollcall PASSWORD '$PGPASS'"
else
su postgres -c "psql -q -c \"CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'\""
runuser -u postgres -- psql -q -c "CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'"
fi
[ "$(su postgres -c "psql -tAc \"select 1 from pg_database where datname='rollcall'\"")" = 1 ] || su postgres -c "psql -q -c 'CREATE DATABASE rollcall OWNER rollcall'"
[ "$(runuser -u postgres -- psql -tAc "select 1 from pg_database where datname='rollcall'")" = 1 ] || runuser -u postgres -- psql -q -c "CREATE DATABASE rollcall OWNER rollcall"
ok "role and database rollcall"
fi
@@ -139,36 +137,55 @@ ok "$("$HERE/rollcall" version)"
say "5/9 configuration"
getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall
install -d -m 0750 -o root -g rollcall /etc/rollcall
FQDN=$(hostname -f 2>/dev/null || hostname)
CHANGED=0
if [ ! -f "$CONF" ]; then
AGENT=$("$HERE/rollcall" keygen) ADMIN=$("$HERE/rollcall" keygen)
FQDN=$(hostname -f 2>/dev/null || hostname)
python3 - "$CONF" <<P
python3 - "$HERE/config.example.json" "$CONF" <<P
import json, sys
c = json.load(open("$HERE/config.example.json"))
srv = "$SERVER"
c = json.load(open(sys.argv[1]))
c.update({
"listen": ":80", "server_url": srv,
"listen": ":80", "server_url": "$SERVER",
"database_url": "postgres://rollcall:$PGPASS@127.0.0.1:5432/rollcall?sslmode=disable",
"admin_token": "$ADMIN", "agent_token": "$AGENT",
"commission_append": f"boot=live components fetch={srv}/rollcall/filesystem.squashfs rollcall.server={srv} noautologin rollcall.token=$AGENT console=tty0 console=ttyS0,115200n8",
"rack_name": "$NAME", "tls_listen": ":443", "tls_names": sorted({"$FQDN", "$ADDR"}), "admin_http": "loopback",
"commission_append": "", # derived from server_url, follows it when it changes
"rack_name": "$RACK", "tls_listen": ":443", "tls_names": sorted({"$FQDN", "$ADDR"}), "admin_http": "loopback",
"node_pools": {}, "bmc_hints": [],
})
c["cluster_vars"].update({"domain": "$DOMAIN", "cluster_name": "$NAME", "slurm_controller": "$NAME", "slurm_controller_ip": "$ADDR", "ssh_keys": []})
c["cluster_vars"].update({"domain": "$DOMAIN", "cluster_name": "$RACK", "slurm_controller": "$RACK", "slurm_controller_ip": "$ADDR", "ssh_keys": []})
c["ipam"]["provider"] = ""
json.dump(c, open(sys.argv[1], "w"), indent=2)
json.dump(c, open(sys.argv[2], "w"), indent=2)
P
chown root:rollcall "$CONF"; chmod 0640 "$CONF"
ok "$CONF (tokens generated; admin_http loopback: the web UI is HTTPS-only)"
ok "$CONF written (tokens generated; the web UI is HTTPS-only: admin_http loopback)"
else
ok "$CONF kept"
# an existing installation: the provisioning address may have arrived since
if python3 - "$CONF" "$SERVER" "$ADDR" "$FQDN" <<'P'
import json, sys
p, srv, addr, fqdn = sys.argv[1:]
c = json.load(open(p))
old = c.get("server_url", "")
names = set(c.get("tls_names") or []) | {addr, fqdn}
changed = old != srv or set(c.get("tls_names") or []) != names
gen = c.get("commission_append", "")
if old and old in gen and "rollcall.token=" in gen: # the old generated form: let it follow server_url
c["commission_append"] = ""
changed = True
c["server_url"], c["tls_names"] = srv, sorted(names)
if changed:
json.dump(c, open(p, "w"), indent=2)
sys.exit(0 if changed else 1)
P
then CHANGED=1; ok "server_url now $SERVER (was different), tls_names updated"
else ok "$CONF kept"
fi
fi
# ------------------------------------------------------------------ 6. install
# ------------------------------------------------------------------ 6. services
say "6/9 services"
install -d /srv/rollcall/image
(cd "$HERE" && ./install.sh >/dev/null)
for i in $(seq 1 30); do curl -sf -o /dev/null http://127.0.0.1/ca.pem && break; sleep 1; done
(cd "$HERE" && sh ./install.sh >/dev/null)
for _ in $(seq 1 30); do curl -sf -o /dev/null http://127.0.0.1/ca.pem && break; sleep 1; done
curl -sf -o /dev/null http://127.0.0.1/ca.pem || die "rollcall did not come up: journalctl -u rollcall"
ok "rollcall running, CA $(rollcall ca show | awk '/fingerprint/{print $2}')"
@@ -180,9 +197,9 @@ if [ -s $IMG/vmlinuz ] && [ -s $IMG/initrd.img ] && [ -s $IMG/filesystem.squashf
elif [ -n "$IMAGE_URL" ]; then
tmp=$(mktemp -d -p /srv/rollcall)
curl -fsSL -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS" || die "$IMAGE_URL/SHA256SUMS missing"
for f in vmlinuz initrd.img filesystem.squashfs; do curl -fL --progress-bar -o "$tmp/$f" "$IMAGE_URL/$f"; done
(cd "$tmp" && grep -E ' (\*)?(vmlinuz|initrd.img|filesystem.squashfs)$' SHA256SUMS | sha256sum -c --quiet -) || die "checksum mismatch"
mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rmdir "$tmp" 2>/dev/null || rm -rf "$tmp"
for f in vmlinuz initrd.img filesystem.squashfs; do curl -fsSL -o "$tmp/$f" "$IMAGE_URL/$f"; done
(cd "$tmp" && sha256sum -c --quiet SHA256SUMS) || die "checksum mismatch"
mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rm -rf "$tmp"
chmod 0644 $IMG/*
ok "downloaded and verified from $IMAGE_URL"
elif [ $BUILD = 1 ]; then
@@ -191,34 +208,42 @@ elif [ $BUILD = 1 ]; then
(cd $IMG && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS)
ok "built"
else
echo " no image yet: rerun with --image-url URL or --build-image (machines cannot commission without it)" >&2
echo " not yet: rerun with --image-url URL or --build-image before machines commission" >&2
fi
# ------------------------------------------------------------------ 8. network model
say "8/9 network model"
FAB=provisioning
rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null
if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then
rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null
rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null
rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null
rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null
ok "subnet $NET with ranges"
# ------------------------------------------------------------------ 8. provisioning network
say "8/9 provisioning network"
if [ -z "$IFACE" ]; then
echo " not decided yet - nothing modelled, no DHCP served. When it is:"
echo " rollcall net discover what each interface sees (LLDP, DHCP servers, neighbours)"
echo " $0 --iface IFACE ... or the web UI: Network -> New subnet"
else
ok "subnet $NET exists, kept"
fi
if [ $DHCP = 1 ]; then
rollcall vlan add "$FAB" 0 -dhcp -rack "$NAME" >/dev/null
ok "DHCP/PXE on $IFACE by rack controller $NAME"
fi
if [ -n "$NAT" ]; then
echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf
sysctl -q -p /etc/sysctl.d/60-rollcall-forward.conf
install -d /etc/nftables.d
printf 'table ip rollcall_nat {\n chain postrouting {\n type nat hook postrouting priority 100;\n oifname "%s" ip saddr %s masquerade\n }\n}\n' "$NAT" "$NET" > /etc/nftables.d/rollcall-nat.nft
grep -q 'include "/etc/nftables.d/\*.nft"' /etc/nftables.conf 2>/dev/null || echo 'include "/etc/nftables.d/*.nft"' >> /etc/nftables.conf
nft -c -f /etc/nftables.conf && systemctl enable --now nftables >/dev/null && systemctl reload nftables
ok "forwarding + masquerade $NET out of $NAT"
[ $CHANGED = 1 ] && systemctl restart rollcall && sleep 2
off() { python3 -c "import ipaddress,sys;print(ipaddress.ip_address('$FIRST')+int(sys.argv[1]))" "$1"; }
FAB=provisioning
rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null
if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then
rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null
rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null
rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null
rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null
ok "subnet $NET: reserved $(off 1)-$(off 20), static $(off 50)-$(off 99), dynamic $(off 100)-$(off 200)"
else
ok "subnet $NET exists, kept"
fi
if [ $DHCP = 1 ]; then
rollcall vlan add "$FAB" 0 -dhcp -rack "$RACK" >/dev/null
ok "DHCP/PXE on $IFACE"
fi
if [ -n "$NAT" ]; then
echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf
sysctl -q -p /etc/sysctl.d/60-rollcall-forward.conf
install -d /etc/nftables.d
printf 'table ip rollcall_nat {\n chain postrouting {\n type nat hook postrouting priority 100;\n oifname "%s" ip saddr %s masquerade\n }\n}\n' "$NAT" "$NET" > /etc/nftables.d/rollcall-nat.nft
grep -q 'include "/etc/nftables.d/\*.nft"' /etc/nftables.conf 2>/dev/null || echo 'include "/etc/nftables.d/*.nft"' >> /etc/nftables.conf
nft -c -f /etc/nftables.conf && systemctl enable nftables >/dev/null 2>&1 && systemctl restart nftables
ok "forwarding + masquerade $NET out of $NAT"
fi
fi
# ------------------------------------------------------------------ 9. OS images
@@ -226,15 +251,15 @@ say "9/9 OS images"
if [ -n "$OS" ]; then
runuser -u rollcall -- rollcall os sync "$OS" || echo " os sync $OS failed; the nightly timer retries" >&2
else
echo " none imported now; rollcall-os-sync.timer fetches the catalog nightly (or: runuser -u rollcall -- rollcall os sync debian-13)"
echo " none imported now; rollcall-os-sync.timer syncs the catalog nightly (now: runuser -u rollcall -- rollcall os sync debian-13)"
fi
cat <<EOF
Done. Rollcall region "$NAME" on $SERVER
web UI https://$ADDR (CA: rollcall ca show; trust /ca.pem in your browser)
admin a certificate for your laptop: rollcall admin-cert request NAME (needs the token once:
jq -r .admin_token $CONF)
machines set them to PXE boot on the network of $IFACE; unknown ones commission themselves
check rollcall doctor
Done: rollcall region "$RACK" on $SERVER
web UI https://$ADDR (trust the CA from /ca.pem: $(rollcall ca show | awk '/fingerprint/{print $2}'))
admin from your laptop: rollcall admin-cert request NAME (token once: jq -r .admin_token $CONF)
check rollcall doctor
EOF
[ -z "$IFACE" ] && echo " next rollcall net discover (then: $0 --iface IFACE)"
exit 0
Regular → Executable
View File
File mode changed.
Regular → Executable
View File
File mode changed.