diff --git a/CHANGELOG.md b/CHANGELOG.md index cde2538..acdc413 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -73,9 +73,13 @@ publish a tag that has no section here. - Admin client certificates: `rollcall admin-cert request NAME` (key stays on the client, CSR only), `list`, `revoke`; the region accepts them as admin over HTTPS; CLI/console use them from `client.json`, the web UI works without a token when the browser presents one (`.p12` written). -- `deploy/bootstrap.sh`: a region on a fresh Debian 13 host in one command (packages, PostgreSQL, - TFTP tree, generated config, services, commissioning image downloaded and verified or built, - provisioning subnet with ranges and DHCP, optional NAT, first OS image). +- `deploy/bootstrap.sh`: a region on a fresh Debian 13 host. Day 0 needs nothing about the network: + packages, PostgreSQL, TFTP tree, config, services and a verified commissioning image, the region + reachable on its management address with no DHCP or subnets. Run again with `--iface` once the + provisioning network is known: server address updated, subnet with ranges, DHCP, optional NAT. +- `rollcall net discover`: per interface link, speed, addresses and implied subnets, switch/port/VLAN + from LLDP, DHCP servers that answer, neighbours, and what to do with it. +- `commission_append` follows `server_url` when left empty. - `rollcall doctor`: checks configuration, database, listeners, certificates, TFTP files, image checksums, dnsmasq, DNS as machines see it, clock, foreign DHCP servers on the provisioning network (DHCPDISCOVER probe), rack controllers, OS images and IPAM, with what to do. diff --git a/cmd/rollcall/config.go b/cmd/rollcall/config.go index 848851b..cfa22fb 100644 --- a/cmd/rollcall/config.go +++ b/cmd/rollcall/config.go @@ -18,7 +18,7 @@ type Config struct { MasterKeyFile string `json:"master_key_file"` // 32-byte hex key for AES-GCM at-rest encryption AdminToken string `json:"admin_token"` // bearer token for the management API / web UI AgentToken string `json:"agent_token"` // X-Rollcall-Token for the agent endpoints ("" = open) - CommissionAppend string `json:"commission_append"` // kernel cmdline used in generated pxelinux files + CommissionAppend string `json:"commission_append"` // kernel cmdline used in generated pxelinux files ("" = derived from server_url) IPMITool string `json:"ipmitool"` // path, default "ipmitool" BMCHints []BMCHint `json:"bmc_hints"` // out-of-band BMC mapping for machines without in-band IPMI @@ -83,6 +83,10 @@ func loadConfig(path string) (*Config, error) { c.ServerURL = "http://" + c.Listen // host:port, IPv6 already bracketed ([::1]:80) } c.ServerURL = strings.TrimRight(c.ServerURL, "/") + if c.CommissionAppend == "" { // follows server_url (it often changes once the provisioning network is known) + c.CommissionAppend = fmt.Sprintf("boot=live components fetch=%s/rollcall/filesystem.squashfs rollcall.server=%s noautologin rollcall.token=%s console=tty0 console=ttyS0,115200n8", + c.ServerURL, c.ServerURL, c.AgentToken) + } if c.ImagesDir == "" { c.ImagesDir = filepath.Join(filepath.Dir(c.ImageDir), "images") } diff --git a/cmd/rollcall/discover.go b/cmd/rollcall/discover.go new file mode 100644 index 0000000..10a2999 --- /dev/null +++ b/cmd/rollcall/discover.go @@ -0,0 +1,324 @@ +package main + +// rollcall net discover: what this host sees on each interface, for the day +// when the provisioning network is not decided yet. Addresses (and the subnet +// they imply), the switch port and VLAN from LLDP (lldpd), DHCP servers that +// answer, neighbours; then a suggestion of what to do with each interface. + +import ( + "encoding/json" + "flag" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "time" +) + +type discoIface struct { + Name string `json:"name"` + Kind string `json:"kind"` // physical | vlan | bond | bridge | virtual + MAC string `json:"mac"` + Up bool `json:"up"` + SpeedMbps int `json:"speed_mbps,omitempty"` + Addrs []string `json:"addrs"` + Subnets []string `json:"subnets"` + Default bool `json:"default_route"` + Switch string `json:"switch,omitempty"` + Port string `json:"port,omitempty"` + VLAN string `json:"vlan,omitempty"` + DHCP []string `json:"dhcp_servers"` + DHCPProbe string `json:"dhcp_probe,omitempty"` // why it was not probed + Neighbours int `json:"neighbours"` + Suggestion string `json:"suggestion"` +} + +func discoverCmd(args []string) error { + fs := flag.NewFlagSet("net discover", flag.ExitOnError) + asJSON := fs.Bool("json", false, "JSON output") + noProbe := fs.Bool("no-dhcp-probe", false, "do not send DHCPDISCOVERs") + _ = fs.Parse(args) + ifs := discoverInterfaces(!*noProbe) + if *asJSON { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(ifs) + } + fmt.Printf("%-12s %-8s %-5s %6s %-22s %-28s %-14s %s\n", "INTERFACE", "KIND", "LINK", "SPEED", "ADDRESSES", "SWITCH / PORT / VLAN", "DHCP SERVERS", "NEIGH") + for _, i := range ifs { + link := "down" + if i.Up { + link = "up" + } + speed := "-" + if i.SpeedMbps > 0 { + speed = fmt.Sprintf("%dG", i.SpeedMbps/1000) + if i.SpeedMbps < 1000 { + speed = fmt.Sprintf("%dM", i.SpeedMbps) + } + } + sw := "-" + if i.Switch != "" || i.Port != "" { + sw = strings.Trim(i.Switch+" / "+i.Port+" / "+orDash(i.VLAN), " /") + } + dh := strings.Join(i.DHCP, ",") + if dh == "" { + dh = orDash(i.DHCPProbe) + } + name := i.Name + if i.Default { + name += "*" + } + fmt.Printf("%-12s %-8s %-5s %6s %-22s %-28s %-14s %d\n", name, i.Kind, link, speed, trunc(strings.Join(i.Addrs, " "), 22), trunc(sw, 28), trunc(dh, 14), i.Neighbours) + } + fmt.Println("(* carries the default route)") + fmt.Println() + for _, i := range ifs { + if i.Suggestion != "" { + fmt.Printf("%-12s %s\n", i.Name, i.Suggestion) + } + } + return nil +} + +func discoverInterfaces(probe bool) []discoIface { + def := defaultRouteIface() + lldp := lldpNeighbours() + nets, _ := net.Interfaces() + var out []discoIface + type probeRes struct { + servers []string + err error + } + probes := map[string]chan probeRes{} + if probe { // in parallel: a DHCP server may take seconds to offer (it pings the address first) + for _, n := range nets { + if n.Flags&net.FlagLoopback != 0 || n.Flags&net.FlagUp == 0 || skipIface(n.Name) || ifaceKind(n.Name) == "bridge" || ifaceKind(n.Name) == "virtual" { + continue + } + ch := make(chan probeRes, 1) + probes[n.Name] = ch + go func(name string) { s, err := dhcpProbe(name, 6*time.Second); ch <- probeRes{s, err} }(n.Name) + } + } + for _, n := range nets { + if n.Flags&net.FlagLoopback != 0 || skipIface(n.Name) { + continue + } + d := discoIface{Name: n.Name, MAC: n.HardwareAddr.String(), Kind: ifaceKind(n.Name), Default: n.Name == def, Addrs: []string{}, Subnets: []string{}, DHCP: []string{}} + if b, err := os.ReadFile("/sys/class/net/" + n.Name + "/operstate"); err == nil { + d.Up = strings.TrimSpace(string(b)) == "up" + } + if b, err := os.ReadFile("/sys/class/net/" + n.Name + "/speed"); err == nil { + if v, err := strconv.Atoi(strings.TrimSpace(string(b))); err == nil && v > 0 { + d.SpeedMbps = v + } + } + addrs, _ := n.Addrs() + for _, a := range addrs { + ipn, ok := a.(*net.IPNet) + if !ok || ipn.IP.IsLinkLocalUnicast() { + continue + } + ones, _ := ipn.Mask.Size() + d.Addrs = append(d.Addrs, fmt.Sprintf("%s/%d", ipn.IP, ones)) + d.Subnets = append(d.Subnets, (&net.IPNet{IP: ipn.IP.Mask(ipn.Mask), Mask: ipn.Mask}).String()) + } + if l, ok := lldp[n.Name]; ok { + d.Switch, d.Port, d.VLAN = l[0], l[1], l[2] + } + d.Neighbours = neighbourCount(n.Name) + switch { + case !probe: + d.DHCPProbe = "not probed" + case !d.Up: + d.DHCPProbe = "link down" + case d.Kind == "bridge" || d.Kind == "virtual": + d.DHCPProbe = "skipped" + case probes[n.Name] == nil: + d.DHCPProbe = "skipped" + default: + r := <-probes[n.Name] + if s, err := r.servers, r.err; err != nil { + d.DHCPProbe = "needs root" + } else { + for _, x := range s { + if ifaceWithAddr(x) == "" { + d.DHCP = append(d.DHCP, x) + } else { + d.DHCP = append(d.DHCP, x+"(this host)") + } + } + if len(s) == 0 { + d.DHCPProbe = "none" + } + } + } + d.Suggestion = discoSuggest(d) + out = append(out, d) + } + sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name }) + return out +} + +func discoSuggest(d discoIface) string { + foreign := 0 + for _, s := range d.DHCP { + if !strings.HasSuffix(s, "(this host)") { + foreign++ + } + } + v4 := "" + for _, s := range d.Subnets { + if !strings.Contains(s, ":") { + v4 = s + break + } + } + vlan := "" + if d.VLAN != "" { + vlan = " (switch says VLAN " + d.VLAN + ")" + } + switch { + case d.Kind == "bridge" || d.Kind == "virtual": + return "" + case !d.Up: + return "no link: cable it to the provisioning network" + vlan + case d.Default && foreign > 0: + return "uplink: management network with its own DHCP (" + strings.Join(d.DHCP, ",") + "); keep it for the region's access" + vlan + case d.Default: + return "uplink: the region is reached here" + vlan + case v4 == "": + return "link up, no IPv4 address" + vlan + ": give it its address on the provisioning network, then bootstrap.sh --iface " + d.Name + case foreign > 0: + return v4 + " already has DHCP (" + strings.Join(d.DHCP, ",") + "): either the network team turns it off for this VLAN, or keep it and point it at this host (next-server " + strings.Split(d.Addrs[0], "/")[0] + ", or a DHCP relay); then bootstrap.sh --iface " + d.Name + " --no-dhcp" + default: + return "candidate provisioning network " + v4 + vlan + ": bootstrap.sh --iface " + d.Name + " (or Network -> New subnet in the web UI)" + } +} + +func skipIface(name string) bool { + for _, p := range []string{"docker", "veth", "virbr", "br-", "cni", "flannel", "tap", "vnet", "wg", "tun"} { + if strings.HasPrefix(name, p) { + return true + } + } + return false +} + +func ifaceKind(name string) string { + base := "/sys/class/net/" + name + switch { + case exists(base + "/bridge"): + return "bridge" + case exists(base + "/bonding"): + return "bond" + case exists("/proc/net/vlan/" + name): + return "vlan" + case exists(base + "/device"): + return "physical" + } + return "virtual" +} + +func exists(p string) bool { _, err := os.Stat(p); return err == nil } + +func defaultRouteIface() string { + b, err := os.ReadFile("/proc/net/route") + if err != nil { + return "" + } + for _, l := range strings.Split(string(b), "\n")[1:] { + f := strings.Fields(l) + if len(f) > 2 && f[1] == "00000000" && f[7] == "00000000" { + return f[0] + } + } + return "" +} + +func neighbourCount(iface string) int { + out, err := exec.Command("ip", "-j", "neigh", "show", "dev", iface).Output() + if err != nil { + return 0 + } + var ns []map[string]any + _ = json.Unmarshal(out, &ns) + n := 0 + for _, x := range ns { + if st, _ := x["state"].([]any); len(st) > 0 && st[0] != "FAILED" && st[0] != "INCOMPLETE" { + n++ + } + } + return n +} + +// lldpNeighbours: interface -> [switch name, port, vlan] from lldpd. +func lldpNeighbours() map[string][3]string { + out := map[string][3]string{} + bin, err := exec.LookPath("lldpcli") + if err != nil { + return out + } + b, err := exec.Command(bin, "-f", "json0", "show", "neighbors", "details").Output() + if err != nil { + return out + } + var doc struct { + LLDP []struct { + Interface []struct { + Name string `json:"name"` + Chassis []struct { + Name []struct { + Value string `json:"value"` + } `json:"name"` + } `json:"chassis"` + Port []struct { + ID []struct { + Value string `json:"value"` + } `json:"id"` + Descr []struct { + Value string `json:"value"` + } `json:"descr"` + } `json:"port"` + VLAN []struct { + VlanID string `json:"vlan-id"` + PVID bool `json:"pvid"` + } `json:"vlan"` + } `json:"interface"` + } `json:"lldp"` + } + if json.Unmarshal(b, &doc) != nil { + return out + } + for _, l := range doc.LLDP { + for _, i := range l.Interface { + var sw, port, vlan string + if len(i.Chassis) > 0 && len(i.Chassis[0].Name) > 0 { + sw = i.Chassis[0].Name[0].Value + } + if len(i.Port) > 0 { + if len(i.Port[0].Descr) > 0 { + port = i.Port[0].Descr[0].Value + } else if len(i.Port[0].ID) > 0 { + port = i.Port[0].ID[0].Value + } + } + var vs []string + for _, v := range i.VLAN { + s := v.VlanID + if v.PVID { + s += "(untagged)" + } + vs = append(vs, s) + } + vlan = strings.Join(vs, ",") + out[filepath.Base(i.Name)] = [3]string{sw, port, vlan} + } + } + return out +} diff --git a/cmd/rollcall/doctor.go b/cmd/rollcall/doctor.go index 19755ab..c63ec41 100644 --- a/cmd/rollcall/doctor.go +++ b/cmd/rollcall/doctor.go @@ -98,8 +98,10 @@ func doctorCmd(args []string) error { } if ports[fmt.Sprintf("%s/%d", p.proto, p.port)] { d.ok(fmt.Sprintf("%s/%d", p.proto, p.port), p.what) + } else if p.port == 69 && !ports["udp/67"] { + d.ok("udp/69", "TFTP starts with the first VLAN served") } else if p.port == 67 { - d.warn("udp/67", "nothing listens for DHCP", "no VLAN with dhcp on this rack yet (rollcall vlan add FABRIC VID -dhcp -rack "+cfg.RackName+"), or dnsmasq is down") + d.warn("udp/67", "this host serves no DHCP yet", "no provisioning network decided: rollcall net discover, then deploy/bootstrap.sh --iface IFACE (or a VLAN with dhcp in the web UI)") } else { d.fail(fmt.Sprintf("%s/%d", p.proto, p.port), "nothing listens for "+p.what, "systemctl status rollcall dnsmasq") } @@ -189,7 +191,7 @@ func doctorCmd(args []string) error { } else { d.ok("dns (host)", "resolves outside names") } - if srvHost != "" && ports["udp/53"] { + if srvHost != "" && ports["udp/53"] && ports["udp/67"] { r := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "udp", net.JoinHostPort(srvHost, "53")) }} @@ -211,8 +213,10 @@ func doctorCmd(args []string) error { } // other DHCP servers on the provisioning network - if iface != "" && !*noProbe { - servers, err := dhcpProbe(iface, 4*time.Second) + if iface != "" && !*noProbe && !ports["udp/67"] { + d.ok("dhcp probe", "skipped: this host serves no DHCP yet") + } else if iface != "" && !*noProbe { + servers, err := dhcpProbe(iface, 6*time.Second) switch { case err != nil: d.warn("dhcp probe", err.Error(), "run as root; or --no-dhcp-probe") diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index 1eacf77..9c253eb 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -28,6 +28,7 @@ func usage() { rollcall tui [-dump WxH -page P -machine ID -plain] rollcall serve [-config PATH] region: API, database, web UI, embedded rack controller rollcall doctor [-deep] [-no-dhcp-probe] check this installation and say what to fix + rollcall net discover [-json] what each interface of this host sees (LLDP, DHCP, neighbours) rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate) rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host) @@ -79,6 +80,9 @@ func newFlags(name string) *flag.FlagSet { return flag.NewFlagSet(name, flag.Exi func main() { log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds) + if len(os.Args) > 2 && os.Args[1] == "net" && os.Args[2] == "discover" { + os.Args = append([]string{os.Args[0], "net-discover"}, os.Args[3:]...) + } if len(os.Args) < 2 { // A bare `rollcall` in a terminal opens the console; in a pipe or a // script it prints its help, as it always did. @@ -108,6 +112,11 @@ func main() { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) } + case "net-discover": + if err := discoverCmd(args); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } case "doctor": if err := doctorCmd(args); err != nil { fmt.Fprintln(os.Stderr, "error:", err) diff --git a/deploy/bootstrap.sh b/deploy/bootstrap.sh index 2386b53..2e54596 100755 --- a/deploy/bootstrap.sh +++ b/deploy/bootstrap.sh @@ -1,31 +1,39 @@ #!/bin/bash -# Rollcall region on a fresh Debian 13 host, in one go. Run as root from a -# checkout (deploy/) or an unpacked release (rollcall + deploy/ + agent/). +# Rollcall region on a fresh Debian 13 host. Run as root from a checkout +# (deploy/) or an unpacked release (rollcall + deploy/ + agent/). # -# ./bootstrap.sh --iface eth1 [--name NAME] [--domain D] [--gateway IP | --nat UPLINK] -# [--image-url URL | --build-image] [--os debian-13] [--no-dhcp] [--yes] +# Day 0 - nothing about the network is decided yet: +# ./bootstrap.sh [--image-url URL | --build-image] [--name NAME] +# installs everything and starts the region on this host's management +# address: web UI and API work, no DHCP, no subnets. Then look around: +# rollcall net discover interfaces, switch ports (LLDP), DHCP servers, neighbours # -# --iface interface on the provisioning network; it must already have -# its IPv4 address (this host's address there becomes the server URL) -# --name rack controller name of this host (default: short hostname) -# --domain DNS domain handed to machines (default: rollcall.internal) -# --gateway default router for machines on the provisioning network -# --nat UPLINK make this host the router: forward + masquerade out of UPLINK -# (default when --gateway is not given: the interface of the default route) -# --image-url directory with vmlinuz, initrd.img, filesystem.squashfs and SHA256SUMS +# Later - once the provisioning network is known (run again, nothing is lost): +# ./bootstrap.sh --iface IFACE [--gateway IP | --nat UPLINK] [--domain D] [--no-dhcp] +# IFACE must have its address on that network; this host's address there +# becomes the server URL machines use, the subnet is modelled with ranges +# and (unless --no-dhcp) this host serves DHCP/PXE on it. +# Or do the same from the web UI / CLI (rollcall subnet add, vlan add -dhcp), +# or import the prefixes from NetBox (rollcall ipam import). +# +# --name NAME rack controller name of this host (default: short hostname) +# --image-url directory with vmlinuz, initrd.img, filesystem.squashfs, SHA256SUMS # --build-image build the commissioning image here with live-build (~20 min) -# --os NAME import one OS image now (e.g. debian-13); the nightly sync does the rest -# --no-dhcp model the network but do not serve DHCP (another DHCP server is in charge) -# -# Safe to run again: existing configuration, database and image are kept. +# --gateway IP default router for machines on the provisioning network +# --nat UPLINK this host routes for them: forward + masquerade out of UPLINK +# (default with --iface and no --gateway: the default route's interface) +# --domain D DNS domain handed to machines (default rollcall.internal) +# --no-dhcp model the network but leave DHCP to the existing server +# --os NAME import one OS image now (e.g. debian-13); a nightly timer syncs the catalog +# --yes do not ask set -euo pipefail HERE=$(cd "$(dirname "$0")" && pwd) ROOT=$(cd "$HERE/.." && pwd) -IFACE="" NAME=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0 +IFACE="" RACK=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0 while [ $# -gt 0 ]; do case "$1" in --iface) IFACE=$2; shift ;; - --name) NAME=$2; shift ;; + --name) RACK=$2; shift ;; --domain) DOMAIN=$2; shift ;; --gateway) GATEWAY=$2; shift ;; --nat) NAT=$2; shift ;; @@ -34,7 +42,7 @@ while [ $# -gt 0 ]; do --os) OS=$2; shift ;; --no-dhcp) DHCP=0 ;; --yes|-y) YES=1 ;; - -h|--help) sed -n '2,24p' "$0"; exit 0 ;; + -h|--help) sed -n '2,31p' "$0"; exit 0 ;; *) echo "unknown option $1 (--help)" >&2; exit 2 ;; esac shift @@ -44,40 +52,29 @@ say() { printf '\n\033[1m== %s\033[0m\n' "$*"; } ok() { printf ' \033[32mOK\033[0m %s\n' "$*"; } die() { printf ' \033[31mERROR\033[0m %s\n' "$*" >&2; exit 1; } [ "$(id -u)" = 0 ] || die "run as root" -. /etc/os-release -[ "$ID" = debian ] || echo "note: tested on Debian 13; this is $PRETTY_NAME" >&2 - -# ------------------------------------------------------------------ 0. what and where -say "0/9 provisioning network" -if [ -z "$IFACE" ]; then - def=$(ip -4 route show default | awk '{print $5; exit}') - cand=$(ip -4 -o addr show scope global | awk -v d="$def" '$2 != d {print $2}' | sort -u) - [ "$(echo "$cand" | grep -c .)" = 1 ] || die "say which interface faces the machines: --iface IFACE (candidates: $(echo $cand))" - IFACE=$cand -fi -CIDR_ADDR=$(ip -4 -o addr show dev "$IFACE" scope global | awk '{print $4; exit}') -[ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; configure it first (netplan / ifupdown)" -read -r ADDR NET PREFIX FIRST <<<"$(python3 - "$CIDR_ADDR" <<'P' -import ipaddress, sys -i = ipaddress.ip_interface(sys.argv[1]) -print(i.ip, i.network, i.network.prefixlen, i.network.network_address) -P -)" -[ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; create the subnet and ranges by hand" -off() { python3 -c "import ipaddress,sys;print(ipaddress.ip_address('$FIRST')+int(sys.argv[1]))" "$1"; } +OSNAME=$(. /etc/os-release; echo "$ID $VERSION_ID") +case "$OSNAME" in debian*) ;; *) echo "note: tested on Debian 13; this is $OSNAME" >&2 ;; esac +CONF=/etc/rollcall/config.json UPLINK=$(ip -4 route show default | awk '{print $5; exit}') -if [ -z "$GATEWAY" ]; then - GATEWAY=$ADDR - NAT=${NAT:-$UPLINK} +addr_of() { ip -4 -o addr show dev "$1" scope global | awk '{print $4; exit}'; } + +# ------------------------------------------------------------------ 0. where +say "0/9 this host" +if [ -n "$IFACE" ]; then + CIDR_ADDR=$(addr_of "$IFACE") + [ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; give it its address on the provisioning network first" + read -r ADDR NET PREFIX FIRST <<<"$(python3 -c 'import ipaddress,sys;i=ipaddress.ip_interface(sys.argv[1]);print(i.ip,i.network,i.network.prefixlen,i.network.network_address)' "$CIDR_ADDR")" + [ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; model it by hand (rollcall subnet add / range add)" + if [ -z "$GATEWAY" ]; then GATEWAY=$ADDR; NAT=${NAT:-$UPLINK}; fi + [ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not $IFACE" + echo " provisioning $IFACE $CIDR_ADDR (subnet $NET), machines' gateway $GATEWAY${NAT:+ via NAT out of $NAT}, DHCP $([ $DHCP = 1 ] && echo on || echo off)" +else + [ -n "$UPLINK" ] || die "no default route: give this host network access first" + ADDR=$(addr_of "$UPLINK" | cut -d/ -f1) + echo " no provisioning network yet: the region starts on $UPLINK ($ADDR) without DHCP or subnets" fi -[ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not the provisioning interface" SERVER=http://$ADDR -cat </dev/null apt-get install -yq $PKGS >/dev/null -# until rollcall writes its configuration, dnsmasq must not grab port 53 on every address install -d /etc/dnsmasq.d -[ -e /etc/dnsmasq.d/00-rollcall-base.conf ] || printf '# rollcall bootstrap: bind only where configured (systemd-resolved keeps 127.0.0.53)\nbind-dynamic\n' > /etc/dnsmasq.d/00-rollcall-base.conf +# until rollcall serves a VLAN, dnsmasq must not answer DNS/DHCP everywhere +# (rollcall's generated configuration adds the interfaces of the VLANs it serves) +printf '# rollcall bootstrap: answer only on loopback until rollcall serves a VLAN\n# (systemd-resolved keeps 127.0.0.53)\nbind-dynamic\ninterface=lo\n' > /etc/dnsmasq.d/00-rollcall-base.conf systemctl restart dnsmasq || true -ok "$(echo $PKGS | wc -w) packages" +ok "installed (lldpd shows which switch port and VLAN each interface is on)" # ------------------------------------------------------------------ 2. database say "2/9 PostgreSQL" systemctl enable --now postgresql >/dev/null -CONF=/etc/rollcall/config.json +PGPASS="" if [ -f "$CONF" ]; then - ok "config exists, database settings kept" + ok "configured before, kept" else - PGPASS=$(head -c 24 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28) - if [ "$(su postgres -c "psql -tAc \"select 1 from pg_roles where rolname='rollcall'\"")" = 1 ]; then - su postgres -c "psql -q -c \"ALTER ROLE rollcall PASSWORD '$PGPASS'\"" + PGPASS=$(head -c 32 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28) + if [ "$(runuser -u postgres -- psql -tAc "select 1 from pg_roles where rolname='rollcall'")" = 1 ]; then + runuser -u postgres -- psql -q -c "ALTER ROLE rollcall PASSWORD '$PGPASS'" else - su postgres -c "psql -q -c \"CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'\"" + runuser -u postgres -- psql -q -c "CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'" fi - [ "$(su postgres -c "psql -tAc \"select 1 from pg_database where datname='rollcall'\"")" = 1 ] || su postgres -c "psql -q -c 'CREATE DATABASE rollcall OWNER rollcall'" + [ "$(runuser -u postgres -- psql -tAc "select 1 from pg_database where datname='rollcall'")" = 1 ] || runuser -u postgres -- psql -q -c "CREATE DATABASE rollcall OWNER rollcall" ok "role and database rollcall" fi @@ -139,36 +137,55 @@ ok "$("$HERE/rollcall" version)" say "5/9 configuration" getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall install -d -m 0750 -o root -g rollcall /etc/rollcall +FQDN=$(hostname -f 2>/dev/null || hostname) +CHANGED=0 if [ ! -f "$CONF" ]; then AGENT=$("$HERE/rollcall" keygen) ADMIN=$("$HERE/rollcall" keygen) - FQDN=$(hostname -f 2>/dev/null || hostname) - python3 - "$CONF" <

/dev/null) -for i in $(seq 1 30); do curl -sf -o /dev/null http://127.0.0.1/ca.pem && break; sleep 1; done +(cd "$HERE" && sh ./install.sh >/dev/null) +for _ in $(seq 1 30); do curl -sf -o /dev/null http://127.0.0.1/ca.pem && break; sleep 1; done curl -sf -o /dev/null http://127.0.0.1/ca.pem || die "rollcall did not come up: journalctl -u rollcall" ok "rollcall running, CA $(rollcall ca show | awk '/fingerprint/{print $2}')" @@ -180,9 +197,9 @@ if [ -s $IMG/vmlinuz ] && [ -s $IMG/initrd.img ] && [ -s $IMG/filesystem.squashf elif [ -n "$IMAGE_URL" ]; then tmp=$(mktemp -d -p /srv/rollcall) curl -fsSL -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS" || die "$IMAGE_URL/SHA256SUMS missing" - for f in vmlinuz initrd.img filesystem.squashfs; do curl -fL --progress-bar -o "$tmp/$f" "$IMAGE_URL/$f"; done - (cd "$tmp" && grep -E ' (\*)?(vmlinuz|initrd.img|filesystem.squashfs)$' SHA256SUMS | sha256sum -c --quiet -) || die "checksum mismatch" - mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rmdir "$tmp" 2>/dev/null || rm -rf "$tmp" + for f in vmlinuz initrd.img filesystem.squashfs; do curl -fsSL -o "$tmp/$f" "$IMAGE_URL/$f"; done + (cd "$tmp" && sha256sum -c --quiet SHA256SUMS) || die "checksum mismatch" + mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rm -rf "$tmp" chmod 0644 $IMG/* ok "downloaded and verified from $IMAGE_URL" elif [ $BUILD = 1 ]; then @@ -191,34 +208,42 @@ elif [ $BUILD = 1 ]; then (cd $IMG && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS) ok "built" else - echo " no image yet: rerun with --image-url URL or --build-image (machines cannot commission without it)" >&2 + echo " not yet: rerun with --image-url URL or --build-image before machines commission" >&2 fi -# ------------------------------------------------------------------ 8. network model -say "8/9 network model" -FAB=provisioning -rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null -if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then - rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null - rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null - rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null - rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null - ok "subnet $NET with ranges" +# ------------------------------------------------------------------ 8. provisioning network +say "8/9 provisioning network" +if [ -z "$IFACE" ]; then + echo " not decided yet - nothing modelled, no DHCP served. When it is:" + echo " rollcall net discover what each interface sees (LLDP, DHCP servers, neighbours)" + echo " $0 --iface IFACE ... or the web UI: Network -> New subnet" else - ok "subnet $NET exists, kept" -fi -if [ $DHCP = 1 ]; then - rollcall vlan add "$FAB" 0 -dhcp -rack "$NAME" >/dev/null - ok "DHCP/PXE on $IFACE by rack controller $NAME" -fi -if [ -n "$NAT" ]; then - echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf - sysctl -q -p /etc/sysctl.d/60-rollcall-forward.conf - install -d /etc/nftables.d - printf 'table ip rollcall_nat {\n chain postrouting {\n type nat hook postrouting priority 100;\n oifname "%s" ip saddr %s masquerade\n }\n}\n' "$NAT" "$NET" > /etc/nftables.d/rollcall-nat.nft - grep -q 'include "/etc/nftables.d/\*.nft"' /etc/nftables.conf 2>/dev/null || echo 'include "/etc/nftables.d/*.nft"' >> /etc/nftables.conf - nft -c -f /etc/nftables.conf && systemctl enable --now nftables >/dev/null && systemctl reload nftables - ok "forwarding + masquerade $NET out of $NAT" + [ $CHANGED = 1 ] && systemctl restart rollcall && sleep 2 + off() { python3 -c "import ipaddress,sys;print(ipaddress.ip_address('$FIRST')+int(sys.argv[1]))" "$1"; } + FAB=provisioning + rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null + if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then + rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null + rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null + rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null + rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null + ok "subnet $NET: reserved $(off 1)-$(off 20), static $(off 50)-$(off 99), dynamic $(off 100)-$(off 200)" + else + ok "subnet $NET exists, kept" + fi + if [ $DHCP = 1 ]; then + rollcall vlan add "$FAB" 0 -dhcp -rack "$RACK" >/dev/null + ok "DHCP/PXE on $IFACE" + fi + if [ -n "$NAT" ]; then + echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf + sysctl -q -p /etc/sysctl.d/60-rollcall-forward.conf + install -d /etc/nftables.d + printf 'table ip rollcall_nat {\n chain postrouting {\n type nat hook postrouting priority 100;\n oifname "%s" ip saddr %s masquerade\n }\n}\n' "$NAT" "$NET" > /etc/nftables.d/rollcall-nat.nft + grep -q 'include "/etc/nftables.d/\*.nft"' /etc/nftables.conf 2>/dev/null || echo 'include "/etc/nftables.d/*.nft"' >> /etc/nftables.conf + nft -c -f /etc/nftables.conf && systemctl enable nftables >/dev/null 2>&1 && systemctl restart nftables + ok "forwarding + masquerade $NET out of $NAT" + fi fi # ------------------------------------------------------------------ 9. OS images @@ -226,15 +251,15 @@ say "9/9 OS images" if [ -n "$OS" ]; then runuser -u rollcall -- rollcall os sync "$OS" || echo " os sync $OS failed; the nightly timer retries" >&2 else - echo " none imported now; rollcall-os-sync.timer fetches the catalog nightly (or: runuser -u rollcall -- rollcall os sync debian-13)" + echo " none imported now; rollcall-os-sync.timer syncs the catalog nightly (now: runuser -u rollcall -- rollcall os sync debian-13)" fi cat <