From 736a7e38a3e074174c2b03fdb445adbf0453db56 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Tue, 27 Oct 2026 23:58:40 +0100 Subject: [PATCH] Network model, IPAM, interface layouts and the region/rack split (MAAS-style) - fabrics, VLANs, subnets, IP ranges (dynamic/reserved/static), IPAM with adoption of addresses machines already have; rollcall fabric|vlan|subnet|range|ip - interface layouts (physical/bond/vlan, NICs by selector: boot, name, mac, fastest:N, ib:N) per machine or per tag; deployment allocates addresses and writes cloud-init network-config plus udev name rules (RHEL cloud images boot with net.ifnames=0); release frees them; rollcall layout, rollcall net show|set - region/rack: rollcall rack long-polls its desired state (per-MAC PXE files kept in the database, dnsmasq configuration and host files) and a BMC command queue, caches boot and OS images (kernels/initrds prefetched for TFTP), proxies the node API and tags requests with its name; the region runs an embedded rack controller; install-rack.sh, rollcall-rack.service - dnsmasq configuration generated per rack from the VLANs it serves (split pool for two racks); reload script restarts only on change and restores the last working config if dnsmasq does not come back (drilled in the lab) - web: Network page, per-machine network tab, layout action, tag layout; console: Network page, network section in the machine view, layout action - tests for candidates, plans (default, bond+VLAN, pinned, DHCP fallback), network-config, udev rules and dnsmasq rendering - lab: pve-vbmc address script finds the interface per BMC address Verified in the lab: embedded rack (10.213.77.0/24) and a remote rack controller on a second SDN segment (10.213.78.0/24): enlistment, commissioning, BMC through the rack's queue, image cache, deployment and phone-home through the rack; Debian 13, Ubuntu 24.04 and Rocky 9 with a static boot NIC and with an active-backup bond carrying VLAN 200. Co-Authored-By: Claude Opus 5.5 --- .gitignore | 1 + CHANGELOG.md | 12 + README.md | 110 +- .../rollcall-agent.cpython-314.pyc | Bin 36048 -> 0 bytes .../rollcall-deploy.cpython-314.pyc | Bin 28126 -> 0 bytes cmd/rollcall/client.go | 7 +- cmd/rollcall/config.go | 8 + cmd/rollcall/deploy.go | 76 +- cmd/rollcall/ipmi.go | 19 - cmd/rollcall/main.go | 18 +- cmd/rollcall/netapi.go | 848 ++++++++++++ cmd/rollcall/network.go | 1214 +++++++++++++++++ cmd/rollcall/network_test.go | 165 +++ cmd/rollcall/rackctl.go | 867 ++++++++++++ cmd/rollcall/rackd.go | 414 ++++++ cmd/rollcall/server.go | 99 +- cmd/rollcall/stateless.go | 99 +- cmd/rollcall/store.go | 86 +- cmd/rollcall/tags.go | 11 +- cmd/rollcall/tui.go | 145 +- cmd/rollcall/web/app.js | 154 ++- cmd/rollcall/web/index.html | 1 + deploy/config.example.json | 2 + deploy/install-rack.sh | 56 + deploy/install.sh | 15 +- deploy/rack.example.json | 12 + deploy/rollcall-dnsmasq-reload.sh | 53 + deploy/rollcall-dnsmasq.path | 3 +- deploy/rollcall-dnsmasq.service | 4 +- deploy/rollcall-rack.service | 21 + lab/pve-vbmc/pve-vbmc-addrs.sh | 13 +- 31 files changed, 4357 insertions(+), 176 deletions(-) delete mode 100644 agent/__pycache__/rollcall-agent.cpython-314.pyc delete mode 100644 agent/__pycache__/rollcall-deploy.cpython-314.pyc create mode 100644 cmd/rollcall/netapi.go create mode 100644 cmd/rollcall/network.go create mode 100644 cmd/rollcall/network_test.go create mode 100644 cmd/rollcall/rackctl.go create mode 100644 cmd/rollcall/rackd.go create mode 100644 deploy/install-rack.sh create mode 100644 deploy/rack.example.json create mode 100644 deploy/rollcall-dnsmasq-reload.sh create mode 100644 deploy/rollcall-rack.service diff --git a/.gitignore b/.gitignore index 93e3748..eae723b 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ *.key /site/dist/ .DS_Store +__pycache__/ diff --git a/CHANGELOG.md b/CHANGELOG.md index 2993c84..edd8cd9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,18 @@ publish a tag that has no section here. with elevation, jobs, events, tags, profiles, OS images; connect screen that saves `~/.config/rollcall/client.json`, which the CLI also reads; `rollcall tui -dump` renders one screen. First Go tests (console rendering and keys). +- Network model after MAAS: fabrics, VLANs, subnets, IP ranges (dynamic/reserved/static) and + IPAM; addresses already in use are adopted. `rollcall fabric|vlan|subnet|range|ip`. +- Interface layouts (physical, bond, VLAN; NICs by selector) per machine or per tag; deployment + writes cloud-init `network-config` and udev name rules into the image; release frees addresses. + `rollcall layout`, `rollcall net show|set`. +- Region/rack split: `rollcall rack` (dnsmasq state, per-MAC PXE files, BMC commands, image cache, + node API proxy) long-polls the region; the region runs an embedded rack controller. + `deploy/install-rack.sh`, `rollcall controllers`, `rollcall dhcp`. +- dnsmasq configuration generated from the model, restarted only when it changes and rolled back + if dnsmasq does not come back. +- Web UI: Network page (rack controllers, subnets with ranges and usage, layouts), network tab per + machine, layout action; console: Network page, network section, layout action. - `rollcall version`; CD-built binary carries `git describe`. - Release notes from `CHANGELOG.md`. diff --git a/README.md b/README.md index 5d6332a..ad7d9d0 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,7 @@ unknown server ──PXE──▶ dnsmasq (DHCP+TFTP) ──▶ pxelinux default |---|---| | `cmd/rollcall/` | Go server + CLI + embedded web UI (`web/`) | | `agent/` | `rollcall-agent.py` (runs inside the live image), its systemd unit, `build-image.sh` (live-build recipe) | -| `deploy/` | `install.sh`, `rollcall.service`, `config.example.json`, `pxelinux-default`, `os-sources.json`, sync/dnsmasq/controller units, `overlays/` | +| `deploy/` | `install.sh` (region), `install-rack.sh` + `rack.example.json` (rack controller), units, `config.example.json`, `os-sources.json`, `tags/`, `overlays/` | | `images/` | stateless root image recipes (`build-rootfs.sh`, `compute-debian13/`) | | `lab/` | Proxmox test lab: `pve-vbmc` (virtual IPMI BMC for VMs backed by the Proxmox API), cloud-init for the PXE server VM | | `site/` | Project website (static) | @@ -295,6 +295,114 @@ tag rules see `.Node.Building`, `.Node.Room`, `.Node.Rack`, `.Node.Unit`, so a rack can drive a tag (`{{ eq .Node.Rack "R12" }}`) or a Slurm `topology.conf` overlay. +## Network and rack controllers + +### Model + +As in MAAS: **fabric → VLAN → subnet → IP ranges**. + +* A **fabric** is a switched layer-2 domain (`ethernet` or `infiniband`). +* A **VLAN** has a VID (0 = untagged; a partition key on IB), an MTU, and + says whether rollcall serves DHCP/PXE on it and which **rack controller** + does (a second one splits the dynamic pool with it). +* A **subnet** has a purpose (provisioning, mgmt, bmc, storage, ib, other), a + gateway, DNS servers and a domain. +* **Ranges** inside a subnet are `dynamic` (the DHCP pool PXE and + commissioning use), `reserved` (never handed out: routers, switches, + controllers) and `static` (where machines get their addresses; without one, + anything outside dynamic/reserved). +* **IPAM** records every address handed out (`rollcall ip list`), pinned ones + (`rollcall net set ID -static ens18=10.0.0.60`) and reserved single + addresses (`rollcall ip reserve 10.0.0.5 -name switch-1`). Addresses a machine + already had are adopted when its subnet is defined. + +```sh +rollcall fabric add hall-a +rollcall subnet add 10.50.0.0/24 -fabric hall-a -vid 0 -purpose provisioning -gateway 10.50.0.1 -dns 10.50.0.1 -domain dc.example +rollcall range add 10.50.0.0/24 reserved 10.50.0.1 10.50.0.20 +rollcall range add 10.50.0.0/24 static 10.50.0.50 10.50.0.199 +rollcall range add 10.50.0.0/24 dynamic 10.50.0.200 10.50.0.250 +rollcall vlan add hall-a 0 -dhcp -rack hall-a-rack1 # this rack controller serves DHCP/PXE here +rollcall subnet add 10.60.0.0/24 -fabric hall-a -vid 200 -purpose storage +rollcall dhcp hall-a-rack1 # the dnsmasq configuration it generates +``` + +### Machine interfaces + +A **layout** describes interfaces in terms that fit many machines: NICs are +chosen by selectors (`boot`, `name:X`, `mac:X`, `fastest:N`, `ib:N`, `all`), +not by MAC. Interfaces are `physical`, `bond` (802.3ad by default) or `vlan`; +each link names a subnet (CIDR, name, or `boot`) and a mode: `auto` (an address +from the subnet), `static` (a given one), `dhcp` or `link_up`. + +```json +{"interfaces": [ + {"type": "physical", "select": "boot", "links": [{"subnet": "boot", "mode": "auto", "default_gateway": true}]}, + {"type": "bond", "name": "bond0", "members": ["fastest:2"], "mtu": 9000, "links": [{"mode": "link_up"}]}, + {"type": "vlan", "parent": "bond0", "vid": 200, "links": [{"subnet": "10.60.0.0/24", "mode": "auto"}]}, + {"type": "physical", "select": "ib:0", "links": [{"subnet": "ib0", "mode": "auto"}]} +]} +``` + +`rollcall layout add gpu-bond layout.json`, then either per machine +(`rollcall net set ID -layout gpu-bond`) or per tag (`rollcall tags add gpu +… -layout gpu-bond`). Without a layout the boot NIC gets an `auto` address on +its subnet; without any subnet model, deployed machines use DHCP as before. + +At deployment the layout is resolved against the machine's inventory, +addresses are allocated, and two files go into the image next to the +cloud-init seed: `network-config` (cloud-init v2; cloud-init renders it for +netplan, NetworkManager or ifupdown) and a udev rule that pins each NIC's name +to its MAC, because RHEL-family cloud images boot with `net.ifnames=0`. +Release returns the allocated addresses. `rollcall net show ID` previews it all. + +Verified in the lab: Debian 13 and Ubuntu 24.04 (netplan) and Rocky 9 +(NetworkManager), with a static boot NIC and with an active-backup bond +carrying VLAN 200. + +### Region and rack controllers + +`rollcall serve` is the **region**: database, API, web UI, IPAM, image store. +**Rack controllers** sit next to the machines, one or two per network segment +(hall, building, site): + +* they serve DHCP and TFTP with dnsmasq on the VLANs assigned to them; the + configuration, per-MAC PXE files and host files are computed by the region; +* they run BMC commands (BMC networks are rarely routed to the region); +* they serve boot and OS images from a local cache, filled from the region on + first use (kernels and initrds are prefetched, TFTP cannot wait); +* they pass the node API (commissioning, deploy jobs, overlays, phone-home) + through to the region and tell it which rack a machine came through. + +A rack controller only needs to reach the region: it long-polls its desired +state and a command queue. The region host runs an **embedded** rack +controller (`rack_name`, default its hostname), which is all a single-server +installation needs. Remote ones: + +```sh +# region: config.json gets "rack_secret": "" +# rack host (Debian/Ubuntu), from deploy/ with the binary next to it: +cat > /etc/rollcall/rack.json <", + "url": "http://10.213.78.1", "listen": "10.213.78.1:80"} +EOF +sh install-rack.sh # dnsmasq, syslinux, cache dirs, rollcall-rack.service +rollcall controllers # on the region: it shows up online +rollcall vlan add hall-b 0 -dhcp -rack hall-b +``` + +dnsmasq is restarted only when its generated configuration changes, and only +if it comes back; otherwise the last working configuration is restored (a +hand-written DHCP configuration for the same interface is the usual cause: +`install.sh` points it out). A machine that moved to another segment gets +`rollcall net set ID -rack NAME`. + +The lab runs both: `pxe-server` (embedded, 10.213.77.0/24) and `pxe-rack2` +(VM 213, remote, 10.213.78.0/24 on SDN vnet `pxenet2`), with node002 behind +pxe-rack2 and its virtual BMC on pxe-rack2. Rack ↔ region traffic is plain +HTTP in the lab; BMC passwords travel on it for queued commands, so put the +region behind TLS on real networks. + ## Lab `lab/` documents a Proxmox SDN test setup: an isolated vnet, a PXE server VM diff --git a/agent/__pycache__/rollcall-agent.cpython-314.pyc b/agent/__pycache__/rollcall-agent.cpython-314.pyc deleted file mode 100644 index 3027ddc58c4261d2aa3cf0c0f3a231eff9438dea..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 36048 zcmcJ&c~Dz-o-cZi7PKRY)nE)_2V=l^$Byk_7Vj7zu)GAD01{X+i+m)!C`*%`t{R-A z+c-&&m5Jv@nWSIHq;o^6`@XnecikpEJrieU?(Ks`B3kWw$<*YJms_u1IdSUtboYIA z-{<=~I!6KvJ2SW5vH9sNzkT`czwfUu(`M#y9Xb2?6aQBi$NdXEC|5?}=H4RDag&_D z1-Ubv!0+dSJp0uIHSAX#)Usb)P{)4tK|TA;2xhQfL(sr}jX@*(H3d!V*BmsnUrW%+ zer-YPX>C`LQhf5at0Z}^+zFaZs1;q+@(<^+a$FI2+EL008bN!87j#>-@~6X=)26Pn zM6{HjV3wfI=YrXSOUTIQM)+WkP%0SMT`um7>@H6z6U^)`U$7wdD#41dK(HY!6fzO6 z7VHR%1P8)30v{+AoVY6yvJkq2Y=osk4#F}a7h$=q7|->m8-$W=CpgYi$XagdI+N14#BC{ey1ta# zlz3A5tOVm&&i-nv`fq4Bu62BL)E5Z&#>aeP!>;~e?^tlVOPm-BxW>l@y)NHaz~}M0 zf@i%hum7xf)GPLnxJG;z@MRQ{8_ed`@sSblU@+k7cb)AQJs0~$N_7GGjEk4oxdz9_ zhJ3>lqL(7N?7z?^C`1%pAexQHMDzh^)lm1nM=~KZtkUK{LHd6D*V z5buU5HxV@*ME))+(m#IDD+XA@63JZSLqllKL9aOG9dQkgdPb0!Yuq11-vqXsU26IU zxl%jN27~_Xjg3v~8#Xj;Y}mTBp~>ZP)rj8n6F$-Fsa4~nIRUThm~bdHkFjx91C)tG z%dSAM|FUbs@2c^52PTGxQT^J~qywXa6R5(@L3QZ7qLM>u50CYaV&ukMgQB-T=uOQb zIDXDMwzHw3K}nYK%ccO5A%?3afMIm?2V5uWg+%Ao_t2Lm^%))acz1fd{*m#^YMUFL z7#~0ZG?UbK^>UmB|7Dt1L2tjuRdb}*``g@ZI2xhrCE4M-8=X=#yZoU^2UcLl=YvR6y zOTO*ZDc`#k7^u9!?*ZQ9p!JrTBSD;GPJ#|=3xBoxknOSL@SJv`-nt>*h_?qf`k|4D zz}e1PU0m-+GlFsLp(A_a+Q8*NJR=bFV20Of;@Z)`a9l?-FhKRJa>eu6z%(k8xSlFYX%N_jd$j{M zC1MNw@lrvvH4(?Jb!)q*jAfTwqk?s`sfa@?hNzZC+o&mjz!w+S3zQ&V8Jq_-DMIH6 z+6m1Xt~|9J)N9MGl=wP1zK`3XUBmf!3~HI+4Nizy7vkVLS7NLhT%81FTw=d3;0-kN z(6|Q9dPhd$nS=d)MECY#Nc|JRxGw0u6pR~!zELk5=F6vWBjzEfHO9?> zi2;DSL2n=s*JAm^&AtFFmwtd_%#xrOH;b$mgIIPV^-bK_fvMFw9_#>M^|XuPxERmu zkxOa6MZR-^Ox!rN~sVh&tmlHnv+KE?ANcJ6)amQ@a2ixA? zCg1<-9ZS(S9Iww`wC6pSdB>6)vsBDmDw416YVq-pfeMuP-4;VjKKG&7N@2dOWw-Xj zsw@hdwG{5+5e}wKK57D6^_{G~$(4DWSEk)X#23rmpja5jbS5TM>QqNsx)fZQz>k_Q z>7_g@8rD4+*r4e1*Xl%CYOFKk8gVerk0<&PID+~zuEqF@)Q=nR7pO!q$=$Kqr?pY* znrGW5_bjd|4ENtKOf_BIdS&bL&62J3v&7~}wde9Nk@0>{AZ|a< zeWbI^OEZ*B$69T&$3*Im0Cm4I^^lhOYaL!~rEj|spudcksn)2mV6x?|Myt>H%vJhA z@M`F}P`LE@rzK-uB!~cW%O10o%v(xmI_1Re*)*AQ{urPs{{1!e+&g?d_ik=WhW0}} zkB|-Kk`6W?BAY<}zzPMFOQH)-A)0pZef)%GLffZK5x>T*>3YBv>PkR4WU5}u^qK%1 z63~qTw<(cB2^)esm7pEQU=(pa4Rv=3`$vN$Lm>>6IzUNS)r0yGY*PkCtUAwlLoVOV zZpq?)SiDRB6Ekl9v!%su`p~4q?T2alx^2qw8k&Yv<` z=*`5)J!o7UT*m>9^Yd;#-3ST1l81tWhEZM-cMz-zhzFW0>`KNHG(6X3Qr;(j1+9|P zMdTos`*m)elBb8|*`qrQM&-DMpv1vL9B1N+fk9=`p^@N!V}b$=nG}@Y&>81<_SEXx zm;e^}M|?q`i(tPP3|#bKEVUyTY?B|)J|bFc^)&uLpb=1WaUTEs;{1gGbt9wqWLRZL zlp{Wh_j~ad=tlq&!eRsMV9%NEi`pw=_PTj{-KX~YDea;q^XjfEyTWxbOVxsI0$TMtp1>9O(}P8u6vk=b_2snHUWAQS^{&Kp?xrszIL$<`Lq5e_d$mLKTaXACyU;&r3G4=hDvhtJhmoz_=Gk6xQ z^97b>V^!V;FZ;crT;*wLQm6778F+*B!T#ZpWxzKcs0Tah85cv=!~<(x$jP1(97nwe zJW{QOAv3nXYPE6QIq&6wxD)M<^Bx8rv2Ns5V@Buq$sHi>LxiLF3rrxGdw7k$dy*liPv`CJssI~gGsrvn#$rj1D<@5Yi;WeM+SI6=NDPQ>U zW~u3Dv`(08nH-(hZyXOKznM?(iyr4d0oIpHar)tfion8GRc<;wmkmITkx;h^R zx6WiSBfdZoLK2t9N8+BEaY#q}V0_L(oHF9AZ4ld0UjV6KXUMRRy>rD4ee%~hzb?+3 z;{5iI#$>8B%ByTO;sbvX8e@#ExD78vA_3zW{eBQGGR-1x#~Uz4dJipqATuyg83QBZ zgXe(R1}8+3cs;gCMWP%#HVYW>@8E$*CJmP(0 zt&~|GH8)JQE@tISd!||!9og42uVsGfa82nJE%vLsuj~#t$1Ih%EtQMeMPbitPrUNP zO!lkeGyR_wua6aPnJ?ZFE#7w1`AP8(DeDoWw`N^EeC2Ss{8MZ3qRsI_UnJ+PlGjUS zb7G}i7D~6=+&EY9pLg5}{^mO$e@E)`#*Pdv92r{7E4=>1wI^O0iRG=E&s#Us9nEW! zk?G=^;@6J9a{SekQ?1tx*96RkVn(m`PgT}fmq zl=tQ;Z-t6{MyM*_TggqRbaRa)5Ul!=N{{QheIYO7LPMs`ao1r`Zjfsl0L=O!E{_`y zC2n*Oi=PWw_aJmB?@a!2|HKHW6i-J_To*t_kSGyE)b&tkq5D3MH`wnR30dfstBpMn zr{MK^;`(m(<4`5?o<8;=l;5hz=sm8(-cg9{U2;@wDzO^39!?=$;}&8w`eceVWC7v; zreNeNgqVz)4cWTcuku&%C=e~iC7gdQUBvJrpx zIuMkBv7Y35xDH56&O3!A6o&$^68Ig1e}=!rP3l2AKukuHm9!|EwGaww#K(|X=iA)d znz*K6y|@=o36ZT#DViREPQ`o4FYbyLpbhBVhDT_;M*_PV)KA}`RDh-2m#C9e*73G? z#(8t~2hY6!%v()=-Sm$44*^;j-*uGo*UhC6A+uK2S{NdWYlpo-8FwB*BuJ#ZGzR_R z;8~ed)RK@{KP-+<_(h^xAVr)Q1sfTldwxI!ImE?{a<`rvi1VY%S;P#v-s01U2$0GJ zLSVJQAzCvh+wa)&!O!qz7u&emcQ|K{>DYC z^Xj21hr+dyqi-F1{n$)%v~u(8g`4MpIq|cJxiit-J(9a$8ayi%_!g{B{MMSYXtZ6m zUa>A(bEnT=pSU&=J{irgp2?eaevtcq?(8$srjA<{>DWmi*y*-#RiyN-%GWDrGTx}4 zT{Fjjr2kMqHyVAkXVIEHef0XVYsbP{UpgaIZoTRJW$w>%rQIi@%}7}oZh5W!mG($O zw5Vxz=UnMWl^<5#vPJi}7p*zd{nx$Myx}8JAmE|d{tvwGdvA99WOVMd)N?G>b8^1t zq;zUH+H>}RG?ikHiP0yL6tg0XDXU3y7yQ0S3lP#Ya zjCZp0(433cE{4y)6nbg*lyT8o5Z)H?{MGK6L%%4$>HYcIMeCXyfq9w((Xx(cNyjaN zbkq&Gmh}XG7gDYh{QX$ofI*?jeeck6LLU>gK^69;Fu;gS;yjX};ExBEOow|xFDS4T zr2bjZ2f+PHnd;+Nf_4e9o&q;fBX%Vj0hu*UTb4dq*8TNtfDoXg0_8(%u&n&-e{1;% zQ3+bY2cM9C!RS~ZXaMejph2GyG(h5W{+OFjA@adg96y?_=5Ccjf%u_V&eNaa*Sj;) z(m5G|LFv=`+bJ0QhF+C50(qZ?m06Bs+!=zA`ZWbJnt+WSnAV|6G8IgPT+D2AGE~8Z z9)b%iM7n`N|HL0P^#|fQ@W}npU-`Tv9`P`Wj2oeF#XZDM`i_zQ;Q&Kz0Pl3)$Mn4E z0q_XghgtL6eeZXALNmXzSNS#fI% zAhRzBojG2FGFrQi)dXroSv_Y-;EG$uR1J>fTUIMV7b$DMW@!%U{ z52iy+9@ilB>E4j>&<6ExqkI?7bjvDtU#OoXjpFyvAGHQ1!WVq z_`1Nj2m%J&9RCC~qM&BtJoLJQaoxrKv7jt!(W0A3Ns1}NuwH_TQ&cM$6)}P$zKg$r zgCU|jM(fn@ba1LKYFraDmP^L+NY-Qv6JRddGOr%Ha%_5Qc+aP{5@4z7nwh*GU7pfS zUHty3n5}r;RvZpQZRPXEibZGE_1tT@)1kA6hT*G7%?36lF{a1}LC^XAt#h5Ki9Z#TcN z3woZjSI$nKfBxKayI$Wk*(#YgFIw$a_g>jMy%*Th>WW#bBx_Y<5722rQTW(vkG=BP ztEZy{b(5`Aws~FNoxGy2*hHJrzh`;d@?+aQ_JlBhj7J>vvqwI+WKJIV!`BwhvL!(L z@UL9Pwo=`%>T?ke;#s|j3}vVy2Y@zq zjbUFQMk*_uz7&U&N4grPt(Pi)6r3xRn)PqaGnL&=Ur&|Y{?_%(Q)(gPuTW2wLI2iy zu2S;+)+tmLztM195Ok?Ch6n>N>cEq1eyRkDLW*Do7$!6gq@e-Zeg2|Qn3`Hxz2vEg z^zeyU1#Zf1=+qosBj@Zl`VGPwHxD*nC4Af}OcF{`NE{bP9L##921;uMEg1cDbF@^A zQRdbQ@VgzUZT@4QIXmBf+zm<5pHiAGCeJ3D7fs(r|df4z=II}eYXV|i7s6d_miQw|%eZennF^H z#f!K-i+RS3gkoi?vk86xuCClwk1>`XEx07XZ=c;$@0#l3V=?da;*+ofJ^E;)e+0d_Z*3uYNEn zjsyY(E)*gzLB$yo110_z-+qX{z(GtqGLmuJc5aATH^!{ZlC^nOoNQ-OhDBS})st6F zPCpY7Keep|8E3Ix-F#*9bl&qjCD-}|%X*owzj|2wBK9p{PXulvRUmG*5%L)6k3vuu_un`bvnwk4z^SyMb3HN6(vzqXl*S$?+a zKba)w_PIR^*1a+7Vaa;<)`ltmGmG^~)8~%j2>+J;b$w(k>e@UTnrr%K^M{+I{f|Xk zdlwxg5$9XEujfX-6D{3(bImR1Z*o7*z4df-|LNHNK?(o+2V?ur&hI-b`NpIB{ELo) z6xl|sXscATbykS2eq?_2BR5Y(S9dHrT#=@?Hov}krX*V494p@~mG8bO#LC*{%i89S zN6QW`I!a|hja2tYv~p*xvQ?^Voy&?{tsW5(y;vD>tfE<#s{)6seV;ecz`qCs?Iy0*L|$XKsboE>IGCKlfs-# z3acNy1|~aJpJ6koq7>4EBqb_D5;t;UDdZ;v2$x+7auT!+{JNBHAdmq^nFSxO&XpjgQ;ZlD6rhj*ChBUdE z|J}J6)8uAxo0g4{sViHqe`=W2IluzQEFrs5pRPyTRv~8z zDV3}Ciw)oM7-0oLF=mHV$YXbgpNo|LGkMS!4 z(e>~VOspVbO^;P5?y-TGBR-W~UH{g6Q^!)t*Y&@UU+QQo`IWlO%SO}u(9z_CGPh+} zD$BP>1#&dqcncNkC{(TxLuFTm>J?(tD0wtmlsl$%o0nIs+YkT=4h>apC(*q?_#tC` z+lA55v9WV9iA#dMP*(3_HIJ>YZ|<)jI{oCPr)zspSH$(LL^Q@TWWiHBLnf5sy7jok zpCc8+?w}+gl^gR0FOG}n@URj33jYLd21DkA_09k&TFv?`(CUjm&#+hg8Pdk}`Hd8!G_d1jx5WDFct%+r} z4B~&FcY7(Kezb3J1j4aV@C*>SjuLGK8;FsB`2XN@fWWItXt5inf)YQa3@G^DDb^}R zX?4~ z$vzOX56#<$q~Rx|bEBUOk4gTEQTrvb_{f~DdSPd{JJNCE^kTMas%_C|xoW;*CVHgh zdi%BZaMeqPBU^vG=e>@<>5w+=zqR4k`QJ=@JRu$HmCigKJ<=atH?U~U3v0vf2$WGd zFArQFdT~fvvvF2PkgS((Wks70Me`0n>}&peo5Q)Ut$t;7UnxCLnFGmY%XSU5A{ovU9$8K)>N#C6J*3sV_`}mmDec~T`@7fGG z&{!Fuv2t>mS$ADrR_^2hGm!POr z6$%I!HwSJ%PX;}Ze!<{1VI>LEWr*Wx8sq0*@FV^=y64%v#g7Y50f-B^1P7r55DAF? z6EBE`2-F||`;cGu^3zevp=(c1m(EnZYmzn}oUb`}wPmX1+SBuvLz3>$A8uO?F{$$R z?Q3kX(E?S%#iGqjy)udTfs6$)yOg0YZC-`S|O(y zO-P7ET<9qMgXpc}*=eyk%wZLZfKg;^a!03#1Tg?9)N4ks8HKW|OU1^@8;)bNS%#7$ znVT)cZH9DaYD?%F*z5=fs>?%5GP?iQOEP_{lFTcV1ZxgZP7i65VA-~h<3PDl%dFq3 zJP1Y8_Mz+Bl$VY!P-a=87Mb6w77)Cqi~+Gn%PwU%g5T&is^eVF>S%oMI6tV4=p_fM zAFW+>VI4VB*AZy?ht0w)w<&E8%AC+cMl8`w*;ETR4@xIDZtw=r(tcC%ExeO8nS4F7 z9t>qN(-&ButgY$ouB+`7A4imsy=}rj;=QE41{J|xgz^sgV4?%78>sWVp5%A!Whk*| z|K7P!N%w@`&(vPnbP7{_ibT}ZncmZ<*MgA3Hb8nEs!hhDbS@GJ1-92TtlgdR#ov#u zpYfc4l4T!7?PF0Pcy&Yg$9LD7<67Sk z1Xq+rI{GHAf%(c)loMzoe_Z1q4bXg0tTz?=O>1Jz`dB_$m`?vEVgC#9a@Iy2buq`5dB>Jd9ovA$?{%+sil3fruIF<+lF^cQ*GA^t`&qgMzm39 zMbuge(S@`58zTqW$QRC*Y-^&X;>kU#T}OFDh#G5P(ot3pLSXN;z2TOZ56t9DwoTw%9vA9|wZgOjbF z8O)2-Yu~#3`sG>Y8&Acmo9C;Wqt!d6bP`j5-Z5Fx)SPetk5N-aN@F+Aic#ZsRw|@Q zdtccbX}NKL6;?8@D}Y&Sy8Rr57#^Te?%pH1rIZ7-S_Kc%YArQVMeM={#|8zW><3#q5ah0P!<+;LufGP0hS|r`_&~;bLBm*SScU;1q}L~?dOKk? z(_*5fuqIyzEg90YoyQpu-c6hifx9!ToRG0WB@YxZoFEGx0~$qN%2&D_%IO3>4@se# zQ!4oaNG_{|!Zt!TW9ZS7?Rz@XM^J6up>a$f!>r^4v)S}9R2FqbJx;RUtvv?8mSW&Z zwa!fT-TgWS60akud(elf!?qp!1h>zqtxwR@cfIU+wyQhlc8lOL&0W1Q&TU;|T9yc+x z$MhO{;03Q3vZri(g0P|SkA_B>t7lpk?5FP^gl&)qwo0_GdNeo@H(_ritW=uMWp+G1vkwN3q_nmPQtX^SF1oRwoDsj|Tz;pf!o8l%ailM4_Mnd%m z1n7mRg4uq?`9^Hi6TL&eOW3mqB%A{@Ccce+691M0Jsz=XR(%WOCgcasXcazSM>u?H zjQPlQMy)l)=teUhNa%dLm!RDV#ErDaiglV1#ziBv;ggw!iWpBwLMHsQ=}9D9%A^&w zCR7jfL?F(?%!^6{4KwB&#Pf(~qmp%yjd;Xjy4BK$`&B#QKU4iF`0w}&{5M2RsCDY1 z)^#!K2Fbc%Rs%zqsRULDC5Jau}=Bqj){M4oIKNc>1?hGDCGq6wI zifDr>Wiit}ZT$W@whM8(;)NaI($^|qsf<{nh3lkE2NwztlKjExieyLhlD%q5cgOC$ zdhyD|tIu3{25f&uWLPS0h}j!&+Zz$_PNw7P@Ri|MX7PMx@eT2{i?3Xal{C(mG|q0C zt-5KD);}68X^Cd;nbLj^Zof60o6yeqqOSGOKQK$un-f1TntL+EB-9)28A|fuPr`_2 z=GZKB8$~y~kziy0Ms_7dom-oo@(dmmnJQXSbB)9U%+L@M_bMJOWYj(tPcSyNAkO~Gxxf}0V40iLB z7IGM?jp@@?M=fh2`3sgBvQIj2<-p5NMOvfzH4E0-MWgkq<%(sxHD+A(RVB1KcdOVw z&Odxz$JzFi+6DOq2pYYgwJ%5a_uQ&|1vP)arVimCUaHqB+!6e{=Kv;Kz9A)L7kWbQ zucpHd*iltWtUwrs$N!Gt>O=4x&D3RA(z3u0_>9d5n5**juzI+8;2J)OJCe2Om?4^? zC%wu=leAy>chpw3W>HELv?;ZtSR48 zF)`I81rrPQW$7HrbZM}(voQrb9Q+#@H}r7nXd5A`Mv(i6)gK!nDTA)vTsEx-_|VqR-E#!-4v6ITM_Xv8Ri&-xy0 zFN0#RG$tRgbsn3ib$H`}P0T2`YEQ_}DBl24qrARh+FosF9QJtvjq5vQs=`D>fc(YW zsNRE;KvkH3EcqLwO;TQfB)CXcKgdqLi5t%j_V*Dv6EZa_x4_=x*i8+s1YRn)AU>!_ zNCu!jk&C2zhLXhs05CBF0a=OkLo?$Wpw1DohB#~%3F9ZO8y*=S5Z6-7COnJl=#Hr> zm|YEtmc%-GS5E=SOkm{FPWOB1(~l^iB4mLqNrzo@$%=+r`al8MRJ=8svn^(8hQwdS%`tn)yuBpCN9|=`)E!w6={xgp=w37Zr73bW(i*vh-5INz zK6P$@8Lqs8WYLj#LwjQn@u9Jzb@N5*W{yUS8l$V$&sP6p^IXf^#<`)nqUes!MF%0> z^O3=s)l$XgXz`Yt8Nb%wa^BM3+B|nI`lxXKcl|FWZt-)0TNQJsqPx2m9jk6s!a}G# zay(*-7BcX4gvfJEPURq({3JgyYg$R=1?DF~S@;CRqwfFVCv5HOe6`OAI zvw@q{v!|mKtqa+0(z$VFhXFH=+{CVyTrP9_-7?OWjcqnHZ??{?nlazpvrw@+RK51H^ z#@-;uUTRgDHm$muL`|EMF0G(X<@IDAKudbdsI(*bTgshL`8w=my9WG1FxL>*Up(s@ zge#bl5s!ax#Mj9H2BepGjGI*3!P*S*07eovV?)el49vjU@rwXRUf=N9fpPE(@FhdL z56LX90L8+14MvD5{E0p=;`PFP5?TAhb&1h~4G)09uv27th_4`#>`6(5C_slC%DB}+-v;EFguF_cHnFBocmYtOy|<>ck(EEymecp7QH zW6mUD*z!U+Jb} z%2yD^%dgbQJb@%1q>Nk6)3*}rrR*cyR~Qp`(pY(JrCEYPxqoG5o>{xzKQYU{y!HC7 z7k5Q!V%c?4cHK`-{DIA}KP#_-OE>WY%GTB&81(u2M!Z3=!>EQx_d{CnQrm`jlsiB23jzG)y+^D90VZ;!%3R$Ro62 zx%ozb0D-K`kh|K5oznqt-?9(YDf;E>H6>>0p)$rw~@K^^<8ZvlGQ(#B}$j zfF=4-val0l;$SOW9XBYuQ?NAf1#l#@{#R#cdf#$qR2TsNO4N3CrE~R|mZMWsEBH?y zF#)nc+PfoUz~GW_yDGP)T`oy#;v#A%ZgRsfF_T+jJeE2ENa0`LDMpYL#%f-``MZGg z9}%nyr^52vykdSTwgx{$`LQjO@Jgi>FC-@PCl< znuW|%+Cs21iVU!*h&%b5c%g(lyt*gje!jiO*kefZ)+}B)_+(-gWemx3>$e>b=4+u_TMugqa_E=JOTm2jg`!ua{tD1x*yRFDwhmG<{pDNiWh$kzI)8;Oy z`-TuNhw@7q&s-rdWjr&KUS2kyk3ixCqs^{@SSb->Zd;P{N*WFO9d5@^4uei=8len(npTee?blAZGCuO!=~bqHx?9Tc z6msE%yf9^+s_U(rYtXZ=mFd2^o$PB#x^F^N%3HOU5Z@_Or~8ufPAy%i5o(3CLS2Iq zby$}!MuWCN@6QtI{n_p;tO>~K37J~o4IUV=mYT7NFXU|7#c`3Qr-fa?h@p9R)`kA_aR`2vIE5R5Um?BvQk z;UFGvEF%t+hZWtYu zwRqYgaY8hp+hyihJJLV)NAvzJSBtDt3>nrpuz#VPv2j;Q(Qr@?r_9)KC)ebUmcuQr zN+NMJiVzE_938bt`~>-LC$SmV*%{h*h8ZEj1rE8wAu}NQ4T=-qr{E<&;Qbblz11bu~;siRNQ+CiG^M!F9+FEtU z)3AkW;1VChL_?T{DfA;hw)IbD3hgbg=Z0HmI^M0Fy*$76(U%R={B*^2!;6MDa^`eZ^DMKBCGMSKh=bi_cJG`hwLfZ+gr={Q9v7KdXCp_@?vik(c+*HO;v{YW}eK z<-OA_)1K>lU)=k~u|F{H0pGQ6`z?%agcHr77?r}!NqlftdK;=F5n z=a)3sIzu{_IJgs5VdEolgTMbW`Ie3w*gSxk-kkJf1 z`nVO~#0QThp1yG$DWHqnsU({3vSUIqZe=gXR$weR6lXQ=3-}=SqImE_Is}eyjQWxm zf1F=mYX~`$>)zGP0EkF)EM)1P7=Vu7b+{i}KdJ9SS#7d+W!GLnuZuXTKnju`jIn+`xM za^eVmryhvw2#AT}^n8Q95aklrfRWwFZYDTUE}owu#TS- zW0aOeJxt7y=KKpq`WH7R*R}W>irtUDK%S1tR{q6)U`k6)!cW0E&${MQV=<)fj)VLc zI^&-o`0jzJ^WQssM;5g(cg5?l>GQUw8bZQFOZN1M@THk7$-ItjKA7GV&YNzC+Df18 zfNS>3>Y1iDkH35(Jn(YwEOtXonP91xnRPq6X0~i0bMv!%5vQzThJUmA*fzPqGz%x~T$CfqSG(GTxUfR--Gwlr@e{*oA^o=1Yw?67>egd)AbB zF}Hvk@s);Odk&jh`K%FaZl(K3%XGJFl=#xA7whGB2=P79q})~Pb8j?vt=tid&YbJ{ z*Yd+_;QU06^96i?ZwYUCb7Lg-E+gA8lt z)qT(HlX4%I28N>6;l$Qg#Wz^Ro7zYV=_Y1c-Z=1veKrqP7WqHRpL71aFxI?(zIlJN z`QWV+((x0q<7eiNW6SB|(c@5zoSQ#>Zn4k>A7It5SEIMr&SuP)ZMkWlYx!vJhkI|e z{OtirI5FRLLOO+F2V(=i`2nBw#JT9eNNhly9}uGhK{(hdyns!;yzL@Srk@pe&D?6& zROY-c9|GhBLK;zf%iN}@c^`Jj7|h>&5|d({Kx#Dm@U37pvj;{g^m^GStY0uTu~E2V z%6%RNq@SDY)4Aacw6paM<4moDF=B7b(5z_R*f%pEAwJ&Z-N+7izq%^25yl&B@0cXx zhDB4>^uaf?BRy{v{J3>i`_8`Eb92qNeA3Aw$v8|&R^8Yb4!pWeGFCu)W6PLqhbGFL zIl1R|WI02V@#ss2lnJ=#%)_k1sRYLFAC6gF^Hvw8RQG!)-aawge^c|Hyzlf$>vrDk znJb&ikxDy0weJ1OM4Nf7oZd85Ib9ImhlySBhVQ4@v)w;gC7HG_=yojX41a#$*#pxV zn54}!gR|Q|5`HG$>XEwpusvC_dlqzF^wCuFpFi`xXTs%S-(NM%>|ZF{Om*{w_eX*= z-LnPno%KhODC|2#v|o=rSbC%4w32(C145mCBJfUN;}m#Z(PlGt^DEZBRtJo zCTjb;)rDQVb%Ove>P6Uu{_cUuC21G*Pw*6+B9K>cvYAo3YC<fQCY1!M}+0)l^=2he z_;|ok9s%(UfVXJ^ab>KYi>7?3piVNbgW;_q>(8FNqamWl$M`^LYx$X*cD=^Vl2T5!6&@R$@_%wtA(SN2mp4ikW)&16qZ$R zt`OG@0arrID`c+_$C8ZGVD@M9lGa5oKSRh7avLWNe}Dr3wq_26i@6cMc|dg@UH zpkQBhFgMPl1;}E9b;NW2vgcLmGe{f{7<`PaUa+Q)Q95Ckpt`#OYppNmT-<}s-)QF# zm8$1&B-#m=R1ldjzq&y3FTYaeEkr)cuU=BNuxq1s$w4dd8>^h>QCkRe)PBFO@j_E0 zXrg*L`fN8$S_3=9gGdU%LXKh>5FulaPMNdx0hYiN1aU=DjQPe%I8N|Ke3pW96g*GC z3ltFD4CSt0{60NVr5FV*(|^Pai=!0$fP&``gbFBUoXJLvy2s~%Cpw%JHWt9SYPC)n zmx^FD2WWT5`3Y|T=N<|2jZAM6r~@r8j?e)m33W1~@AZrWj~j7Z8at>&B%y~$`ZRg- zETeOiv@a36Fk2;`vmo^2I_eYhoN_+Si{f8XRsREG#NkgDF;sGv*{$OE8Tm@6Wr)^f zef1M0yN>I}{l}B14D^t{#uMADef>v~)o73`O0;(3?+}Ua!1X88%1uM6;^N`)KWC z8_{|d;fhE`xcHN->e(!`Wkr&CarK&r^Odb25{ZObELa^r|4Kfdtn<3uMT2ST==Uxz zqm;HS7@HY7|9vSs1YITBHvQ=fxKD@E*qF6+-dg(Rrdavb`SPv5$d@cF%TD~E8S>qh zs*XDD!F>*QbYG*Tux``da_vWDJiw#-jw^Zq{sDcq;RGOg_@NHFKG6vqDach%eO(j3_KC)FO=jhj^sC!g_J4B-3{rr01 z{s)&#O2B28TL&XO^*FmU7&s#ZtEYi;=oM9*$xb<1hJ#7iS1t=R0NWl6aSwXAG$$UV z^Aypg!p~%LdV%BRYoi7*75r%t2hXZ1jszYzfX>P2(c=IKB`SXL&d{R)b)&WeO3Wf9 zLRYbJm;8lMO^T2f$7AXIhTVpPzlNZh-TAPUA#r}3F_fji@Z{Nmab4Gu?jFb(o<*MG zj{tGvaJL*h3l^L~@(^5l;n;;C*wp~k)`U#8>u^Y75I(v^GKr~m$iP$YnHU9o4T%an z7LFj7I77kP6cDtP#RPvtckfW}V+!7-fB{z`sFK149MSL}>C+T~xE?;j-~jDeiZM$s zc%S$_eSH;g*k)=vMUJk7EdkY7{5fJ!9)Zshi);z(SGPa6eR?=*DaP>$^Onk}r5Xne znH^Uft~A8Vu6eU7(h=Dv)$No@cg^)IbUz;L9+1wRm&~rHStLe1YkD)W)quyQLf8mi zFTPeBb5_qgt7o*cg5<1@Ikzo1x7}>HlfU}<fa^6w-=0(Zg_zj#o71Bv<^$|WCh}6!s%<}KH&upD7oo%1p zDmiv6SRa7}(bWUb9hg3U_3(3t{{$CAroul4wxE$8mesaqXntkM+q1#&t5P%FZ?Lp# zEx+2S!~NfCwG?JlwpHu@wyvqIO#kaLJ?;mQQoSs=V+8#F2J78aiw^|=(0ND?wviBc zMpi*f_%I7$=n8RqrJ#fMJVES1Uz9#~b&AveC6zSBoIx3=seB13i{=buy4dMN?OArg z{V`?*U9cxobqXt#s}h5~D#5SRRjEb79a8#wtLZ^ZngYfZN>R!1`}Nm?1+0t;>e3N+ z$uo8V?2~c0RIrTWCz*pEZ1i{Z=E&^_0htDLCG!)Uy{dLNsc{ytx2#bQ1}7(} z3ErrIrMn8I@RZl9f+{@a_qvo4@wBS9Eaj=7w<6`Ku$QQ5b|L2K-kRjoMk7!Qhx<+)kjh9y6PA$m_ckawcByyD8@9v4rKEwL-LSg=vMagE4Ex^DN|L6-=LGh=38k#Qo0SC0>cW?@Wi3h7rKaC2_W)0uWS8ih5DIxyqwspnRx%R}ZzMcigFMAvZN7Mk^F*AYpaxM2H{z!4~C1Dhx9%D0m_$~V8g zN3}QcG%cHf8++_&+qg+-BZ#lfE5zB7jFYNgKirZ6bG9mRFvIsMbt6r+QoA$+ts`ZI zL%gv3!VK5-IB=r;_N|Fo1~|Tk{v`!V*qM}W2JTR5qmCSK14I*Q{|qxRR7(9HQBpu4 zp~jMR`DZ{jpW$KOoy0FYld)4q^uY2K5KK+A`tCif#1M7~yOVZvB{+VENb=sb%6E{> zSjR**3u{g?_>PfZVmr`Puf$FM_F(Tj+`AdJ6A`V7<1ix0nwKnzlVY0y;&}}b&+S@r zTPWWr$>)TnfKlBUZ1OFyFIEMsZ!EwqPgRbA?-|t6E{GvL0Y-E zHOX{f>yZ;XgK$ph*f0*VX0nr=u!4p$b5M0h%s0W!EJcFu6n^@RXF=7oy~G)E4U~g< zAa`MhMUprPf*5wpr5Hiyv-4t@4>-0Tm~9M_H!A&4d<{W?3n#kdHpSH$(rkBy3>_`| z54E?sLfI*Mz=HnKM*OYCxo`eawi6(fnM_Dq6qo@;XcM&5Nz0CPIJ6!v--Z)ygAs-Z z(Uc3v4A+kj)svloi-~$jLdH0uxE9?M(j(aJf-wfPmh9Y1=8;z<*eSk8&BK7fkqFHOcob^hBbZ%Z#hUArv@Df+eHaND5^>8$@z6Y*g}tnKSDw zX0pO$h$I7)rGYf^wbkOI)J;JONL3xzdoOthFR+v8#lN7pWVk5nsb8n0q}e6|96Ht_ zZowHw$^?_Qs1rC|c^Z96Uuhb}P3@Pco9Pq^rup7Wc?0((wzeIlN75aOB-W#&ko_WK znZ*z2Sr482C_L)na3nYgB%W0yx)d(?z)#D1ccQ^@YAS&fnJH#Ua(FOiO9Rer#^8ew zCxyl}Lt~637F`szkMg2PBmM&g5JAXC6JqnjWh`Z!WQDo}{PN)=B#dz_zY^T^Xuoev z{Ac7rwF@+Z3?^-lE*7&ch`{L-_RrZb|3{P4B~L-`jtcF6dgQF4|?d{R@7RC#Q{ z=w7rrub#MaV)|^I z=g&>X$kP7ITZ=G)Or$GmreDP+HqI zH}OyBq>fY4>A?lBPwIW*o|exU;o)*ndsmlXs`+hx!ArrZ)+_snpI;m94!28=vQMq$ zUzsS{H(A{JP48X)o6Ayj=R)I=sIzOn@yJZ+%VpQAuT{sg%I34mB2AxWRpRZuvrE=> z-^t2*sS39l&mN>E8dojo3O+9>BgX-e%h8giSjkqY1a6iQL@oR0OSXoczij_m`&{KO z552H&ss+9X_f74?>&Z6Bvj46oL%;P-<|=e!W;t_c@Z1j6RC08{GxVLT!tj=8R?SrF z-3)l=Bl`lBI92(w?z;I!^UUCTz8N3fIoI!+uiv#$|L8YBd|jIP`bS^yi5#74xyx|} z_#>LH$Z7u(&5gEj%U`zPyxY z9KUjW`XnY)%vLGckk4&fWs1-4t2In zn<7U)u~w5TzVY+KsWDfMhPfJRrs31g}<4 zKOq@Q7j$L6wOIf4n`1oZDENlcnrasFSJPx%s&H?OY=)sn)7zDik_BrMP1I`W#dE_0 z5$EfA92Ok2;*6Q!Jzx7a+P4!-O|`7 z%&HWe+{Xx=8)wk606ADL=Dp_h6D)Fn=d+$H|s zh>ibTz>E;%%ky99ti0}CA;(*P$60>Inb`l#-*Gnf-}yVv%-(1Jj?4Xg{e}+=?;AeI zd_VK%GqLrZx7T+*>zUdW-Z)eGNya*<=Xk8=%zV$8N$uAeI^Ok7risu0W|fAo`Npj0 zbG~ut`TVaP8NB1`95ZkGy3oRBeO;8zpWwe>x%sjvOYoigF<+bp<{q8;uXLn9osC3)cnc2Mk pn+7wI72>0fKI*Jfp#^OLS(xnm?!GHM&$~a><-p=gSIGM6{{wMcVjTbg diff --git a/agent/__pycache__/rollcall-deploy.cpython-314.pyc b/agent/__pycache__/rollcall-deploy.cpython-314.pyc deleted file mode 100644 index da5dca180c88a8082bde3f366bd2977028b49975..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 28126 zcmbuo33MCRnJ!oW3Oh(}-5<%w*0xGqR*< zC-$7~`R*-L0i(<@suK)el%k7yN76$Iv_&+}VqkRnXPvk{$hD6}rUuqa; zn&Fu+GsN(kqnfaWKDA*jed@wG`eef_ed@z{`ZRC+T8(Wg0Vp-*er@}#c6 zNcmjy*BVl(!>y8J`8eoX>_|!RNrQM2=jVgX8n?Ud89bujUKj*YJh#YxyGhTb``r zi#yITjJJgL$0v>lD&@>VB`VM&M$C8mx2S*O<4+qctCOexfmPG1aP7Dh^ z_XM{;D0VjQzvvUlcRQQCe(`*h;J(xp5yMuBIapU=yA0>LS7qkqyL=K6w%$qP5?51^TTrMgRkKO9DvgkUhtjfxG)y4+qb z=jKj#_Z;O$NFv5bxFYwTyrcGuemT8?6MUhd5GGHO)MQDF?q2SaKO_(6hGt)w@_Wvs z+y1dhcX&!bd$yxpf*5WNxY5`4HtwYVQS>2EZL&?F;3c2H1xH6Qdfn*kmGC&3A2MTf zBDqQQTIA~GxjN0CXy82KKF@h^YNCM-Sz=u1AUMb@1Z77*y(HBxzpRyIO1*_Y22~Hy~DGo)ra4RC$V?w;?=_^ zJN4c)JOWo-jE!m0hBSj}E|+F7vC9~&?cIrRC6kODWKo#9eg#e`5qKWcc(ce(O0AJV ziDXSYi(P}CLmx4plNaif9QwwRp>raj9}P^2<9!Y`t`DKB;kfR^sUvZnctwmG#IQFw z6?SOjx(RVC&XNrxk~Y?H@gh1;&B``ul&fwEU5V=g!7;&(TpWI4E4*oDEu$bRMtzdC zY`SZmHP6((JaCIGUN>4^uDQPL+P0XnY~7SGk^;;2a4v*2M2# zKSpnw8KjZK>sLomjRFjFF|8V&86CeQ zQuIj4OqNnY{n<(=`Bb1OZ{m#_vI_Oe7sfR%vO3Ihy*Ta*1mc+a1lMgtmi=AA431~um6gI7IA7MBm_M^QH<*Vf8!Rv2nZZ@PkMY< zM`0ljFoG&4J%FTf8$UHk>gpDRpb%%D4*DnQI`atvnh+NJA%{M$12h)$(fqicZu_|Q z{H3@iB$C;ztE&S*A^jGL6^{(8_!_)vX5ExCoB4cZR9rSqzPW8-;O8~5>UPOAc_R!@ z;$6glYK-{jCy-_O(5FUQH2ZqlwX*r_mF(*E%#t^sSk(P|V0Fu$kGJf3)w3kNfB6@e zZ|Z*eEvfwE&C}9=b9G>Nd0<%LpOT((OCz4uk*AkOo?aaZEsum^BZ4GelEll>$Q8*Q zkxHM5Wj-sJo?Wxrrh5hKRK}CKSdECF0OEJF>7}XLSX~19G{c%H?UZg0v##tPQUV%ds;>J@0bkZHVI2&?@$Kyt? zZ`3^%z?vErNn`5jhk2!< zQ~$RX1peN7$Zr0?%p&-KT}S+!j6>DB52`fqJ;<&)6_l?bYMN?pW?qRc% z#q=Cwn2^S$NjE>dMyW#q5@Q&WGUj+0!{Cr0D}}mbK3=O~`prsw^2zIznl7ORp#}hr zRqFK8I-TrEAdzRZbtrWF1jCq_3V=&`L}WCRo!p#2h9YJKyf^j5H3ysyJ)INm(@?-4 z#>NLS5Ed`_F_*dkCfgjsyFU!12YVRv9|k}Gk`&kA|8QJ$k?voL;?VsYHzvL#>_JP~ z@e|$fa9UV1a7@_qqQf!!mR0-qW&8Hq_Qo0Anl*E_^J?eZ_El@$inVU7p!l_#x%07- zZBog0si0xCpnbWZeNl`R?7Mno=1{ci)6%kQ?)lo8t{0AcZe+5W?wA-`*7R|CBeM9E z;(%ex8!>dS0^htM`^s6U`(j_hHaP zAW9mYS3mOBLC;&Brw()_d-oQ!0iZR8WgSmUwa2su8y?yx&x9D;*Hvjj;3qz(9dA%{iS zPx5c02_r%8mAD?5l1~)C%3y$W`7QuC2gMcF_~RNOuJMT$Bmh&YlL<;0-0DCU31pu> zf}eO4BLq-e-|%sL$BS*Vd!OGM^~S8FtJdmeYxRn?cF7AEY{{B!y4tjA;g&7jn*|HC zKPg{yu5Ig(_MVXXhE|^PNsiGwI>uUckI`7iHPeUx(P~GjjGXAM?+wmq?r2Ts{Ppa@ z>lN23Rr%>zT}w`vr`( zaISW)U{08?oiC6I>thy%snMJ5#f>N9)(?6X&&UA1pnwr{!5=*;Q_{fT(y7?~C#9j0 zTl+oIKCe{cixrGYS);%!Uh4X4J*(il?V4>hYs+%hmNzd+_GTay_zX=1Mq$qX@-vfd z*7Ce1x^p%|#9C%P*y%W2t^HL=!C|}cS34{S|JrUnTxtDv2}{D2I^tJX_LS&9)Ewfv z>-4{=tHaAbWN3(=sqM+v|3khWVGk-(oq{ml1E68tn6f9CFisTkqz1ye0wxH$5u6nk zn7xj2VDvnTcLT47Z{z{XXe{vlu-V0gExZ;7y(Mny7eJ)}Wlu-3n(md+#lcfP*p$x2 zOsdU67-_OYSZUoioGD3u4$7$LX$n+Q&pM@U1~i;D{2uN>M3xTnD@+uc0)Dh6D zw?VoSmW4=Wp-$e|rr}}28X6vsXDid%q(nK9O(dHhd)?8x_}G$GGS$tnFIu8qFZa%Q zUmIV~E5F|N{k~XUEn>~&?bP>A#qze$w}Q&mg1V0j>K42|d;0CCZ{)2M^dR24hLvx< zYF#xIFPn;Qn@TB0)#p(s7M~*%24u=~?>7B51{%-J?mwfz*JMPm3|9Of8TQLhU>p$vDvap*g05`>E|H6*=*~h;W37Yvz z<)I?|2SwIH_2vzmpU@I?#t5PYB>3MvV}!AljtRqSQrRKt*jGaul@+5!9YCu#yJlTT z0~QVbCFxZluqwLDv%LPS2262-f&gQK>b`LXt5q~n1w24b!Y?^J{2iYBEXmg z=XBH!?wl7`i7C0dfMEouCcVi;M$huNx%;xm7b55qWx6?Z@{&7^S_Qg<$nF3hKtY8d z;w7G`E;G7HM-eAZ!Ao6ed8KpC*PnRxiG}>1>leN6j=wX0LlfKa=#B21ZU3d?qYf$8 zInzZb?!r5)&fK(avque5QQM;TM%AjV>kA`YZ(rRvF zDCPA|PB4c-UfSOwv1H%U;=*@Hh%9%}0FNNO3LnWZieph6kK5JtE=LF?Gud{S1)OB& ztLE8^s~Ioa*Rl%d^5=~U`HT7`-;Im!1>X^+fitTE=avV~t!3p$#knK%rx%VcUV8Ui zi{H9YjrE(=qnTkp1M1l>mF`~3i`hFRQzxA`G?tzVw4GjjOCA8SXauro9c*7;Uk{cM z8FK3NIjnJ<8v?5F$0up(4AR zt~=qsLnxj&xzEn)StZ3k>?EyIER&@fRjmfAB5?kB$K?wPD|q8N$k8 z?)?iqx6aK@gTu$`PHDgwQ#p`cAaFRVHH6W>*(F)y5R#CZsB@{@Gs+t!L%PW&*!6VP zHsOSCVuCuC&c(KB3Hlr`1=#_%>Kl<*@Lg5ZG5G=IbV$d_=e13Ya>gn7DwzSC9z@_0 z5Jv*%9~dA2B<9|q!P~DT$1wOdy9ncOK$~_z)UK0if&XgKfxRS)^1&^P1||j?U~ad} z4wOM%+~gVe2fTt0xGd0HQ0Bf#Z(IklC271r?3)nd8blDe003d&yhujAj+>sASI%|3 z#dB#VoAv|GbqB=GCX(;pAwk>;?~l{WC)ouvht_O%g04g_ym<2Wwk+`D^RMMcBeCr2 z)$DpHyMBJ;w#{)*&*T)+brrXb5Ud&UIzOZiK(c00RYco(>`QIq2!6rHslG!eyBqos zIeQzNkN0;s3>b-K^l zf9hzT(_s|8k8j8lhcXg63=b0s8ebF$HX{509uZxjRYDbss3qRNCmtCG8QD-MVu`6hK=Q(EN|Jy?23 zU9Un5WU1-_sT^`y%xQKC+iptRv!7uubfs+_tE_pRA%|ds*$xEI1%xt{yVh@1TAFA`$gkIZh3m!p79j~)FR>}W9Nc|+>ATpHRItR5nD z+}IB6Yl5I}a=Z}&fdmbn(hqE{6e;Ela(!vXLFLM-JqahiZrac?EtPhTs_pMW`wcF` z!|QZuQ`%Gtj<$-nQ{Ju_ecbRBa7HaIAl$swW!N|}h_kh*Z3U5NAROZquCXdb{f3$| zS`5nE5y@w=hLrKb>|y-LECCvlS^EFg%S_rM`{ub$<=Ub@DJ>aP`@(0@-ejlpze&5) z^$qmtYu0y;(o1z-vEv|4!Yq7Ee{z))d>)zW#ExjitR?p<%jc)A3Z;Zg*YTcO;zVK( zfZiv9rH~OLXjED;2A&P%D#;!exOAA?!VNt%_NxR~a=8=zA--7QfMdjh=LhYLSSN#W=FS6L z#K4VQ0a~{Ki2x{PC=$_uz;ueF(gjeSI%w(yg0@MGZzW7Zt+0UN!jIvB>7;{d2srGo z;4Ov7iJSHZ+!G^ScW1m5xV3kf>N=@sy^{Esq*OkHyt z-a$_#@`>}lD@1Tixn4TKjjl(CuN4K*!Q^}ueo8!gbQ0`AI7mFQSSUXo!vp0dhY3&2 z55Y*F>GUR2F}F}BqHPCzA8SCQlrfYG+>>5WBy5KSqg`SVEpC>In-}=SymyPVkgtIi1t{{s`T{>f-9nB+y}uNh;uORpNEJKx&%#;%2eA0JrkS+3dp;$t^<-f;b9 z&+6eb%ZJZMXQx&VUz~kx=CN0e)7_Ho(x*o2bXdlj>{-!*m#)lpt=PHwL-V6QIf{mq zy;HU{c0-8mJa)VG*r&P0bMDuDulnYDUl0C8aItE!_wTmfC|GGZ@=xW}bDFi%ve$RL zx@*4ZhX)r%ZAKk~kuD5L!@gTjj!J`L(pXU9 zLMvq#)@o{B>xEpkq6TDs>DGnXg@T1=mkuqB{}0EP4&1a#&L^aSbJDpEke&H8+j4kJr{F11K2=RAa+cj78gM;&{DwN zv_R(0E#xg27DlDgU9n6kkhHI{=I4%m``B`B$MmrkcF&p>XzEKH)2IG)w~@)XtohU3 zS|(#$BW}Z5`Jk@wXeIkAZ7K1q4e)S6NH}J+L70XLa0ZEsADupgd(N0rwH}qu3Lwcf~^}Kl#4w3vmB%%-PM~i#{rL>ro zU{d>_jw;DLRpHyY^uOG9t=Qjk*F~eFayoVOYTqu zHvzC87V-2paDj^xO>qm22|@QM95+nS7mzKZ^66tCg`gVAHM--bi({T~B3F#)8r_ou zfulruMnEy5kvS@Y>On4r9QxAI*eLjjfFT4y*}s(*{Zc-#acm_gGF;~^V#$5KPi^7w%@ij5Gh~reFLL6&Dd7h{B_niQxZKb znMzmKGQ50j+Oopd5jo<9`U+k+FxPY2TK(_}i=u6Fd6KDOg{_q99h`5IO!X_QgS-|; zdnHr(ZMH(Dg;IAB!A{Bhy+^Rdra^gcJS_3P;gC#S)oE#Nb#FYBhH-`ndz%$5o*8S^ zVplwZU155-)@M3OAgN!mY42kqQ))}YzDU*0!0vxE!2!@y%nUd_@H3-J58vc6!Z%Z# z)Y4)NnN#pME6r^qX)P*k1(=YD@uaRFnBQgxP4f&s(`9bbrQ`$iXR~{FDsShr6y|>j z8uuZq%hFGfjZIFW4@duk;-CdeaynfQ=qjkQSII!&0K8dtE~5gA9~54NOfFNus=uO? zNYU$fh|WjI#%Gg%3hT6zIcrm9(hFp6gO0C_=Tf1!(xQ;nsqNpA{H_w>a|V^F7^ii} zqfm@aE9NuTspqR2$R4ykjg)SfZ`UP@i3%keOcoQ<>wG`b2y z8D~}~O-z}UgUOy3xiZkpVi%jb6VRqsrA>TEQ|j56j4$QOT$&B1Q!=d_DJd&5rM3#b zazo1hhjqu7 z5H)b16h$Pv8Ct@s=6Dly`XgBzqe2%W`*3J+jX^F%jyXsN_HiCm#5InJ11C6g$`K=m zT}r4Cdi#@<>H<}^HBzvZdzK6Pd|YS3vI5`WI=M)`1M#7YT;n7v;u`&23)ct=Ovkl! zMv9RH0}U_N==E@o&K-D#;Y4usz}CpVt=x!spzS2*9}T$24vbC(0`$a1DtY(%1(1U< zLYTaGU{t&^>46;x%pt@BlaO;nOoRX=B;uDW*VxI$*%7xmj?>=b#IZ|`9DzUuArD?( z+?dE5(QiTJTqKJaP*4(**mi)Lj1(lyR|r*^5EBw@Yyn_f79&LR<+0{q_cGxQd&c*2 zW1uKOC=z-I-zO~;eoeA}2v1hpg!FO5C;Wy){sX)}{43la7_gp&LcBUkWWwtQB)>K& zQse*{QkVTAR6Gp=v>jktLFup}l11MyYe4lM?e7M<0_81X0P%v8cw{3|%_jl)A0sO} zSzPYGl&lg?1sSk6@J|Yq&q@`g=m>Endn55Eijpb(ClY=g9=-u_DoA7@AiPT=3E_&4 zPECwL_2S~FK;TW>M3xc}n$YYXDpvvrm)JCbryUH76uUCsz;f68=BN$My`&gr^U$Wz|Sq zcF)wTW#+s9Bx&86F}wTfZmQ5)wQ4O}v6ijdGG~uIe>Ccb*q$IrcMME!pXR3L=IJ|( zWR#g1D;Z?&AO|NIZni(sq zkLrmoSLy#|&s#4jp4&37{gLAblTjTyW6qd0T{T6EUpqXno9BOcVqsvRdoh18e8YHa zcdyibR66>&^u&<#_^`C?soN&^n$h~)pMU$$=c-nWrJ%^9^47(!cTc@@N^0%8@#u~4 zZ=d<#nbjj_mXDl?9eI59$nf%!;n)#qUMrF8wL~PI z9k*o>33&FGBH0KZ8gq}Cm=8-TkGAPQ++#=hH|8ApAGKMJ>CGP9;DXMOV6OhNSvT>j%%U`Lr@KI zm?_mJe1QJ-ZJ0C`F}e8NBUsf&a(^S?p`dpT$St8SLV%m_SH$~kcqz;{;cxIBFQdFG zLL-Vvp_{^p`>&(~fpgQ$U)RVCxSuw?T}f#n;b)|{VN|5-P!p+vgk;t!dvVkqhD>`F zc_=FmM>?2oKB(XFr%pMCdj^Ow3`Tu%6ATT9kDNZ*h1>)U&c;PMze}bP#x(UOcqH?IKS!1??uUf(1nR;Vt!S&;BEg6=#c2eezl`wA( zu4U&-Y(CLJ&5d?1S9Yvs>`Bt2MK2tDZEVH5Wuf-38o+p(=~}a8%ywVxj&6(D%2#bQ zlC5SwXVtoG*}5%eZTPwAwzXsFvG-5^;`EJq%0WoYVmDtih-L*3rk7z`)$i7#2Sj+yE0p12HHsDI~PJNMfY7oBc(kW08X)aXS zQtSdDPS&JIDk$a!ZjQD|ds5THkYi49XMiuJzK3)vyH$r;)x8RZDLodJ!3h?uiJs}h zu-VX4u7vPI)Q2@k5I&;wvVv#aGy%Q`kRDM!h*{xBBvucLTJRpoPNVWqSr)A5VhI0C z;)(aa;wSEchbd(Z&mI5H@u+^Dy~S=>v*ye=-U`pLbKYyuF6P{_v@PitpS@){JbjqJ zU0t+ol`WE~b{xH`BZEz0-+yIEozO|wZZxFYNVU7P3iU(jI+T%JI`y=4kc3KebS)|_ zuFw@@z`H@Ck(N-mnOgvwfK8Ty>`4O{`M(23>Qbn40>?MGbe17!YBu&Mt>S?SW0n5G z1|`{Bl`cg8wYazPHl>9_YFm{U^hKrU!6DRb9C^@q>Pi5W z{C^}kjs#z&4%`hQ&$LrNi~89uDhB~|gBq0`P0LBv1{)r->8R_kRYJ)pcqh58)I4>` zG$o{rlCnm<4!TB@b3@ve*P>0%P(N~PcFNhgVI=aEL+OD&6r}XQ_>FxibfxBbNFTrx zOO9$$Dh2H_sr{q0ICwBBHJ^TM2=mN~&xQnY6!#S}4{Jp>LD? zKIbM*AQa@Z%2-o6SbL&Lp@YAXzILnhbxMnz8(O5)AKBC+5> zd<$+zfYtpZDC$TqX)7_;_fJk;CWiQ<{;^2q&_MTzqkX5(9;o+Cjs^t}DCuW`l+a5Z zF8UmijOHfyR5%E}w0p!Ch;$-5DUVwY+#^t-g;Z9)@BzGdm=Y6(I^2ekEC7+LAYWNP zFGq;jX(NIn7!34YiAd2`ApkbmBbfEfR{kfs)!O17LIRuFe9f6p_-@|hh$VVKd zSIIFcPmA6UH9J87>I~E>K<%4_FUX4#(NyA07B3D@LeCRxXN-N4*p_K(0>5O#OReA| z+jM246oW%RH1E)==$#0lpdtJu2mmyaU>Kwwg^ec0Hf{-u!->Ad zH4`GyDN(G162ecI7eH8=h28SskSKnJ0O(vx#q?pQ`s%k+L;Yy`OM$t=D<^m4VfZMn8>aUfQ^Czi8!<}jg#k3WC>HSO!hSBXZ8ng?pdxjSj#f16lwoa@tjr4s=aO7@_7a%llQY2vu$Q9>VY((gwlf2ZeAL2N&Fe< z%w@@TWyKU(&&;Kl8I(3jdCjrR9Wy#Y%kO!9kCeZADfhjR_r1UHu38U&Y(*EcqKBiq z<}#%0I<(#f&F7Y7Tg!^Ajp##X^{?0-ThA|<^UR%|ZhD*ZfIxf=swbGh;OzYEwO%-XGZ)-<@Tcux(#@)4V6)>>`~yTAa3v~yL5PN zI_ue_&Y23`DYom<$kZXd4MrQ%0g@Y)x`|42D%OD0noYvxQb283$5sIEItjeXz*iy! zjjfRJ8(K%0beo)$XssBWL>^*v5(`xo+frJa(4T36_MD04Gt+!*NDoU*qh~f5PwM*BkP<^IrD%IB$&)u}JN%IR)@1&MWMwIK#}6rMOJ6o*awbpXgQVrW z`5Q);unjjkC+<&5okyom!N$_vZ2=uPvyz_LkB~lPgn_%35jl>E`|ue~UFxaNXVyG4 zJ*A{QiG8HA9)ny=#YEG~miR1ZY98=cHalh2VueCGF=TOC(#c1XGOCBkF8Wm*Gx-&# zitlV%gL@z?OpcKq5~-Z8`L>Er@wrXv>IV+|L;?p+ttF4o2hLP5SfjMSsYxrRDpasA z<;fQTw<>n(n=DEUWL(z%GE^uw1riOjR? zn_7xISTK@(GMa# z7}pE}rzEl4n#FPd#O3z3CRDA)jR{jDN?bn8_bAvs&BQy4tSCsapGXOCO}ubKAsV+V zQAIc0b;WfL^v5lzkX$D&QozX7ldN1^<8Wvr*3^Dk zQ##}@665O;KeX!%QzKB(m?9=tp7U4YW_Mu1P4UpU!2>$KBAo81;FBg4^E# z35g@0GDPT@3;&8Qr=Wnh@ET{w0zqmz9v~N<$m(hoo1=0c7MNAsL^xi+{RF{)SHVO) zj#Nctm;ws~rP*E-Lxn6}(8Wa-Pnkee}llGifR^-2DdQO$a8{`KA0 zc9Sbx<_G4x7xEV_EO-{P7uy!=r0PAZxqDY~_pVje&0m=JEaWV7EwoGJEzzFe=M@vJ zuOZDBFeodydhz#J`M}olTVd*$Rj{60czw_J_Ylleszv_YvD}XL+TP#upZDC<-I$V2 z3`yO?v3*b7%6)3ZGq&mpeC!EGAyEp3fi6#Jj%o3j)Ew8|1OHibLi>L+Ox_9YOxLIA zS#kNB+P92v7~itLVP9kyANxDwQf`bpu*!8WbKN&QH``)d|E=QwsNqvP48xmMJnXiq z><;c5DS(2Df?=(g%09DX%se|Y39-YfrFPj;3$|8-EYIf{ik~u!&27% z+qMHlaCB57e|Zo}H+l2n1?SIO7sYoky>n^lQf&8;SZnXiTB-lEbmpA&q(`!OVNamYF+qCfWt_d=sptbU!n`ZCo_Q z>N}*ZdzP{#>pqFy_s9FY6L@I-T6PW86I&(w)?22n#OP07|JD6kb!33FekmTtdi+`5 z(H7A9x7&|Z z=r-K0MA%9c1--$NCH$Z|Pg=EZ^;mmQw9v!|ElI{eIz%>UvQ8BL)1tLnTnP!Z6ql)> zRm6d@kVIhBV;MGegfyehrQfIwuO5YllssvTgLswI^k8q2W)v<%$IDp!IGn-2RFPeL zb+dB5nJNQ8MV~jOqGvb)t*L7DDQZEsG__=aJ+3}f5zfhl@Ui+eod5Bf`*NmnH zO7ai16fCWcBdj8soAWB)#4M=n%`;mz_8it_WF5hMZ9B4uJbcckZ>op-*M0L6Y85Vr z%r4`o%$QTho(IB|jx9F#>Y3gd@ z%U#B_ZLCmw&R3?e1R)-PNYS*h4zK`odCqC&tDF|TdQe5c`^hQ0$)!UY`5NV0YVR>3 zKs;+v^A^4?$=3b1!V4GMKm-{Hm$Zv-Hiv938^kkPWpxKLW(O)8xXQ*&?xp-P6FeO# zpAh{LqHzitju1vAvXJl%5+Vp;M9bAj%4ACc2M4u~KmbTu7@DPt+feI)Gf_JNN5q&g zM}S3xZ1QkGc#~9N;E3_TfiI0vJSCrt8wsBXp-7>k;X*lr@-3w#tuN99tWds?ROLD* zP($}9P#&t-Lh7Qb6??ge7T0{HE_7ID2oM z^GZ+nB;TksHoiI*SRM;V6H~FVi_++&TW2o=At71+LHkDLx|WUf zRueEBX$KZaJcf#)?H2wogyIIQc1WUyRy3Fzdy5%}R+*MWdYnLC)yWX{A#XE&Vi!eU z*K+csqpxhA6JNjd>ZSQhvC8Jv%3V_BuEmy_E$f*%v;OD(FPxt~vIeb<-f#ELoSUnX z${QEkrh8Y|j&*BZ)cL|eh?K3FGvR2@OV7sWEv81xOxtY7)sE<~n6+}%S}R#==Xb`8 zTNc2SL$qS<@&fxa)7z%SqFDW2Y3shF_8YY~3M3O+ax|rRbC-bD_59@7`AMmK*Ai{$ zBkOEdG!NY@qxvWIC_C5wN?^VVde2J1$LF+CnVP+@YM%0DV};GjY~FO&%oG%EzVj^6 zf+;|ScUVY@jh zxt>}HOs)pR<$$;vxUw9$g7x$a^jE-kpU|ArLal&2mPf`Be_DG^yJjzXZOfbNTjsws z|Jb@Pc8lA!cp+A@=Z;=$-#2{(YD>1<>AnZ$&V+;_$KX98K6wzmgduu*ctq|LC%4Y3 zH}(PbAoWlO=`>a?anOCyZL|R0Aiv*WH>#8ooL}=z;}DK|s*$?+%SqHzmE=}3LytiE zsG^_H_y!LHmYGBo+?wQcqTT89*pf)(5HxG};#BQomm&3_OClAhSw&WXW@*#Pv@?{R z@|i>*$JbcAcA$f83=pbKfF-3fjX&TpUx>1WC~WC5D6ai)^rl1C-4 zMaP#UzqVma@}cXD)+lWT0#pcTcoC$~1b}UFe36n5Qs!^yeKWZ}pi~B;^#m$9;5mhD zBuI{XS^tq(h~RJ)wj(-{3td+K=m9Eii}-{fQEZK55Vh8wC!x6t=Es3Z;n%6K;^_DJ z&L4n99W*F$b2OxNaSLuP0LI~Q3tr)UeAAzWK^Dkq7s(_UIQkXHq5=_)ey088Ms?W~ z9a-Vmq|qM|kDdZ=l28nufBG>E1KYn%r_fE3PWUL0a=9-j`6I~LLB=3?#L()JLjtwZ z5&1kfp|o;@$ft(|r$<6CCLBgBLB3vHF3Ue!SIv) zvC17YwMvPz(F?ddq-(xiD%rNkl7nRzh1V-=0kz}DAs|5?(F?`Q1Bz)Z{O!w8L(Et>*S=z` zQn>|E*0#m26I2)^NUozZae{1@{bh9A#NWWx#Oky9C zAt#1gbp(H)kTXY?4PlIB*OqvOK&+?L8&vfZA0%h_uOSeXT_x(0(CTYyS9fYsOHhGN zDw;2A7^H8VS{_7zMl>XK8`aX-_aKqT$w8eGKZdP?crq^3L28v46To8dqf=7!$yzrO zSCDO}1T4|TQCf~05|Z^=aq-WlyF(!d(eRn61QsE`2lJG+q`N&8@+Oeosb%`fZrtRQ z*@V5i4i@XWW@ttBWHHPooR>lyE)=TTgI*@&WEg?R+eWm_0db$}H1K)56DM`Z z=rpEXp~bixg3tnIA6~6Yp6__d39|7QsJ(6Ghagc1CoBjN-0UG?5`T-91>maZbf2oB0Q4(A|fIrG1 z_`H#vQK-{Uqg2Gxzm37gb@=NT5E^a%^B9pFk9(5dPe6)r$PZfFhI}rt}ru+Ew1GpH|eI=41cbgO4S41RP zCWG|vh`?4h;z+iYs`NlDiQLhETk-J6Rm5@1g?MDYKh)`nn`QMUPZ(xN^4~duX$VnW zMm@OZ$b{^}xQRfFo2T%-VWO$z$fH-3Q%nPeWHp|nWL^mCNS4wZBO$Wk>9u^qGbD7F zcwNMMnRr9QBcv%sFz6j!DTXlwgyN=O0@P$Q<{SyN(^ukAtcDCExl~^a#kIbcxNh=t zShOIHP{Fd3u^D#44GWsH_!d!HOBk!*R{R(C0E+>|JAQRMdO4QKt!CCqnRWAzu4XhX zXEZH{v5Z!bZGez)7_t1vtMX`J^zRK zAmJq2o)y#H-`le&u2S}zYYry1#DJIF*q^db6?my-E~K6*4M6W zYg`!po9#XJcd{6( zeYXB;eYAb9J!Ywv^^qIrbkQl85`)@SFfQAwvHW9W`F!qz;Z0b~U|@Eym^#*Qr9u14 zCuc6e2zj1(s&R;HP*Msax#-R2YvYR%$@J(7d-(TO+aK>YGPeBtj27dEdNMf}PwVFQ zOST;=rk($bJ?kq_r~2&w=bnLd`j4>oA(;*OT0&5dM=xCSNw%^TQ#mHUT)PJO*uHD~ z<}Uwr?!sd~E%>>(wDWH-M)$>XJ74PhOU&aIl4#xQ8SD9kM`(EOs5Xv-)Z#7 zPwv4Z3m7-k;XV)mfVQ~q@~9ZskAww0NKC6k>pzQ=wg*6jMUx!bzCYyQDPPK3A2$vU zdxIX_j#o+d8%bFq9uc|<#7a#d+eYvaZvbB02J11XH8uh41_hyqL}ezT5?6oAwAC$A zf}GB>@g5-v1quo%0Qf)2>whC21qD>t+elvb5O0up6bj6e&@YKciJ8wx2unC2LCzwL zBqG=pXhHU}@Eo#`?)$@UGqTYJ5GR1z{qkLboq`v+@ShR?2_9r^8qMddRm0vZVl=i- z80#mDnf}lIgt2_WWYL)1Pni6F+|lx`@g3v4neSvS{n_e{zK?hGecL diff --git a/cmd/rollcall/client.go b/cmd/rollcall/client.go index 65a460d..4536748 100644 --- a/cmd/rollcall/client.go +++ b/cmd/rollcall/client.go @@ -225,6 +225,8 @@ func runClient(cmd string, args []string) error { return c.tagCmd(cmd, args) case "location": return c.locationCmd(args) + case "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp": + return c.netCmd(cmd, args) default: return c.statelessCmd(cmd, args) } @@ -318,7 +320,7 @@ func (c *client) tagCmd(cmd string, args []string) error { } for _, t := range defs { if t.Name == args[1] { - fmt.Printf("tag %s (%d machines)\ncomment: %s\nmatch: %s\noverlays: %v\n--- cloud_config ---\n%s\n", t.Name, t.Machines, t.Comment, t.Match, t.Overlays, t.CloudConfig) + fmt.Printf("tag %s (%d machines)\ncomment: %s\nmatch: %s\noverlays: %v\nlayout: %s\n--- cloud_config ---\n%s\n", t.Name, t.Machines, t.Comment, t.Match, t.Overlays, orDash(t.NetLayout), t.CloudConfig) return nil } } @@ -331,9 +333,10 @@ func (c *client) tagCmd(cmd string, args []string) error { match := fs.String("match", "", "Go template over .Node that renders true, e.g. '{{ gt .Node.GPUs 0 }}'") ov := fs.String("overlays", "", "comma-separated overlays for stateless nodes with this tag") cc := fs.String("cloud-config", "", "file with a #cloud-config or #!/bin/sh fragment applied at deployment") + layout := fs.String("layout", "", "network layout for machines with this tag") comment := fs.String("comment", "", "comment") _ = fs.Parse(args[2:]) - t := TagDef{Name: args[1], Match: *match, Comment: *comment} + t := TagDef{Name: args[1], Match: *match, Comment: *comment, NetLayout: *layout} if *ov != "" { t.Overlays = strings.Split(*ov, ",") } diff --git a/cmd/rollcall/config.go b/cmd/rollcall/config.go index e3a3db3..6033ab4 100644 --- a/cmd/rollcall/config.go +++ b/cmd/rollcall/config.go @@ -32,6 +32,11 @@ type Config struct { // disk-based deployment OSDir string `json:"os_dir"` // ///{disk.raw.zst,meta.json}, served under /os/ OSSources string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync` + + // region/rack: this server runs an embedded rack controller named RackName; + // remote rack controllers (rollcall rack) authenticate with RackSecret + RackName string `json:"rack_name"` + RackSecret string `json:"rack_secret"` } // BMCHint maps a machine MAC to a known BMC (virtual lab nodes). @@ -78,5 +83,8 @@ func loadConfig(path string) (*Config, error) { if c.OSSources == "" { c.OSSources = "/etc/rollcall/os-sources.json" } + if c.RackName == "" { + c.RackName, _ = os.Hostname() + } return c, nil } diff --git a/cmd/rollcall/deploy.go b/cmd/rollcall/deploy.go index 6fcf7f7..d06c8f2 100644 --- a/cmd/rollcall/deploy.go +++ b/cmd/rollcall/deploy.go @@ -49,6 +49,8 @@ type DeploySpec struct { Packages []string `json:"packages,omitempty"` UserData string `json:"user_data,omitempty"` // full #cloud-config override Wipe string `json:"wipe,omitempty"` // release: quick | full + NetworkConfig string `json:"network_config,omitempty"` + LinkRules string `json:"link_rules,omitempty"` } type seedFile struct { @@ -165,7 +167,11 @@ func (s *Store) SetDeployed(ctx context.Context, id, os, version string) error { } func (s *Store) SetReleased(ctx context.Context, id string) error { - _, err := s.pool.Exec(ctx, `UPDATE machines SET state='ready', os=NULL, os_version=NULL, deployed_at=NULL, booted_at=NULL, last_seen=now() WHERE id=$1`, id) + if _, err := s.pool.Exec(ctx, `DELETE FROM ip_addresses WHERE machine_id=$1 AND kind='auto'`, id); err != nil { + return err + } + _, err := s.pool.Exec(ctx, `UPDATE machines SET state='ready', os=NULL, os_version=NULL, deployed_at=NULL, booted_at=NULL, last_seen=now(), + mgmt_ip=(SELECT ip FROM ip_addresses a WHERE a.machine_id=machines.id ORDER BY ip LIMIT 1), ib_ip=NULL WHERE id=$1`, id) return err } @@ -253,6 +259,11 @@ func yamlList(l []string) string { } func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile { + base := a.cfg.ServerURL + m, merr := a.store.Get(context.Background(), j.MachineID) + if merr == nil { + base = a.urlFor(m) + } meta := fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", j.ID, sp.Hostname) ud := sp.UserData if ud == "" { @@ -274,20 +285,27 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile { if len(sp.Packages) > 0 { fmt.Fprintf(&b, "packages: %s\n", yamlList(sp.Packages)) } - fmt.Fprintf(&b, "phone_home:\n url: %s/api/v1/phone-home/%s\n post: [instance_id, hostname, fqdn, pub_key_ed25519]\n tries: 10\n", a.cfg.ServerURL, token) + fmt.Fprintf(&b, "phone_home:\n url: %s/api/v1/phone-home/%s\n post: [instance_id, hostname, fqdn, pub_key_ed25519]\n tries: 10\n", base, token) b.WriteString("final_message: \"rollcall: deployed $hostname up after $UPTIME s\"\n") ud = b.String() } - if m, err := a.store.Get(context.Background(), j.MachineID); err == nil { + if merr == nil { if frags, err := a.tagFragments(context.Background(), m); err == nil && len(frags) > 0 { ud = multipartUserData(ud, frags) } } - return []seedFile{ + files := []seedFile{ {Path: "var/lib/cloud/seed/nocloud/meta-data", Mode: "0600", Content: meta}, {Path: "var/lib/cloud/seed/nocloud/user-data", Mode: "0600", Content: ud}, {Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\n"}, } + if sp.NetworkConfig != "" { + files = append(files, seedFile{Path: "var/lib/cloud/seed/nocloud/network-config", Mode: "0600", Content: sp.NetworkConfig}) + } + if sp.LinkRules != "" { + files = append(files, seedFile{Path: "etc/udev/rules.d/70-rollcall-net.rules", Mode: "0644", Content: sp.LinkRules}) + } + return files } // ---------------------------------------------------------------- PXE @@ -337,9 +355,9 @@ LABEL recommission ` // deployAppend derives the deploy-mode cmdline from commission_append (same image, no agent token). -func (a *App) deployAppend(token string) string { +func (a *App) deployAppend(base, token string) string { var parts []string - for _, p := range strings.Fields(a.cfg.CommissionAppend) { + for _, p := range strings.Fields(a.appendFor(base)) { if strings.HasPrefix(p, "rollcall.token=") || strings.HasPrefix(p, "rollcall.nopoweroff") { continue } @@ -351,13 +369,13 @@ func (a *App) deployAppend(token string) string { func (a *App) writePXEDeploy(m *Machine, j *Job, sp DeploySpec, token string) error { body := fmt.Sprintf(deployTmpl, time.Now().Format(time.RFC3339), m.ID, j.ID, j.Kind, sp.OS, sp.Version, sp.Disk, - m.ID, j.Kind, sp.OS, j.ID, a.deployAppend(token), a.cfg.CommissionAppend) + m.ID, j.Kind, sp.OS, j.ID, a.deployAppend(a.urlFor(m), token), a.appendFor(a.urlFor(m))) return a.writePXEBody(m, body) } func (a *App) writePXEDeployed(m *Machine) error { body := fmt.Sprintf(deployedTmpl, time.Now().Format(time.RFC3339), m.ID, deref(m.OS), deref(m.OSVersion), - deref(m.NodeName), deref(m.OS), m.ID, deref(m.OS), a.cfg.CommissionAppend) + deref(m.NodeName), deref(m.OS), m.ID, deref(m.OS), a.appendFor(a.urlFor(m))) return a.writePXEBody(m, body) } @@ -443,40 +461,64 @@ func (a *App) handleDeploy(w http.ResponseWriter, r *http.Request) { fail(w, 409, err) return } - // node name + management IP: keep an existing one, otherwise allocate like a stateless node + // node name: keep the existing one unless another is asked for name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP) if req.Hostname != "" && req.Hostname != name { name = "" } if name == "" { - if name, mgmt, ib, err = a.allocate(r.Context(), str(a.cfg.ClusterVars["name_format"]), req.Hostname); err != nil { + if name, _, _, err = a.allocatePools(r.Context(), str(a.cfg.ClusterVars["name_format"]), req.Hostname, false); err != nil { fail(w, 409, err) return } - if err := a.store.SetNodeAddr(r.Context(), m.ID, name, mgmt, ib); err != nil { - fail(w, 500, err) + m.NodeName = &name + } + // addresses: the network plan (layout, IPAM); without a network model, the legacy pools + netcfg, links := "", "" + var warnings []string + if len(a.subnetsSnapshot()) > 0 { + plan, err := a.netPlan(r.Context(), m, true) + if err != nil { + fail(w, 409, err) return } - m, _ = a.store.Get(r.Context(), m.ID) - if err := a.exportDnsmasq(r.Context()); err != nil { - log.Printf("dnsmasq export: %v", err) + warnings = plan.Warnings + if pm, pib := plan.primaryIPs(); pm != "" { + mgmt, ib = pm, pib + } + nb, _ := json.MarshalIndent(networkConfig(plan), "", " ") + netcfg = string(nb) + "\n" + links = linkRules(plan) + } else if mgmt == "" { + if _, mgmt, ib, err = a.allocatePools(r.Context(), "", name, true); err != nil { + fail(w, 409, err) + return } } + if err := a.store.SetNodeAddr(r.Context(), m.ID, name, mgmt, ib); err != nil { + fail(w, 500, err) + return + } + m, _ = a.store.Get(r.Context(), m.ID) keys := req.SSHKeys if keys == nil { keys = a.configSSHKeys() } sp := DeploySpec{OS: meta.Name, Version: meta.Version, - ImageURL: a.cfg.ServerURL + "/os/" + meta.Name + "/" + meta.Version + "/disk.raw.zst", + ImageURL: a.urlFor(m) + "/os/" + meta.Name + "/" + meta.Version + "/disk.raw.zst", SHA256: meta.SHA256, Size: meta.Size, RawSize: meta.RawSize, Disk: disk.Path, DiskSerial: disk.Serial, Hostname: name, Domain: str(a.cfg.ClusterVars["domain"]), SSHKeys: keys, PackageUpgrade: req.PackageUpgrade, - Packages: req.Packages, UserData: req.UserData} + Packages: req.Packages, UserData: req.UserData, NetworkConfig: netcfg, LinkRules: links} out, err := a.startJob(r.Context(), m, "deploy", sp, req.Boot) if err != nil { fail(w, 409, err) return } out["mgmt_ip"] = mgmt + out["rack"] = a.rackOf(m) + if len(warnings) > 0 { + out["warnings"] = warnings + } writeJSON(w, 200, out) } diff --git a/cmd/rollcall/ipmi.go b/cmd/rollcall/ipmi.go index 53192a9..09a165c 100644 --- a/cmd/rollcall/ipmi.go +++ b/cmd/rollcall/ipmi.go @@ -1,14 +1,8 @@ package main import ( - "bytes" - "context" "encoding/json" "fmt" - "os/exec" - "strconv" - "strings" - "time" ) // bmcCreds is what the server needs to talk to a machine's BMC. @@ -54,16 +48,3 @@ func (a *App) credsFor(m *Machine) (*bmcCreds, error) { } return &bmcCreds{Address: b.Address, Port: b.Port, User: b.User, Password: pw}, nil } - -// ipmi runs ipmitool server-side; the password never leaves this process. -func (a *App) ipmi(ctx context.Context, c *bmcCreds, args ...string) (string, error) { - ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - full := append([]string{"-I", "lanplus", "-H", c.Address, "-p", strconv.Itoa(c.Port), "-U", c.User, "-E"}, args...) - cmd := exec.CommandContext(ctx, a.cfg.IPMITool, full...) - cmd.Env = []string{"IPMI_PASSWORD=" + c.Password, "PATH=/usr/sbin:/usr/bin:/sbin:/bin"} - var out bytes.Buffer - cmd.Stdout, cmd.Stderr = &out, &out - err := cmd.Run() - return strings.TrimSpace(out.String()), err -} diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index 5c0a455..ab486c8 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -26,7 +26,8 @@ func usage() { _, _ = os.Stderr.WriteString(`usage: rollcall in a terminal: the full-screen console rollcall tui [-dump WxH -page P -machine ID -plain] - rollcall serve [-config PATH] + rollcall serve [-config PATH] region: API, database, web UI, embedded rack controller + rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines rollcall version rollcall keygen rollcall list @@ -52,6 +53,14 @@ func usage() { rollcall location import FILE.csv [-dry] columns: id|serial|mac|node_name, building, room, rack, unit, asset_tag rollcall location racks | rack NAME -building B -room R [-height 42] rollcall list [-tag T] [-building B] [-room R] [-rack K] + rollcall fabric list | add NAME [-kind ethernet|infiniband] | rm NAME + rollcall vlan list | add FABRIC VID [-dhcp] [-rack R] [-secondary R] [-mtu N] [-name N] | rm FABRIC VID + rollcall subnet list | show CIDR | add CIDR -fabric F [-vid N] [-purpose P] [-gateway G] [-dns a,b] [-domain D] | rm CIDR [-force] + rollcall range add CIDR dynamic|reserved|static START END | rm ID + rollcall ip list [-subnet CIDR] | reserve IP [-name N] | release IP + rollcall layout list | show NAME | add NAME FILE.json | rm NAME + rollcall net show ID | set ID [-layout L] [-rack R] [-static IFACE=IP] [-unstatic IP] + rollcall controllers | controller rm NAME | dhcp CONTROLLER (deploy, power and location take tag:NAME instead of ID to act on every machine with that tag) env: ROLLCALL_URL (default from config listen), ROLLCALL_TOKEN, ROLLCALL_CONFIG; away from the server the console saves URL and token in ~/.config/rollcall/client.json @@ -76,6 +85,10 @@ func main() { } cmd, args := os.Args[1], os.Args[2:] switch cmd { + case "rack": + if err := runRack(args); err != nil { + log.Fatal(err) + } case "tui", "console": if err := tuiDump(args); err != nil { fmt.Fprintln(os.Stderr, "error:", err) @@ -94,7 +107,8 @@ func main() { fmt.Println(randomToken(24)) case "list", "show", "events", "power", "bootdev", "recommission", "delete", "images", "profile", "assign", "unassign", "overlay", "render", - "deploy", "release", "jobs", "job", "tag", "tags", "location": + "deploy", "release", "jobs", "job", "tag", "tags", "location", + "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp": if err := runClient(cmd, args); err != nil { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) diff --git a/cmd/rollcall/netapi.go b/cmd/rollcall/netapi.go new file mode 100644 index 0000000..39da35d --- /dev/null +++ b/cmd/rollcall/netapi.go @@ -0,0 +1,848 @@ +package main + +// HTTP API and CLI for the network model and rack controllers. + +import ( + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "os" + "strconv" + "strings" +) + +// ---------------------------------------------------------------- HTTP + +func (a *App) netChanged(r *http.Request) { + a.reloadNet(r.Context()) + a.hub.bump() +} + +func (a *App) handleFabrics(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + fs, err := a.store.Fabrics(r.Context()) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, fs) + return + } + var f Fabric + if err := json.NewDecoder(r.Body).Decode(&f); err != nil || !tagNameRe.MatchString(f.Name) { + fail(w, 400, errors.New("name is required: lowercase letters, digits, . _ -")) + return + } + if f.Kind != "" && f.Kind != "ethernet" && f.Kind != "infiniband" { + fail(w, 400, errors.New("kind is ethernet or infiniband")) + return + } + if err := a.store.PutFabric(r.Context(), f); err != nil { + fail(w, 500, err) + return + } + a.netChanged(r) + writeJSON(w, 200, f) +} + +func (a *App) handleFabricDelete(w http.ResponseWriter, r *http.Request) { + if err := a.store.DeleteFabric(r.Context(), r.PathValue("name")); err != nil { + fail(w, 500, err) + return + } + a.netChanged(r) + writeJSON(w, 200, obj{"deleted": r.PathValue("name")}) +} + +func (a *App) handleVLANs(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + vs, err := a.store.VLANs(r.Context()) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, vs) + return + } + var v VLAN + if err := json.NewDecoder(r.Body).Decode(&v); err != nil || v.Fabric == "" || v.VID < 0 || v.VID > 4094 && v.VID < 0x8000 { + fail(w, 400, errors.New("fabric and vid (0 untagged, 1-4094, or an IB pkey) are required")) + return + } + if v.DHCP && v.PrimaryRack == "" { + v.PrimaryRack = a.cfg.RackName + } + racks := a.racksSnapshot() + for _, rk := range []string{v.PrimaryRack, v.SecondaryRack} { + if _, ok := racks[rk]; rk != "" && !ok { + fail(w, 400, fmt.Errorf("rack controller %s is not registered", rk)) + return + } + } + id, err := a.store.PutVLAN(r.Context(), v) + if err != nil { + fail(w, 500, err) + return + } + v.ID = id + a.netChanged(r) + writeJSON(w, 200, v) +} + +func (a *App) handleVLANDelete(w http.ResponseWriter, r *http.Request) { + vid, _ := strconv.Atoi(r.PathValue("vid")) + if err := a.store.DeleteVLAN(r.Context(), r.PathValue("fabric"), vid); err != nil { + fail(w, 500, err) + return + } + a.netChanged(r) + writeJSON(w, 200, obj{"deleted": fmt.Sprintf("%s/%d", r.PathValue("fabric"), vid)}) +} + +func (a *App) handleSubnets(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + ss, err := a.store.Subnets(r.Context()) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, ss) + return + } + var sn Subnet + if err := json.NewDecoder(r.Body).Decode(&sn); err != nil { + fail(w, 400, err) + return + } + ip, n, err := net.ParseCIDR(sn.CIDR) + if err != nil || ip.To4() == nil { + fail(w, 400, errors.New("cidr must be an IPv4 network, e.g. 10.0.0.0/24")) + return + } + sn.CIDR = n.String() + if sn.Gateway != "" && !sn.contains(sn.Gateway) { + fail(w, 400, fmt.Errorf("gateway %s is outside %s", sn.Gateway, sn.CIDR)) + return + } + switch sn.Purpose { + case "", "provisioning", "mgmt", "bmc", "storage", "ib", "other": + default: + fail(w, 400, errors.New("purpose is provisioning, mgmt, bmc, storage, ib or other")) + return + } + if sn.Fabric != "" && sn.VLANID == 0 { + vs, _ := a.store.VLANs(r.Context()) + for _, v := range vs { + if v.Fabric == sn.Fabric && v.VID == sn.VID { + sn.VLANID = v.ID + } + } + if sn.VLANID == 0 { // a subnet on a VLAN not declared yet: create it + fs, _ := a.store.Fabrics(r.Context()) + known := false + for _, f := range fs { + known = known || f.Name == sn.Fabric + } + if !known { + fail(w, 400, fmt.Errorf("fabric %s does not exist", sn.Fabric)) + return + } + if sn.VLANID, err = a.store.PutVLAN(r.Context(), VLAN{Fabric: sn.Fabric, VID: sn.VID}); err != nil { + fail(w, 500, err) + return + } + } + } + for _, other := range a.subnetsSnapshot() { + if other.CIDR == sn.CIDR { + continue + } + if _, on, err := net.ParseCIDR(other.CIDR); err == nil && (on.Contains(n.IP) || n.Contains(on.IP)) { + fail(w, 409, fmt.Errorf("%s overlaps %s", sn.CIDR, other.CIDR)) + return + } + } + if err := a.store.PutSubnet(r.Context(), sn); err != nil { + fail(w, 500, err) + return + } + a.adoptIPs(r.Context()) + a.netChanged(r) + writeJSON(w, 200, sn) +} + +func (a *App) handleSubnetDelete(w http.ResponseWriter, r *http.Request) { + cidr := r.PathValue("cidr") + if ips, _ := a.store.IPs(r.Context(), cidr, ""); len(ips) > 0 && r.URL.Query().Get("force") != "1" { + fail(w, 409, fmt.Errorf("%s still has %d address(es) handed out (force=1 drops them)", cidr, len(ips))) + return + } + if err := a.store.DeleteSubnet(r.Context(), cidr); err != nil { + fail(w, 500, err) + return + } + a.netChanged(r) + writeJSON(w, 200, obj{"deleted": cidr}) +} + +func (a *App) handleRanges(w http.ResponseWriter, r *http.Request) { + var rg IPRange + if err := json.NewDecoder(r.Body).Decode(&rg); err != nil { + fail(w, 400, err) + return + } + if rg.Kind != "dynamic" && rg.Kind != "reserved" && rg.Kind != "static" { + fail(w, 400, errors.New("kind is dynamic, reserved or static")) + return + } + sn := subnetByRef(a.subnetsSnapshot(), rg.Subnet) + if sn == nil { + fail(w, 404, fmt.Errorf("subnet %s: %w", rg.Subnet, errSubnetUnknown)) + return + } + lo, ok1 := ip4(rg.Start) + hi, ok2 := ip4(rg.End) + if !ok1 || !ok2 || hi < lo || !sn.contains(rg.Start) || !sn.contains(rg.End) { + fail(w, 400, fmt.Errorf("range %s-%s is not inside %s", rg.Start, rg.End, sn.CIDR)) + return + } + for _, o := range sn.Ranges { + ol, _ := ip4(o.Start) + oh, _ := ip4(o.End) + if lo <= oh && ol <= hi { + fail(w, 409, fmt.Errorf("overlaps the %s range %s-%s", o.Kind, o.Start, o.End)) + return + } + } + rg.Subnet = sn.CIDR + id, err := a.store.AddRange(r.Context(), rg) + if err != nil { + fail(w, 500, err) + return + } + rg.ID = id + a.netChanged(r) + writeJSON(w, 200, rg) +} + +func (a *App) handleRangeDelete(w http.ResponseWriter, r *http.Request) { + id, _ := strconv.ParseInt(r.PathValue("id"), 10, 64) + if err := a.store.DeleteRange(r.Context(), id); err != nil { + fail(w, 500, err) + return + } + a.netChanged(r) + writeJSON(w, 200, obj{"deleted": id}) +} + +func (a *App) handleIPs(w http.ResponseWriter, r *http.Request) { + switch r.Method { + case http.MethodGet: + ips, err := a.store.IPs(r.Context(), r.URL.Query().Get("subnet"), r.URL.Query().Get("machine")) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, ips) + case http.MethodPost: // reserve one address (a switch, a PDU, a VIP) + var x IPAddr + if err := json.NewDecoder(r.Body).Decode(&x); err != nil { + fail(w, 400, err) + return + } + sn := subnetFor(a.subnetsSnapshot(), x.IP) + if sn == nil { + fail(w, 404, fmt.Errorf("%s is in no subnet", x.IP)) + return + } + ok, err := a.store.ClaimIP(r.Context(), IPAddr{IP: x.IP, Subnet: sn.CIDR, Kind: "reserved", Hostname: x.Hostname}) + if err != nil { + fail(w, 500, err) + return + } + if !ok { + fail(w, 409, fmt.Errorf("%s is taken", x.IP)) + return + } + a.netChanged(r) + writeJSON(w, 200, obj{"reserved": x.IP, "subnet": sn.CIDR}) + } +} + +func (a *App) handleIPDelete(w http.ResponseWriter, r *http.Request) { + if err := a.store.ReleaseIP(r.Context(), r.PathValue("ip")); err != nil { + fail(w, 500, err) + return + } + a.netChanged(r) + writeJSON(w, 200, obj{"released": r.PathValue("ip")}) +} + +func validLayout(l NetLayout) error { + if !tagNameRe.MatchString(l.Name) { + return errors.New("name is required: lowercase letters, digits, . _ -") + } + if len(l.Interfaces) == 0 { + return errors.New("a layout needs at least one interface") + } + names := map[string]bool{} + for _, i := range l.Interfaces { + switch i.Type { + case "", "physical": + case "bond": + if len(i.Members) == 0 { + return fmt.Errorf("bond %s has no members", i.Name) + } + case "vlan": + if !names[i.Parent] || i.VID < 1 || i.VID > 4094 { + return fmt.Errorf("vlan %q needs a parent defined above it and a vid 1-4094", i.Name) + } + default: + return fmt.Errorf("interface type %q: physical, bond or vlan", i.Type) + } + for _, lk := range i.Links { + switch lk.Mode { + case "", "auto", "static", "dhcp", "link_up": + default: + return fmt.Errorf("link mode %q: auto, static, dhcp or link_up", lk.Mode) + } + } + n := i.Name + if n == "" && i.Type == "vlan" { + n = fmt.Sprintf("%s.%d", i.Parent, i.VID) + } + names[n] = true + } + return nil +} + +func (a *App) handleLayouts(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + ls, err := a.store.Layouts(r.Context()) + if err != nil { + fail(w, 500, err) + return + } + ms, _ := a.store.List(r.Context()) + for i := range ls { + for _, m := range ms { + if l, _ := a.layoutFor(r.Context(), m); l != nil && strings.HasPrefix(l.Name, ls[i].Name) { + ls[i].Machines++ + } + } + } + writeJSON(w, 200, ls) + return + } + var l NetLayout + if err := json.NewDecoder(r.Body).Decode(&l); err != nil { + fail(w, 400, err) + return + } + if err := validLayout(l); err != nil { + fail(w, 400, err) + return + } + if err := a.store.PutLayout(r.Context(), l); err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, l) +} + +func (a *App) handleLayoutDelete(w http.ResponseWriter, r *http.Request) { + if err := a.store.DeleteLayout(r.Context(), r.PathValue("name")); err != nil { + fail(w, 409, err) + return + } + writeJSON(w, 200, obj{"deleted": r.PathValue("name")}) +} + +// handleMachineNetwork: GET shows the plan and the network-config it renders +// (without handing out addresses); POST sets the layout and pinned addresses. +func (a *App) handleMachineNetwork(w http.ResponseWriter, r *http.Request) { + m, err := a.store.Get(r.Context(), r.PathValue("id")) + if err != nil { + fail(w, 500, err) + return + } + if r.Method == http.MethodPost { + var req struct { + Layout *string `json:"layout"` + Rack *string `json:"rack_controller"` + Static []struct{ Iface, IP string } + Unstatic []string `json:"unstatic"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + fail(w, 400, err) + return + } + if req.Layout != nil { + if *req.Layout != "" { + ls, _ := a.store.Layouts(r.Context()) + found := false + for _, l := range ls { + found = found || l.Name == *req.Layout + } + if !found { + fail(w, 404, fmt.Errorf("layout %s does not exist", *req.Layout)) + return + } + } + if err := a.store.SetMachineLayout(r.Context(), m.ID, *req.Layout); err != nil { + fail(w, 500, err) + return + } + } + if req.Rack != nil { + if _, ok := a.racksSnapshot()[*req.Rack]; *req.Rack != "" && !ok { + fail(w, 404, fmt.Errorf("rack controller %s is not registered", *req.Rack)) + return + } + if err := a.store.SetRackController(r.Context(), m.ID, *req.Rack); err != nil { + fail(w, 500, err) + return + } + m, _ = a.store.Get(r.Context(), m.ID) + _ = a.writePXEAny(r.Context(), m) // its PXE file moves to the new rack + } + for _, ip := range req.Unstatic { + have, _ := a.store.IPs(r.Context(), "", m.ID) + for _, x := range have { + if x.IP == ip && x.Kind == "static" { + _ = a.store.ReleaseIP(r.Context(), ip) + } + } + } + for _, st := range req.Static { + sn := subnetFor(a.subnetsSnapshot(), st.IP) + if sn == nil { + fail(w, 400, fmt.Errorf("%s is in no subnet", st.IP)) + return + } + have, _ := a.store.IPs(r.Context(), sn.CIDR, m.ID) + for _, x := range have { // one pinned address per interface and subnet + if x.Iface == st.Iface && x.IP != st.IP { + _ = a.store.ReleaseIP(r.Context(), x.IP) + } + } + if ok, err := a.store.ClaimIP(r.Context(), IPAddr{IP: st.IP, Subnet: sn.CIDR, MachineID: m.ID, Iface: st.Iface, Kind: "static", Hostname: deref(m.NodeName)}); err != nil { + fail(w, 500, err) + return + } else if !ok { + cur, _ := a.store.IPs(r.Context(), sn.CIDR, m.ID) + mine := false + for _, x := range cur { + mine = mine || x.IP == st.IP + } + if !mine { + fail(w, 409, fmt.Errorf("%s is taken", st.IP)) + return + } + _, _ = a.store.pool.Exec(r.Context(), `UPDATE ip_addresses SET kind='static', iface=$2 WHERE ip=$1`, st.IP, st.Iface) + } + } + _ = a.store.AddEvent(r.Context(), m.ID, "network", fmt.Sprintf("layout=%v static=%v unstatic=%v", deref(req.Layout), req.Static, req.Unstatic)) + a.netChanged(r) + m, _ = a.store.Get(r.Context(), m.ID) + } + plan, err := a.netPlan(r.Context(), m, false) + if err != nil { + fail(w, 409, err) + return + } + ips, _ := a.store.IPs(r.Context(), "", m.ID) + writeJSON(w, 200, obj{"id": m.ID, "layout": plan.Layout, "plan": plan, "network_config": networkConfig(plan), "addresses": ips, + "rack_controller": a.rackOf(m), "modeled": len(a.subnetsSnapshot()) > 0}) +} + +// ---------------------------------------------------------------- CLI + +func (c *client) netCmd(cmd string, args []string) error { + need := func(n int) { + if len(args) < n { + usage() + } + } + sub := "" + if len(args) > 0 { + sub = args[0] + } + switch cmd { + case "fabric": + switch sub { + case "", "list": + var fs []Fabric + if err := c.do("GET", "/api/v1/fabrics", nil, &fs); err != nil { + return err + } + fmt.Printf("%-16s %-11s %s\n", "FABRIC", "KIND", "COMMENT") + for _, f := range fs { + fmt.Printf("%-16s %-11s %s\n", f.Name, f.Kind, f.Comment) + } + case "add": + need(2) + fs := newFlags("fabric add") + kind := fs.String("kind", "ethernet", "ethernet | infiniband") + comment := fs.String("comment", "", "comment") + _ = fs.Parse(args[2:]) + return c.do("POST", "/api/v1/fabrics", Fabric{Name: args[1], Kind: *kind, Comment: *comment}, nil) + case "rm": + need(2) + return c.do("DELETE", "/api/v1/fabrics/"+args[1], nil, nil) + default: + usage() + } + case "vlan": + switch sub { + case "", "list": + var vs []VLAN + if err := c.do("GET", "/api/v1/vlans", nil, &vs); err != nil { + return err + } + fmt.Printf("%-14s %5s %-14s %5s %-5s %-14s %s\n", "FABRIC", "VID", "NAME", "MTU", "DHCP", "RACK", "SECONDARY") + for _, v := range vs { + fmt.Printf("%-14s %5d %-14s %5d %-5v %-14s %s\n", v.Fabric, v.VID, v.Name, v.MTU, v.DHCP, v.PrimaryRack, v.SecondaryRack) + } + case "add", "set": + need(3) + fs := newFlags("vlan add") + name := fs.String("name", "", "name") + mtu := fs.Int("mtu", 1500, "MTU") + dhcp := fs.Bool("dhcp", false, "serve DHCP/PXE on this VLAN") + rack := fs.String("rack", "", "rack controller serving it (default: the region's embedded one)") + sec := fs.String("secondary", "", "second rack controller (the dynamic range is split)") + _ = fs.Parse(args[3:]) + vid, _ := strconv.Atoi(args[2]) + var v VLAN + if err := c.do("POST", "/api/v1/vlans", VLAN{Fabric: args[1], VID: vid, Name: *name, MTU: *mtu, DHCP: *dhcp, PrimaryRack: *rack, SecondaryRack: *sec}, &v); err != nil { + return err + } + fmt.Printf("vlan %s/%d: dhcp=%v rack=%s\n", v.Fabric, v.VID, v.DHCP, v.PrimaryRack) + case "rm": + need(3) + return c.do("DELETE", "/api/v1/vlans/"+args[1]+"/"+args[2], nil, nil) + default: + usage() + } + case "subnet": + switch sub { + case "", "list": + var ss []Subnet + if err := c.do("GET", "/api/v1/subnets", nil, &ss); err != nil { + return err + } + fmt.Printf("%-18s %-12s %-14s %-12s %-15s %-12s %9s %s\n", "SUBNET", "PURPOSE", "FABRIC/VLAN", "DHCP RACK", "GATEWAY", "DOMAIN", "USED", "RANGES") + for _, s := range ss { + dhcp := "-" + if s.DHCP { + dhcp = s.PrimaryRack + if s.SecondaryRack != "" { + dhcp += "+" + s.SecondaryRack + } + } + var rs []string + for _, r := range s.Ranges { + rs = append(rs, fmt.Sprintf("%s:%s-%s", r.Kind, r.Start, lastOctets(r.End))) + } + fmt.Printf("%-18s %-12s %-14s %-12s %-15s %-12s %4d/%-4d %s\n", s.CIDR, s.Purpose, fmt.Sprintf("%s/%d", orDash(s.Fabric), s.VID), dhcp, orDash(s.Gateway), orDash(s.Domain), s.Used, s.Size, strings.Join(rs, " ")) + } + case "show": + need(2) + var ss []Subnet + if err := c.do("GET", "/api/v1/subnets", nil, &ss); err != nil { + return err + } + for _, s := range ss { + if s.CIDR == args[1] || s.Name == args[1] { + fmt.Printf("%s %s fabric %s vlan %d mtu %d dhcp=%v rack=%s %s\n gateway %s dns %v domain %s\n", s.CIDR, s.Purpose, s.Fabric, s.VID, s.MTU, s.DHCP, s.PrimaryRack, s.SecondaryRack, orDash(s.Gateway), s.DNS, orDash(s.Domain)) + for _, r := range s.Ranges { + fmt.Printf(" range %-4d %-8s %s - %s %s\n", r.ID, r.Kind, r.Start, r.End, r.Comment) + } + var ips []IPAddr + if err := c.do("GET", "/api/v1/ips?subnet="+s.CIDR, nil, &ips); err != nil { + return err + } + fmt.Printf(" %d of %d addresses in use\n", len(ips), s.Size) + for _, x := range ips { + fmt.Printf(" %-15s %-8s %-16s %-10s %s\n", x.IP, x.Kind, orDash(x.MachineID), x.Iface, x.Hostname) + } + return nil + } + } + return errNotFound + case "add", "set": + need(2) + fs := newFlags("subnet add") + fabric := fs.String("fabric", "", "fabric (the VLAN is created when missing)") + vid := fs.Int("vid", 0, "VLAN id (0 = untagged)") + purpose := fs.String("purpose", "provisioning", "provisioning | mgmt | bmc | storage | ib | other") + gw := fs.String("gateway", "", "default gateway") + dns := fs.String("dns", "", "comma-separated DNS servers") + domain := fs.String("domain", "", "DNS domain") + name := fs.String("name", "", "name") + comment := fs.String("comment", "", "comment") + _ = fs.Parse(args[2:]) + sn := Subnet{CIDR: args[1], Fabric: *fabric, VID: *vid, Purpose: *purpose, Gateway: *gw, Domain: *domain, Name: *name, Comment: *comment, DNS: []string{}} + if *dns != "" { + sn.DNS = strings.Split(*dns, ",") + } + var out Subnet + if err := c.do("POST", "/api/v1/subnets", sn, &out); err != nil { + return err + } + fmt.Printf("subnet %s saved\n", out.CIDR) + case "rm": + need(2) + q := "" + if len(args) > 2 && args[2] == "-force" { + q = "?force=1" + } + return c.do("DELETE", "/api/v1/subnets/"+args[1]+q, nil, nil) + default: + usage() + } + case "range": + switch sub { + case "add": + need(5) + fs := newFlags("range add") + comment := fs.String("comment", "", "comment") + _ = fs.Parse(args[5:]) + var rg IPRange + if err := c.do("POST", "/api/v1/ranges", IPRange{Subnet: args[1], Kind: args[2], Start: args[3], End: args[4], Comment: *comment}, &rg); err != nil { + return err + } + fmt.Printf("range %d: %s %s-%s on %s\n", rg.ID, rg.Kind, rg.Start, rg.End, rg.Subnet) + case "rm": + need(2) + return c.do("DELETE", "/api/v1/ranges/"+args[1], nil, nil) + default: + usage() + } + case "ip": + switch sub { + case "", "list": + fs := newFlags("ip list") + subnet := fs.String("subnet", "", "only this subnet") + _ = fs.Parse(args[min(1, len(args)):]) + var ips []IPAddr + if err := c.do("GET", "/api/v1/ips?subnet="+*subnet, nil, &ips); err != nil { + return err + } + fmt.Printf("%-15s %-18s %-8s %-16s %-10s %s\n", "IP", "SUBNET", "KIND", "MACHINE", "IFACE", "HOSTNAME") + for _, x := range ips { + fmt.Printf("%-15s %-18s %-8s %-16s %-10s %s\n", x.IP, x.Subnet, x.Kind, orDash(x.MachineID), x.Iface, x.Hostname) + } + case "reserve": + need(2) + fs := newFlags("ip reserve") + host := fs.String("name", "", "what it is (a switch, a VIP)") + _ = fs.Parse(args[2:]) + return c.do("POST", "/api/v1/ips", IPAddr{IP: args[1], Hostname: *host}, nil) + case "release": + need(2) + return c.do("DELETE", "/api/v1/ips/"+args[1], nil, nil) + default: + usage() + } + case "layout": + switch sub { + case "", "list": + var ls []NetLayout + if err := c.do("GET", "/api/v1/layouts", nil, &ls); err != nil { + return err + } + fmt.Printf("%-18s %8s %-50s %s\n", "LAYOUT", "MACHINES", "INTERFACES", "COMMENT") + for _, l := range ls { + fmt.Printf("%-18s %8d %-50s %s\n", l.Name, l.Machines, trunc(layoutSummary(l), 50), l.Comment) + } + case "show": + need(2) + var ls []NetLayout + if err := c.do("GET", "/api/v1/layouts", nil, &ls); err != nil { + return err + } + for _, l := range ls { + if l.Name == args[1] { + b, _ := json.MarshalIndent(l, "", " ") + fmt.Println(string(b)) + return nil + } + } + return errNotFound + case "add": + need(3) + fs := newFlags("layout add") + comment := fs.String("comment", "", "comment") + _ = fs.Parse(args[3:]) + b, err := readFileArg(args[2]) + if err != nil { + return err + } + var l NetLayout + if err := json.Unmarshal(b, &l); err != nil { + return fmt.Errorf("%s: %w", args[2], err) + } + l.Name = args[1] + if *comment != "" { + l.Comment = *comment + } + return c.do("POST", "/api/v1/layouts", l, nil) + case "rm": + need(2) + return c.do("DELETE", "/api/v1/layouts/"+args[1], nil, nil) + default: + usage() + } + case "net": + need(2) + switch sub { + case "show": + var d obj + if err := c.do("GET", "/api/v1/machines/"+args[1]+"/network", nil, &d); err != nil { + return err + } + printNet(d) + case "set": + fs := newFlags("net set") + layout := fs.String("layout", "\x00", "layout name (\"\" = from tags / default)") + rack := fs.String("rack", "\x00", "rack controller serving it (a machine moved to another segment)") + var statics, unstatic multiFlag + fs.Var(&statics, "static", "IFACE=IP pins an address (repeatable)") + fs.Var(&unstatic, "unstatic", "IP drops a pinned address (repeatable)") + _ = fs.Parse(args[2:]) + req := obj{"unstatic": []string(unstatic)} + if *layout != "\x00" { + req["layout"] = *layout + } + if *rack != "\x00" { + req["rack_controller"] = *rack + } + var st []obj + for _, s := range statics { + i, ip, ok := strings.Cut(s, "=") + if !ok { + return fmt.Errorf("-static wants IFACE=IP") + } + st = append(st, obj{"iface": i, "ip": ip}) + } + req["static"] = st + var d obj + if err := c.do("POST", "/api/v1/machines/"+args[1]+"/network", req, &d); err != nil { + return err + } + printNet(d) + default: + usage() + } + case "controllers", "controller": + switch { + case cmd == "controller" && sub == "rm": + need(2) + return c.do("DELETE", "/api/v1/controllers/"+args[1], nil, nil) + default: + var rs []RackController + if err := c.do("GET", "/api/v1/controllers", nil, &rs); err != nil { + return err + } + fmt.Printf("%-14s %-9s %-7s %-24s %8s %-20s %s\n", "CONTROLLER", "KIND", "ONLINE", "URL", "MACHINES", "LAST SEEN", "INTERFACES") + for _, r := range rs { + var ifs []string + for _, i := range r.Interfaces { + ifs = append(ifs, i.Name+"="+strings.Join(i.Addrs, ",")) + } + fmt.Printf("%-14s %-9s %-7v %-24s %8d %-20s %s\n", r.Name, r.Kind, r.Online, r.URL, r.Machines, timeStr(r.LastSeen), strings.Join(ifs, " ")) + } + } + case "dhcp": + need(1) + data, err := c.raw("GET", "/api/v1/controllers/"+args[0]+"/dhcp") + if err != nil { + return err + } + os.Stdout.Write(data) + } + return nil +} + +type multiFlag []string + +func (m *multiFlag) String() string { return strings.Join(*m, ",") } +func (m *multiFlag) Set(v string) error { *m = append(*m, v); return nil } + +func lastOctets(ip string) string { + if i := strings.LastIndex(ip, "."); i >= 0 { + return ip[i+1:] + } + return ip +} + +func layoutSummary(l NetLayout) string { + var p []string + for _, i := range l.Interfaces { + var links []string + for _, lk := range i.Links { + links = append(links, lk.Subnet+":"+orDash(lk.Mode)) + } + switch i.Type { + case "bond": + p = append(p, fmt.Sprintf("%s(%s)[%s]", orDash(i.Name), strings.Join(i.Members, "+"), strings.Join(links, ","))) + case "vlan": + p = append(p, fmt.Sprintf("%s.%d[%s]", i.Parent, i.VID, strings.Join(links, ","))) + default: + p = append(p, fmt.Sprintf("%s[%s]", orDash(i.Select), strings.Join(links, ","))) + } + } + return strings.Join(p, " ") +} + +func printNet(d obj) { + fmt.Printf("%s layout %s rack controller %s\n", str(d["id"]), str(d["layout"]), str(d["rack_controller"])) + if d["modeled"] == false { + fmt.Println(" no subnets are defined: deployed machines use DHCP (rollcall subnet add …)") + } + plan, _ := d["plan"].(obj) + for _, iv := range toList(plan["interfaces"]) { + i, _ := iv.(obj) + desc := str(i["type"]) + switch desc { + case "physical": + desc = str(i["mac"]) + case "bond": + var ms []string + for _, x := range toList(i["members"]) { + ms = append(ms, str(x)) + } + desc = "bond " + str(i["bond_mode"]) + " of " + strings.Join(ms, ",") + case "vlan": + desc = fmt.Sprintf("vlan %s on %s", str(i["vid"]), str(i["parent"])) + } + mtu := "" + if num(i["mtu"]) > 0 { + mtu = fmt.Sprintf(" mtu %d", int(num(i["mtu"]))) + } + fmt.Printf(" %-12s %s%s\n", str(i["name"]), desc, mtu) + for _, lv := range toList(i["links"]) { + l, _ := lv.(obj) + ip := str(l["ip"]) + if ip == "" && str(l["mode"]) == "auto" { + ip = "(allocated at deploy)" + } else if ip != "" { + ip = fmt.Sprintf("%s/%d", ip, int(num(l["prefix"]))) + } + gw := "" + if l["default_gateway"] == true { + gw = " default via " + str(l["gateway"]) + } + fmt.Printf(" %-8s %-18s %s%s\n", str(l["mode"]), orDash(str(l["subnet"])), ip, gw) + } + } + for _, w := range toList(plan["warnings"]) { + fmt.Printf(" ! %s\n", str(w)) + } + b, _ := json.MarshalIndent(d["network_config"], " ", " ") + fmt.Printf(" network-config:\n %s\n", b) +} diff --git a/cmd/rollcall/network.go b/cmd/rollcall/network.go new file mode 100644 index 0000000..461b47b --- /dev/null +++ b/cmd/rollcall/network.go @@ -0,0 +1,1214 @@ +package main + +// Network model, after MAAS: fabric → VLAN → subnet → IP ranges, plus the +// addresses handed out (IPAM), interface layouts for machines and the two +// things rendered from it all: the cloud-init network-config a deployed +// machine boots with, and the dnsmasq configuration of the rack controller +// that serves each VLAN. + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "net" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +type Fabric struct { + Name string `json:"name"` + Kind string `json:"kind"` // ethernet | infiniband + Comment string `json:"comment"` +} + +type VLAN struct { + ID int64 `json:"id"` + Fabric string `json:"fabric"` + VID int `json:"vid"` // 0 = untagged; an InfiniBand partition key on IB fabrics + Name string `json:"name"` + MTU int `json:"mtu"` + DHCP bool `json:"dhcp"` // rollcall serves DHCP/PXE on it + PrimaryRack string `json:"primary_rack"` // rack controller that serves it + SecondaryRack string `json:"secondary_rack"` // second one; the dynamic range is split between them +} + +type IPRange struct { + ID int64 `json:"id"` + Subnet string `json:"subnet"` + Kind string `json:"kind"` // dynamic (DHCP pool for PXE) | reserved (never handed out) | static (where machines get addresses) + Start string `json:"start"` + End string `json:"end"` + Comment string `json:"comment"` +} + +type Subnet struct { + CIDR string `json:"cidr"` + Name string `json:"name"` + VLANID int64 `json:"vlan_id"` + Fabric string `json:"fabric"` + VID int `json:"vid"` + MTU int `json:"mtu"` + DHCP bool `json:"dhcp"` + PrimaryRack string `json:"primary_rack"` + SecondaryRack string `json:"secondary_rack"` + Purpose string `json:"purpose"` // provisioning | mgmt | bmc | storage | ib | other + Gateway string `json:"gateway"` + DNS []string `json:"dns"` + Domain string `json:"domain"` + Comment string `json:"comment"` + Ranges []IPRange `json:"ranges"` + Used int `json:"used"` + Size int `json:"size"` +} + +type IPAddr struct { + IP string `json:"ip"` + Subnet string `json:"subnet"` + MachineID string `json:"machine_id"` + Iface string `json:"iface"` + Kind string `json:"kind"` // auto (allocated) | static (pinned) | reserved + Hostname string `json:"hostname"` + Created time.Time `json:"created"` +} + +// NetLayout describes a machine's interfaces in terms that fit many machines: +// NICs are picked by selectors, not by MAC. +type NetLayout struct { + Name string `json:"name"` + Comment string `json:"comment"` + Interfaces []LayoutIface `json:"interfaces"` + Machines int `json:"machines,omitempty"` + Raw map[string]any `json:"-"` +} + +type LayoutIface struct { + Name string `json:"name,omitempty"` // bond0, bond0.200; physical: the NIC's own name when empty + Type string `json:"type"` // physical | bond | vlan + Select string `json:"select,omitempty"` // physical: boot | name:X | mac:X | fastest:N | ib:N + Members []string `json:"members,omitempty"` // bond: selectors, each may match several NICs + BondMode string `json:"bond_mode,omitempty"` // 802.3ad (default) | active-backup | balance-xor | balance-alb + Parent string `json:"parent,omitempty"` // vlan: the interface it sits on + VID int `json:"vid,omitempty"` + MTU int `json:"mtu,omitempty"` + Links []LayoutLink `json:"links,omitempty"` +} + +type LayoutLink struct { + Subnet string `json:"subnet"` // CIDR, subnet name, or "boot" (where the machine PXE-booted) + Mode string `json:"mode"` // auto | static | dhcp | link_up + IP string `json:"ip,omitempty"` // static + Gateway bool `json:"default_gateway,omitempty"` // this link carries the default route +} + +// ---------------------------------------------------------------- store + +func (s *Store) Fabrics(ctx context.Context) ([]Fabric, error) { + rows, err := s.pool.Query(ctx, `SELECT name, kind, coalesce(comment,'') FROM fabrics ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []Fabric{} + for rows.Next() { + var f Fabric + if err := rows.Scan(&f.Name, &f.Kind, &f.Comment); err != nil { + return nil, err + } + out = append(out, f) + } + return out, rows.Err() +} + +func (s *Store) PutFabric(ctx context.Context, f Fabric) error { + if f.Kind == "" { + f.Kind = "ethernet" + } + _, err := s.pool.Exec(ctx, `INSERT INTO fabrics(name, kind, comment) VALUES ($1,$2,$3) ON CONFLICT (name) DO UPDATE SET kind=$2, comment=$3`, f.Name, f.Kind, f.Comment) + return err +} + +func (s *Store) DeleteFabric(ctx context.Context, name string) error { + _, err := s.pool.Exec(ctx, `DELETE FROM fabrics WHERE name=$1`, name) + return err +} + +func (s *Store) VLANs(ctx context.Context) ([]VLAN, error) { + rows, err := s.pool.Query(ctx, `SELECT id, fabric, vid, coalesce(name,''), mtu, dhcp, coalesce(primary_rack,''), coalesce(secondary_rack,'') FROM vlans ORDER BY fabric, vid`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []VLAN{} + for rows.Next() { + var v VLAN + if err := rows.Scan(&v.ID, &v.Fabric, &v.VID, &v.Name, &v.MTU, &v.DHCP, &v.PrimaryRack, &v.SecondaryRack); err != nil { + return nil, err + } + out = append(out, v) + } + return out, rows.Err() +} + +// PutVLAN upserts by (fabric, vid) and returns the id. +func (s *Store) PutVLAN(ctx context.Context, v VLAN) (int64, error) { + if v.MTU <= 0 { + v.MTU = 1500 + } + var id int64 + err := s.pool.QueryRow(ctx, `INSERT INTO vlans(fabric, vid, name, mtu, dhcp, primary_rack, secondary_rack) VALUES ($1,$2,$3,$4,$5,nullif($6,''),nullif($7,'')) + ON CONFLICT (fabric, vid) DO UPDATE SET name=$3, mtu=$4, dhcp=$5, primary_rack=nullif($6,''), secondary_rack=nullif($7,'') RETURNING id`, + v.Fabric, v.VID, v.Name, v.MTU, v.DHCP, v.PrimaryRack, v.SecondaryRack).Scan(&id) + return id, err +} + +func (s *Store) DeleteVLAN(ctx context.Context, fabric string, vid int) error { + _, err := s.pool.Exec(ctx, `DELETE FROM vlans WHERE fabric=$1 AND vid=$2`, fabric, vid) + return err +} + +func (s *Store) Subnets(ctx context.Context) ([]Subnet, error) { + rows, err := s.pool.Query(ctx, `SELECT s.cidr, coalesce(s.name,''), coalesce(s.vlan_id,0), coalesce(v.fabric,''), coalesce(v.vid,0), coalesce(v.mtu,1500), + coalesce(v.dhcp,false), coalesce(v.primary_rack,''), coalesce(v.secondary_rack,''), s.purpose, coalesce(s.gateway,''), s.dns, + coalesce(s.domain,''), coalesce(s.comment,''), (SELECT count(*) FROM ip_addresses a WHERE a.subnet=s.cidr) + FROM subnets s LEFT JOIN vlans v ON v.id = s.vlan_id ORDER BY s.cidr`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []Subnet{} + for rows.Next() { + var sn Subnet + var dns []byte + if err := rows.Scan(&sn.CIDR, &sn.Name, &sn.VLANID, &sn.Fabric, &sn.VID, &sn.MTU, &sn.DHCP, &sn.PrimaryRack, &sn.SecondaryRack, + &sn.Purpose, &sn.Gateway, &dns, &sn.Domain, &sn.Comment, &sn.Used); err != nil { + return nil, err + } + _ = json.Unmarshal(dns, &sn.DNS) + if sn.DNS == nil { + sn.DNS = []string{} + } + if _, n, err := net.ParseCIDR(sn.CIDR); err == nil { + ones, bits := n.Mask.Size() + sn.Size = max(0, 1<<(bits-ones)-2) + } + sn.Ranges = []IPRange{} + out = append(out, sn) + } + if err := rows.Err(); err != nil { + return nil, err + } + rr, err := s.pool.Query(ctx, `SELECT id, subnet, kind, start_ip, end_ip, coalesce(comment,'') FROM ip_ranges ORDER BY subnet, start_ip`) + if err != nil { + return nil, err + } + defer rr.Close() + for rr.Next() { + var r IPRange + if err := rr.Scan(&r.ID, &r.Subnet, &r.Kind, &r.Start, &r.End, &r.Comment); err != nil { + return nil, err + } + for i := range out { + if out[i].CIDR == r.Subnet { + out[i].Ranges = append(out[i].Ranges, r) + } + } + } + return out, rr.Err() +} + +func (s *Store) PutSubnet(ctx context.Context, sn Subnet) error { + if sn.Purpose == "" { + sn.Purpose = "provisioning" + } + if sn.DNS == nil { + sn.DNS = []string{} + } + dns, _ := json.Marshal(sn.DNS) + var vlan any + if sn.VLANID > 0 { + vlan = sn.VLANID + } + _, err := s.pool.Exec(ctx, `INSERT INTO subnets(cidr, name, vlan_id, purpose, gateway, dns, domain, comment) VALUES ($1,$2,$3,$4,nullif($5,''),$6,nullif($7,''),$8) + ON CONFLICT (cidr) DO UPDATE SET name=$2, vlan_id=$3, purpose=$4, gateway=nullif($5,''), dns=$6, domain=nullif($7,''), comment=$8`, + sn.CIDR, sn.Name, vlan, sn.Purpose, sn.Gateway, dns, sn.Domain, sn.Comment) + return err +} + +func (s *Store) DeleteSubnet(ctx context.Context, cidr string) error { + tag, err := s.pool.Exec(ctx, `DELETE FROM subnets WHERE cidr=$1`, cidr) + if err == nil && tag.RowsAffected() == 0 { + return errNotFound + } + return err +} + +func (s *Store) AddRange(ctx context.Context, r IPRange) (int64, error) { + var id int64 + err := s.pool.QueryRow(ctx, `INSERT INTO ip_ranges(subnet, kind, start_ip, end_ip, comment) VALUES ($1,$2,$3,$4,$5) RETURNING id`, + r.Subnet, r.Kind, r.Start, r.End, r.Comment).Scan(&id) + return id, err +} + +func (s *Store) DeleteRange(ctx context.Context, id int64) error { + _, err := s.pool.Exec(ctx, `DELETE FROM ip_ranges WHERE id=$1`, id) + return err +} + +func (s *Store) IPs(ctx context.Context, subnet, machineID string) ([]IPAddr, error) { + q := `SELECT ip, subnet, coalesce(machine_id,''), coalesce(iface,''), kind, coalesce(hostname,''), created FROM ip_addresses WHERE true` + var args []any + if subnet != "" { + args = append(args, subnet) + q += fmt.Sprintf(" AND subnet=$%d", len(args)) + } + if machineID != "" { + args = append(args, machineID) + q += fmt.Sprintf(" AND machine_id=$%d", len(args)) + } + rows, err := s.pool.Query(ctx, q+` ORDER BY subnet, ip`, args...) + if err != nil { + return nil, err + } + defer rows.Close() + out := []IPAddr{} + for rows.Next() { + var a IPAddr + if err := rows.Scan(&a.IP, &a.Subnet, &a.MachineID, &a.Iface, &a.Kind, &a.Hostname, &a.Created); err != nil { + return nil, err + } + out = append(out, a) + } + sort.Slice(out, func(i, j int) bool { return ipLess(out[i].IP, out[j].IP) }) + return out, rows.Err() +} + +// ClaimIP records an address; false when it is taken. +func (s *Store) ClaimIP(ctx context.Context, a IPAddr) (bool, error) { + tag, err := s.pool.Exec(ctx, `INSERT INTO ip_addresses(ip, subnet, machine_id, iface, kind, hostname) VALUES ($1,$2,nullif($3,''),nullif($4,''),$5,nullif($6,'')) ON CONFLICT (ip) DO NOTHING`, + a.IP, a.Subnet, a.MachineID, a.Iface, a.Kind, a.Hostname) + if err != nil { + return false, err + } + return tag.RowsAffected() == 1, nil +} + +func (s *Store) ReleaseIP(ctx context.Context, ip string) error { + _, err := s.pool.Exec(ctx, `DELETE FROM ip_addresses WHERE ip=$1`, ip) + return err +} + +// FreeMachineIPs releases a machine's addresses of the given kinds (all when none given). +func (s *Store) FreeMachineIPs(ctx context.Context, machineID string, kinds ...string) error { + if len(kinds) == 0 { + _, err := s.pool.Exec(ctx, `DELETE FROM ip_addresses WHERE machine_id=$1`, machineID) + return err + } + _, err := s.pool.Exec(ctx, `DELETE FROM ip_addresses WHERE machine_id=$1 AND kind = ANY($2)`, machineID, kinds) + return err +} + +func (s *Store) Layouts(ctx context.Context) ([]NetLayout, error) { + rows, err := s.pool.Query(ctx, `SELECT name, spec, coalesce(comment,'') FROM net_layouts ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []NetLayout{} + for rows.Next() { + var l NetLayout + var spec []byte + if err := rows.Scan(&l.Name, &spec, &l.Comment); err != nil { + return nil, err + } + var body struct { + Interfaces []LayoutIface `json:"interfaces"` + } + _ = json.Unmarshal(spec, &body) + l.Interfaces = body.Interfaces + out = append(out, l) + } + return out, rows.Err() +} + +func (s *Store) PutLayout(ctx context.Context, l NetLayout) error { + spec, _ := json.Marshal(struct { + Interfaces []LayoutIface `json:"interfaces"` + }{l.Interfaces}) + _, err := s.pool.Exec(ctx, `INSERT INTO net_layouts(name, spec, comment) VALUES ($1,$2,$3) ON CONFLICT (name) DO UPDATE SET spec=$2, comment=$3`, l.Name, spec, l.Comment) + return err +} + +func (s *Store) DeleteLayout(ctx context.Context, name string) error { + var n int + _ = s.pool.QueryRow(ctx, `SELECT (SELECT count(*) FROM machines WHERE net_layout=$1) + (SELECT count(*) FROM tags WHERE net_layout=$1)`, name).Scan(&n) + if n > 0 { + return fmt.Errorf("layout %s is used by %d machine(s)/tag(s)", name, n) + } + _, err := s.pool.Exec(ctx, `DELETE FROM net_layouts WHERE name=$1`, name) + return err +} + +func (s *Store) SetMachineLayout(ctx context.Context, id, layout string) error { + _, err := s.pool.Exec(ctx, `UPDATE machines SET net_layout=nullif($2,'') WHERE id=$1`, id, layout) + return err +} + +// ---------------------------------------------------------------- IPv4 helpers + +func ip4(s string) (uint32, bool) { + ip := net.ParseIP(strings.TrimSpace(s)).To4() + if ip == nil { + return 0, false + } + return binary.BigEndian.Uint32(ip), true +} + +func u2ip(u uint32) string { + b := make(net.IP, 4) + binary.BigEndian.PutUint32(b, u) + return b.String() +} + +func ipLess(a, b string) bool { + x, ok1 := ip4(a) + y, ok2 := ip4(b) + if ok1 && ok2 { + return x < y + } + return a < b +} + +func stripPrefix(s string) string { ip, _, _ := strings.Cut(s, "/"); return ip } + +func (sn Subnet) contains(ip string) bool { + _, n, err := net.ParseCIDR(sn.CIDR) + p := net.ParseIP(stripPrefix(ip)) + return err == nil && p != nil && n.Contains(p) +} + +func (sn Subnet) bits() int { + if _, n, err := net.ParseCIDR(sn.CIDR); err == nil { + ones, _ := n.Mask.Size() + return ones + } + return 24 +} + +func (sn Subnet) netmask() string { + if _, n, err := net.ParseCIDR(sn.CIDR); err == nil { + return net.IP(n.Mask).String() + } + return "255.255.255.0" +} + +func subnetFor(subnets []Subnet, ip string) *Subnet { + for i := range subnets { + if subnets[i].contains(ip) { + return &subnets[i] + } + } + return nil +} + +func subnetByRef(subnets []Subnet, ref string) *Subnet { + for i := range subnets { + if subnets[i].CIDR == ref || (subnets[i].Name != "" && subnets[i].Name == ref) { + return &subnets[i] + } + } + return nil +} + +// freeCandidates lists the addresses IPAM may hand out in a subnet, in order: +// inside static ranges when the subnet has any, never in dynamic or reserved +// ranges, never the network, broadcast, gateway, a rack controller or a used one. +func freeCandidates(sn Subnet, used map[string]bool, avoid []string, limit int) []string { + _, n, err := net.ParseCIDR(sn.CIDR) + if err != nil || n.IP.To4() == nil { + return nil + } + ones, bits := n.Mask.Size() + if bits-ones > 16 { // a /15 or larger: refuse to walk it + ones = bits - 16 + } + base := binary.BigEndian.Uint32(n.IP.To4()) + last := base | (1<<(bits-ones) - 1) + type span struct{ lo, hi uint32 } + var static, blocked []span + for _, r := range sn.Ranges { + lo, ok1 := ip4(r.Start) + hi, ok2 := ip4(r.End) + if !ok1 || !ok2 { + continue + } + if r.Kind == "static" { + static = append(static, span{lo, hi}) + } else { + blocked = append(blocked, span{lo, hi}) + } + } + skip := map[uint32]bool{base: true, last: true} + if g, ok := ip4(sn.Gateway); ok { + skip[g] = true + } + for _, a := range avoid { + if u, ok := ip4(stripPrefix(a)); ok { + skip[u] = true + } + } + in := func(x uint32, ss []span) bool { + for _, s := range ss { + if x >= s.lo && x <= s.hi { + return true + } + } + return false + } + var out []string + for x := base + 1; x < last; x++ { + if skip[x] || in(x, blocked) || (len(static) > 0 && !in(x, static)) { + continue + } + ip := u2ip(x) + if used[ip] { + continue + } + out = append(out, ip) + if limit > 0 && len(out) >= limit { + break + } + } + return out +} + +// allocIP hands out the first free address of a subnet. +func (a *App) allocIP(ctx context.Context, sn Subnet, machineID, iface, hostname string) (string, error) { + existing, err := a.store.IPs(ctx, sn.CIDR, "") + if err != nil { + return "", err + } + used := map[string]bool{} + for _, x := range existing { + used[x.IP] = true + } + for _, ip := range freeCandidates(sn, used, a.rackAddresses(), 256) { + ok, err := a.store.ClaimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: machineID, Iface: iface, Kind: "auto", Hostname: hostname}) + if err != nil { + return "", err + } + if ok { + return ip, nil + } + } + return "", fmt.Errorf("subnet %s is full", sn.CIDR) +} + +// ---------------------------------------------------------------- machine NICs from the inventory + +type invNIC struct { + Name string + MAC string + IB bool + Speed int // Mb/s, 0 unknown + Link bool + IPv4 []string +} + +var speedRe = regexp.MustCompile(`(\d+)`) + +func machineNICs(m *Machine) []invNIC { + var inv obj + _ = json.Unmarshal(m.Inventory, &inv) + var out []invNIC + for _, nv := range toList(inv["network"]) { + n, _ := nv.(obj) + nic := invNIC{Name: str(n["name"]), MAC: normMAC(str(n["mac"])), IB: str(n["link_type"]) == "infiniband", Link: n["link"] == true} + if s := speedRe.FindString(str(n["speed"])); s != "" { + nic.Speed, _ = strconv.Atoi(s) + } + for _, ip := range toList(n["ipv4"]) { + nic.IPv4 = append(nic.IPv4, str(ip)) + } + if nic.Name != "" { + out = append(out, nic) + } + } + return out +} + +func bootNIC(nics []invNIC, macs []string) *invNIC { + if len(macs) > 0 { + for i := range nics { + if nics[i].MAC == normMAC(macs[0]) { + return &nics[i] + } + } + } + for i := range nics { + if len(nics[i].IPv4) > 0 { + return &nics[i] + } + } + if len(nics) > 0 { + return &nics[0] + } + return nil +} + +// selectNICs resolves one selector against the inventory. +func selectNICs(nics []invNIC, boot *invNIC, sel string) []invNIC { + sel = strings.TrimSpace(sel) + kind, arg, _ := strings.Cut(sel, ":") + switch kind { + case "", "boot": + if boot != nil { + return []invNIC{*boot} + } + case "name": + for _, n := range nics { + if n.Name == arg { + return []invNIC{n} + } + } + case "mac": + for _, n := range nics { + if n.MAC == normMAC(arg) { + return []invNIC{n} + } + } + case "fastest", "all": + var eth []invNIC + for _, n := range nics { + if !n.IB { + eth = append(eth, n) + } + } + sort.SliceStable(eth, func(i, j int) bool { + if eth[i].Link != eth[j].Link { + return eth[i].Link + } + if eth[i].Speed != eth[j].Speed { + return eth[i].Speed > eth[j].Speed + } + return eth[i].Name < eth[j].Name + }) + if kind == "all" { + var up []invNIC + for _, n := range eth { + if n.Link { + up = append(up, n) + } + } + return up + } + k, _ := strconv.Atoi(arg) + if k <= 0 { + k = 1 + } + return eth[:min(k, len(eth))] + case "ib": + var ib []invNIC + for _, n := range nics { + if n.IB { + ib = append(ib, n) + } + } + sort.Slice(ib, func(i, j int) bool { return ib[i].Name < ib[j].Name }) + k, _ := strconv.Atoi(arg) + if k < len(ib) { + return []invNIC{ib[k]} + } + default: + for _, n := range nics { + if n.Name == sel { + return []invNIC{n} + } + } + } + return nil +} + +// ---------------------------------------------------------------- plan + +type PlanLink struct { + Subnet string `json:"subnet"` + Mode string `json:"mode"` + IP string `json:"ip"` + Prefix int `json:"prefix"` + Gateway string `json:"gateway"` + Default bool `json:"default_gateway"` + DNS []string `json:"dns"` + Domain string `json:"domain"` +} + +type PlanIface struct { + Name string `json:"name"` + Type string `json:"type"` + MAC string `json:"mac,omitempty"` + IB bool `json:"ib,omitempty"` + Members []string `json:"members,omitempty"` + BondMode string `json:"bond_mode,omitempty"` + Parent string `json:"parent,omitempty"` + VID int `json:"vid,omitempty"` + MTU int `json:"mtu,omitempty"` + Links []PlanLink `json:"links"` + nics []invNIC +} + +type NetPlan struct { + Layout string `json:"layout"` + Ifaces []PlanIface `json:"interfaces"` + Warnings []string `json:"warnings"` + member map[string]invNIC +} + +func defaultLayout(bootSubnet *Subnet) *NetLayout { + link := LayoutLink{Subnet: "boot", Mode: "auto", Gateway: true} + if bootSubnet == nil { + link = LayoutLink{Subnet: "boot", Mode: "dhcp"} + } + return &NetLayout{Name: "(default)", Interfaces: []LayoutIface{{Type: "physical", Select: "boot", Links: []LayoutLink{link}}}} +} + +// resolvePlan turns a layout into concrete interfaces for one machine. alloc is +// called for auto links that have no address yet (nil: show them as unallocated). +func resolvePlan(l *NetLayout, nics []invNIC, boot *invNIC, subnets []Subnet, have []IPAddr, + alloc func(sn Subnet, iface string) (string, error)) *NetPlan { + var bootSubnet *Subnet + if boot != nil { + for _, ip := range boot.IPv4 { + if s := subnetFor(subnets, ip); s != nil { + bootSubnet = s + break + } + } + } + if l == nil { + l = defaultLayout(bootSubnet) + } + p := &NetPlan{Layout: l.Name, Warnings: []string{}, member: map[string]invNIC{}} + used := map[string]bool{} + byName := map[string]bool{} + haveFor := func(iface, subnet string) (string, string) { + for _, kind := range []string{"static", "auto"} { + for _, a := range have { + if a.Subnet == subnet && a.Kind == kind && (a.Iface == iface || a.Iface == "") { + return a.IP, kind + } + } + } + return "", "" + } + // selectors pick among the NICs no earlier interface took ("boot" always means the boot NIC) + avail := func() []invNIC { + var out []invNIC + for _, n := range nics { + if !used[n.MAC] { + out = append(out, n) + } + } + return out + } + for _, li := range l.Interfaces { + pi := PlanIface{Type: li.Type, MTU: li.MTU, Links: []PlanLink{}} + switch li.Type { + case "", "physical": + pi.Type = "physical" + pool := avail() + if li.Select == "" || li.Select == "boot" { + pool = nics + } + ns := selectNICs(pool, boot, li.Select) + var nic *invNIC + for i := range ns { + if !used[ns[i].MAC] { + nic = &ns[i] + break + } + } + if nic == nil { + p.Warnings = append(p.Warnings, fmt.Sprintf("no NIC matches %q", li.Select)) + continue + } + used[nic.MAC] = true + pi.Name, pi.MAC, pi.IB, pi.nics = nic.Name, nic.MAC, nic.IB, []invNIC{*nic} + if li.Name != "" { + pi.Name = li.Name + } + case "bond": + pi.Name = li.Name + if pi.Name == "" { + pi.Name = "bond0" + } + pi.BondMode = li.BondMode + if pi.BondMode == "" { + pi.BondMode = "802.3ad" + } + for _, sel := range li.Members { + pool := avail() + if sel == "boot" { + pool = nics + } + for _, n := range selectNICs(pool, boot, sel) { + if !used[n.MAC] { + used[n.MAC] = true + pi.Members = append(pi.Members, n.Name) + pi.nics = append(pi.nics, n) + p.member[n.Name] = n + } + } + } + if len(pi.Members) == 0 { + p.Warnings = append(p.Warnings, fmt.Sprintf("bond %s: no member NICs match %v", pi.Name, li.Members)) + continue + } + case "vlan": + if !byName[li.Parent] { + p.Warnings = append(p.Warnings, fmt.Sprintf("vlan %d: parent %q is not defined above it", li.VID, li.Parent)) + continue + } + pi.Parent, pi.VID = li.Parent, li.VID + pi.Name = li.Name + if pi.Name == "" { + pi.Name = fmt.Sprintf("%s.%d", li.Parent, li.VID) + } + default: + p.Warnings = append(p.Warnings, "unknown interface type "+li.Type) + continue + } + byName[pi.Name] = true + for _, lk := range li.Links { + mode := lk.Mode + if mode == "" { + mode = "auto" + } + var sn *Subnet + switch { + case lk.Subnet == "boot", lk.Subnet == "" && mode != "dhcp" && mode != "link_up": + sn = bootSubnet + case lk.Subnet != "": + sn = subnetByRef(subnets, lk.Subnet) + } + if sn == nil { + if mode == "dhcp" || mode == "link_up" { + pi.Links = append(pi.Links, PlanLink{Mode: mode}) + continue + } + p.Warnings = append(p.Warnings, fmt.Sprintf("%s: subnet %q is not defined", pi.Name, lk.Subnet)) + continue + } + pl := PlanLink{Subnet: sn.CIDR, Mode: mode, Prefix: sn.bits(), Gateway: sn.Gateway, Default: lk.Gateway, DNS: sn.DNS, Domain: sn.Domain} + if pi.MTU == 0 && sn.MTU > 0 && sn.MTU != 1500 { + pi.MTU = sn.MTU + } + if ip, kind := haveFor(pi.Name, sn.CIDR); kind == "static" && mode != "dhcp" && mode != "link_up" { + pl.Mode, pl.IP = "static", ip + } else { + switch mode { + case "static": + pl.IP = lk.IP + if pl.IP == "" { + p.Warnings = append(p.Warnings, fmt.Sprintf("%s: static link on %s without an address", pi.Name, sn.CIDR)) + } + case "auto": + pl.IP = ip + if pl.IP == "" && alloc != nil { + got, err := alloc(*sn, pi.Name) + if err != nil { + p.Warnings = append(p.Warnings, fmt.Sprintf("%s: %v", pi.Name, err)) + } + pl.IP = got + } + } + } + pi.Links = append(pi.Links, pl) + } + p.Ifaces = append(p.Ifaces, pi) + } + // one default route: the flagged link, else the first addressed link with a gateway on the boot NIC, else anywhere + hasDefault := false + for _, pi := range p.Ifaces { + for _, pl := range pi.Links { + hasDefault = hasDefault || pl.Default + } + } + if !hasDefault { + pickDefault := func(onBoot bool) bool { + for i := range p.Ifaces { + carriesBoot := false + for _, n := range p.Ifaces[i].nics { + carriesBoot = carriesBoot || (boot != nil && n.MAC == boot.MAC) + } + if onBoot && !carriesBoot { + continue + } + for j := range p.Ifaces[i].Links { + pl := &p.Ifaces[i].Links[j] + if pl.Gateway != "" && (pl.Mode == "auto" || pl.Mode == "static") { + pl.Default = true + return true + } + } + } + return false + } + if !pickDefault(true) { + pickDefault(false) + } + } + return p +} + +// primaryIPs returns the address carrying the default route and the first IB address. +func (p *NetPlan) primaryIPs() (mgmt, ib string) { + for _, pi := range p.Ifaces { + for _, pl := range pi.Links { + if pl.Default && pl.IP != "" && mgmt == "" { + mgmt = pl.IP + } + if pi.IB && pl.IP != "" && ib == "" { + ib = pl.IP + } + } + } + if mgmt == "" { + for _, pi := range p.Ifaces { + for _, pl := range pi.Links { + if pl.IP != "" && !pi.IB && mgmt == "" { + mgmt = pl.IP + } + } + } + } + return +} + +// networkConfig renders the plan as cloud-init network-config version 2 +// (netplan syntax; cloud-init converts it for NetworkManager, networkd or ENI). +func networkConfig(p *NetPlan) map[string]any { + eth, bonds, vlans := map[string]any{}, map[string]any{}, map[string]any{} + fill := func(e map[string]any, pi PlanIface) { + if pi.MTU > 0 { + e["mtu"] = pi.MTU + } + var addrs, dns, search []string + dhcp := false + for _, pl := range pi.Links { + switch pl.Mode { + case "dhcp": + dhcp = true + case "auto", "static": + if pl.IP == "" { + continue + } + addrs = append(addrs, fmt.Sprintf("%s/%d", pl.IP, pl.Prefix)) + if pl.Default && pl.Gateway != "" { + e["routes"] = []map[string]any{{"to": "0.0.0.0/0", "via": pl.Gateway}} + } + for _, d := range pl.DNS { + if !hasTag(dns, d) { + dns = append(dns, d) + } + } + if pl.Domain != "" && !hasTag(search, pl.Domain) { + search = append(search, pl.Domain) + } + } + } + e["dhcp4"] = dhcp + if len(addrs) > 0 { + e["addresses"] = addrs + } + if len(dns) > 0 || len(search) > 0 { + ns := map[string]any{} + if len(dns) > 0 { + ns["addresses"] = dns + } + if len(search) > 0 { + ns["search"] = search + } + e["nameservers"] = ns + } + } + physical := func(n invNIC, pi *PlanIface) map[string]any { + e := map[string]any{} + if n.IB { + e["match"] = map[string]any{"name": n.Name} + } else { + e["match"] = map[string]any{"macaddress": n.MAC} + e["set-name"] = n.Name + if pi != nil && pi.Name != "" { + e["set-name"] = pi.Name + } + } + return e + } + for _, pi := range p.Ifaces { + switch pi.Type { + case "physical": + if len(pi.nics) == 0 { + continue + } + e := physical(pi.nics[0], &pi) + fill(e, pi) + eth[pi.Name] = e + case "bond": + for _, n := range pi.nics { + e := physical(n, nil) + e["dhcp4"] = false + if pi.MTU > 0 { + e["mtu"] = pi.MTU + } + eth[n.Name] = e + } + params := map[string]any{"mode": pi.BondMode, "mii-monitor-interval": 100} + if pi.BondMode == "802.3ad" { + params["lacp-rate"] = "fast" + params["transmit-hash-policy"] = "layer3+4" + } + e := map[string]any{"interfaces": pi.Members, "parameters": params} + fill(e, pi) + bonds[pi.Name] = e + case "vlan": + e := map[string]any{"id": pi.VID, "link": pi.Parent} + fill(e, pi) + vlans[pi.Name] = e + } + } + nw := map[string]any{"version": 2} + if len(eth) > 0 { + nw["ethernets"] = eth + } + if len(bonds) > 0 { + nw["bonds"] = bonds + } + if len(vlans) > 0 { + nw["vlans"] = vlans + } + return map[string]any{"network": nw} +} + +// linkRules pins every physical NIC of the plan to its name with a udev rule. +// Cloud images of the RHEL family boot with net.ifnames=0 (eth0, eth1, …) +// and cloud-init cannot rename a NIC to a name the kernel already gave it as +// an alternative name; a NAME= rule applies at boot, before any of that. +func linkRules(p *NetPlan) string { + var b strings.Builder + seen := map[string]bool{} + add := func(n invNIC, name string) { + if n.IB || n.MAC == "" || seen[n.MAC] { + return + } + seen[n.MAC] = true + fmt.Fprintf(&b, "SUBSYSTEM==\"net\", ACTION==\"add\", ATTR{address}==\"%s\", NAME=\"%s\"\n", n.MAC, name) + } + for _, pi := range p.Ifaces { + switch pi.Type { + case "physical": + if len(pi.nics) > 0 { + add(pi.nics[0], pi.Name) + } + case "bond": + for _, n := range pi.nics { + add(n, n.Name) + } + } + } + if b.Len() == 0 { + return "" + } + return "# written by rollcall at deploy time: NIC names as the network configuration expects them\n" + b.String() +} + +// layoutFor picks a machine's layout: its own, else the first of its tags that names one. +func (a *App) layoutFor(ctx context.Context, m *Machine) (*NetLayout, error) { + layouts, err := a.store.Layouts(ctx) + if err != nil { + return nil, err + } + find := func(name string) *NetLayout { + for i := range layouts { + if layouts[i].Name == name { + return &layouts[i] + } + } + return nil + } + if m.NetLayout != "" { + if l := find(m.NetLayout); l != nil { + return l, nil + } + return nil, fmt.Errorf("layout %q does not exist", m.NetLayout) + } + tags, err := a.store.Tags(ctx) + if err != nil { + return nil, err + } + eff := effectiveTags(m) + for _, t := range tags { + if t.NetLayout != "" && hasTag(eff, t.Name) { + if l := find(t.NetLayout); l != nil { + cp := *l + cp.Name = l.Name + " (tag " + t.Name + ")" + return &cp, nil + } + } + } + return nil, nil +} + +// netPlan resolves a machine's interfaces; with allocate it hands out the +// addresses its auto links still lack. +func (a *App) netPlan(ctx context.Context, m *Machine, allocate bool) (*NetPlan, error) { + l, err := a.layoutFor(ctx, m) + if err != nil { + return nil, err + } + subnets, err := a.store.Subnets(ctx) + if err != nil { + return nil, err + } + have, err := a.store.IPs(ctx, "", m.ID) + if err != nil { + return nil, err + } + nics := machineNICs(m) + var alloc func(Subnet, string) (string, error) + if allocate { + host := deref(m.NodeName) + alloc = func(sn Subnet, iface string) (string, error) { return a.allocIP(ctx, sn, m.ID, iface, host) } + } + return resolvePlan(l, nics, bootNIC(nics, m.MACs), subnets, have, alloc), nil +} + +// ---------------------------------------------------------------- dnsmasq + +const dnsmasqBase = `# generated by rollcall for rack controller %s; do not edit (rollcall subnet/vlan commands change it) +bind-dynamic +%sdomain-needed +bogus-priv +expand-hosts +log-dhcp +enable-tftp +tftp-root=%s +dhcp-match=set:efi64,option:client-arch,7 +dhcp-match=set:efi64,option:client-arch,9 +dhcp-match=set:bios,option:client-arch,0 +dhcp-boot=tag:bios,pxelinux.0 +dhcp-boot=tag:efi64,efi64/syslinux.efi +dhcp-hostsfile=%s/hosts.dhcp +addn-hosts=%s/hosts +` + +// dnsmasqConfig renders a rack controller's dnsmasq configuration from the +// VLANs it serves; ok is false when it serves none (the file is then left out +// and dnsmasq keeps whatever configuration it has). +func dnsmasqConfig(rc RackController, subnets []Subnet, dnsDir string) (string, bool) { + var served []Subnet + for _, sn := range subnets { + if sn.DHCP && (sn.PrimaryRack == rc.Name || sn.SecondaryRack == rc.Name) { + served = append(served, sn) + } + } + if len(served) == 0 { + return "", false + } + ifaces := rc.DHCPInterfaces + var relayed []string + if len(ifaces) == 0 { + for _, sn := range served { + local := false + for _, ri := range rc.Interfaces { + for _, ad := range ri.Addrs { + if sn.contains(ad) { + local = true + if !hasTag(ifaces, ri.Name) { + ifaces = append(ifaces, ri.Name) + } + } + } + } + if !local { + relayed = append(relayed, sn.CIDR) + } + } + } + var lines strings.Builder + for _, i := range ifaces { + lines.WriteString("interface=" + i + "\n") + } + tftp := rc.TFTPRoot + if tftp == "" { + tftp = "/srv/tftp" + } + var b strings.Builder + fmt.Fprintf(&b, dnsmasqBase, rc.Name, lines.String(), tftp, dnsDir, dnsDir) + for _, cidr := range relayed { + fmt.Fprintf(&b, "# %s has no local interface here: it is served through a DHCP relay; list the interface\n# facing the relay in the rack's dhcp_interfaces\n", cidr) + } + shared := false + domains := []string{} + for _, sn := range served { + shared = shared || sn.SecondaryRack != "" + if sn.Domain != "" && !hasTag(domains, sn.Domain) { + domains = append(domains, sn.Domain) + } + } + if !shared { + b.WriteString("dhcp-authoritative\n") + } + for _, d := range domains { + fmt.Fprintf(&b, "local=/%s/\n", d) + } + for i, sn := range served { + tag := fmt.Sprintf("s%d", i) + fmt.Fprintf(&b, "\n# %s · %s vlan %d · %s\n", sn.CIDR, orDash(sn.Fabric), sn.VID, sn.Purpose) + if sn.Domain != "" { + fmt.Fprintf(&b, "domain=%s,%s\n", sn.Domain, sn.CIDR) + } + dyn := 0 + for _, r := range sn.Ranges { + if r.Kind != "dynamic" { + continue + } + lo, ok1 := ip4(r.Start) + hi, ok2 := ip4(r.End) + if !ok1 || !ok2 || hi < lo { + continue + } + // two racks on one VLAN: each serves its own half of the pool + if sn.SecondaryRack != "" { + mid := lo + (hi-lo)/2 + if rc.Name == sn.PrimaryRack { + hi = mid + } else { + lo = mid + 1 + } + } + fmt.Fprintf(&b, "dhcp-range=set:%s,%s,%s,%s,12h\n", tag, u2ip(lo), u2ip(hi), sn.netmask()) + dyn++ + } + if dyn == 0 { + // static leases only (known machines), no pool for strangers + _, n, _ := net.ParseCIDR(sn.CIDR) + fmt.Fprintf(&b, "dhcp-range=set:%s,%s,static,%s,12h\n", tag, n.IP.String(), sn.netmask()) + } + if sn.Gateway != "" { + fmt.Fprintf(&b, "dhcp-option=tag:%s,option:router,%s\n", tag, sn.Gateway) + } + if len(sn.DNS) > 0 { + fmt.Fprintf(&b, "dhcp-option=tag:%s,option:dns-server,%s\n", tag, strings.Join(sn.DNS, ",")) + } + if sn.MTU > 0 && sn.MTU != 1500 { + fmt.Fprintf(&b, "dhcp-option=tag:%s,option:mtu,%d\n", tag, sn.MTU) + } + } + return b.String(), true +} + +var errSubnetUnknown = errors.New("subnet not found") diff --git a/cmd/rollcall/network_test.go b/cmd/rollcall/network_test.go new file mode 100644 index 0000000..9ee399a --- /dev/null +++ b/cmd/rollcall/network_test.go @@ -0,0 +1,165 @@ +package main + +import ( + "encoding/json" + "strings" + "testing" +) + +func labSubnet() Subnet { + return Subnet{CIDR: "10.0.0.0/24", Gateway: "10.0.0.1", DNS: []string{"10.0.0.1"}, Domain: "lab.local", Fabric: "f0", MTU: 1500, + DHCP: true, PrimaryRack: "r1", Purpose: "provisioning", + Ranges: []IPRange{{Kind: "reserved", Start: "10.0.0.2", End: "10.0.0.20"}, {Kind: "dynamic", Start: "10.0.0.100", End: "10.0.0.200"}}} +} + +func TestFreeCandidates(t *testing.T) { + sn := labSubnet() + got := freeCandidates(sn, map[string]bool{"10.0.0.21": true}, []string{"10.0.0.22/24"}, 3) + if strings.Join(got, " ") != "10.0.0.23 10.0.0.24 10.0.0.25" { + t.Errorf("candidates: %v", got) + } + for _, ip := range freeCandidates(sn, nil, nil, 0) { + if ip == "10.0.0.1" || ip == "10.0.0.0" || ip == "10.0.0.255" || (ipLess("10.0.0.99", ip) && ipLess(ip, "10.0.0.201")) { + t.Fatalf("handed out %s", ip) + } + } + sn.Ranges = append(sn.Ranges, IPRange{Kind: "static", Start: "10.0.0.50", End: "10.0.0.52"}) + if got := freeCandidates(sn, nil, nil, 0); strings.Join(got, " ") != "10.0.0.50 10.0.0.51 10.0.0.52" { + t.Errorf("static range: %v", got) + } +} + +func allocFrom(next ...string) func(Subnet, string) (string, error) { + return func(Subnet, string) (string, error) { + ip := next[0] + next = next[1:] + return ip, nil + } +} + +func TestDefaultPlanAndNetworkConfig(t *testing.T) { + nics := []invNIC{{Name: "eno1", MAC: "aa:bb:cc:00:00:01", Speed: 1000, Link: true, IPv4: []string{"10.0.0.150/24"}}} + p := resolvePlan(nil, nics, &nics[0], []Subnet{labSubnet()}, nil, allocFrom("10.0.0.21")) + if len(p.Warnings) > 0 || len(p.Ifaces) != 1 { + t.Fatalf("plan: %+v", p) + } + if mgmt, _ := p.primaryIPs(); mgmt != "10.0.0.21" { + t.Errorf("mgmt %q", mgmt) + } + b, _ := json.Marshal(networkConfig(p)) + s := string(b) + for _, want := range []string{`"macaddress":"aa:bb:cc:00:00:01"`, `"set-name":"eno1"`, `"addresses":["10.0.0.21/24"]`, + `"routes":[{"to":"0.0.0.0/0","via":"10.0.0.1"}]`, `"search":["lab.local"]`, `"dhcp4":false`} { + if !strings.Contains(s, want) { + t.Errorf("network-config lacks %s:\n%s", want, s) + } + } + // a pinned address wins over allocation + p = resolvePlan(nil, nics, &nics[0], []Subnet{labSubnet()}, []IPAddr{{IP: "10.0.0.60", Subnet: "10.0.0.0/24", Iface: "eno1", Kind: "static"}}, allocFrom("10.0.0.21")) + if p.Ifaces[0].Links[0].IP != "10.0.0.60" || p.Ifaces[0].Links[0].Mode != "static" { + t.Errorf("static pin ignored: %+v", p.Ifaces[0].Links[0]) + } + // no subnet model for the boot network: DHCP, as before + p = resolvePlan(nil, nics, &nics[0], nil, nil, nil) + if p.Ifaces[0].Links[0].Mode != "dhcp" { + t.Errorf("without subnets the default is dhcp, got %+v", p.Ifaces[0].Links) + } +} + +func TestBondVLANLayout(t *testing.T) { + nics := []invNIC{ + {Name: "eno1", MAC: "aa:00:00:00:00:01", Speed: 1000, Link: true, IPv4: []string{"10.0.0.150/24"}}, + {Name: "ens1f0", MAC: "aa:00:00:00:00:02", Speed: 100000, Link: true}, + {Name: "ens1f1", MAC: "aa:00:00:00:00:03", Speed: 100000, Link: true}, + {Name: "ibp1s0", MAC: "80:00:00:00", IB: true, Link: true}, + } + storage := Subnet{CIDR: "10.20.0.0/24", Fabric: "f0", VID: 200, MTU: 9000} + ib := Subnet{CIDR: "10.30.0.0/24", Fabric: "ib0"} + l := &NetLayout{Name: "gpu", Interfaces: []LayoutIface{ + {Type: "physical", Select: "boot", Links: []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}}, + {Type: "bond", Name: "bond0", Members: []string{"fastest:2"}, Links: []LayoutLink{{Mode: "link_up"}}}, + {Type: "vlan", Parent: "bond0", VID: 200, Links: []LayoutLink{{Subnet: "10.20.0.0/24", Mode: "auto"}}}, + {Type: "physical", Select: "ib:0", Links: []LayoutLink{{Subnet: "10.30.0.0/24", Mode: "auto"}}}, + }} + p := resolvePlan(l, nics, &nics[0], []Subnet{labSubnet(), storage, ib}, nil, allocFrom("10.0.0.21", "10.20.0.5", "10.30.0.5")) + if len(p.Warnings) > 0 { + t.Fatalf("warnings: %v", p.Warnings) + } + if got := p.Ifaces[1].Members; strings.Join(got, ",") != "ens1f0,ens1f1" { + t.Errorf("bond members %v (the boot NIC must not be taken)", got) + } + if p.Ifaces[2].Name != "bond0.200" || p.Ifaces[2].MTU != 9000 { + t.Errorf("vlan iface %+v", p.Ifaces[2]) + } + if mgmt, ibip := p.primaryIPs(); mgmt != "10.0.0.21" || ibip != "10.30.0.5" { + t.Errorf("primary %s ib %s", mgmt, ibip) + } + b, _ := json.Marshal(networkConfig(p)) + s := string(b) + for _, want := range []string{`"bonds":{"bond0":`, `"interfaces":["ens1f0","ens1f1"]`, `"mode":"802.3ad"`, `"vlans":{"bond0.200":`, + `"link":"bond0"`, `"id":200`, `"10.20.0.5/24"`, `"mtu":9000`, `"ibp1s0":{"addresses":["10.30.0.5/24"]`} { + if !strings.Contains(s, want) { + t.Errorf("network-config lacks %s:\n%s", want, s) + } + } + if strings.Count(s, "0.0.0.0/0") != 1 { + t.Errorf("want exactly one default route:\n%s", s) + } + rules := linkRules(p) + for _, want := range []string{`ATTR{address}=="aa:00:00:00:00:01", NAME="eno1"`, `ATTR{address}=="aa:00:00:00:00:02", NAME="ens1f0"`, `ATTR{address}=="aa:00:00:00:00:03", NAME="ens1f1"`} { + if !strings.Contains(rules, want) { + t.Errorf("udev rules lack %s:\n%s", want, rules) + } + } + if strings.Contains(rules, "ibp1s0") { + t.Errorf("InfiniBand ports keep their names:\n%s", rules) + } +} + +// NICs of one speed (VMs, a box of identical 25G ports): the bond must not count the boot NIC. +func TestBondSkipsUsedNICs(t *testing.T) { + nics := []invNIC{ + {Name: "ens18", MAC: "bc:00:00:00:00:01", Link: true, IPv4: []string{"10.0.0.150/24"}}, + {Name: "ens19", MAC: "bc:00:00:00:00:02", Link: true}, + {Name: "ens20", MAC: "bc:00:00:00:00:03", Link: true}, + } + l := &NetLayout{Interfaces: []LayoutIface{ + {Type: "physical", Select: "boot", Links: []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}}, + {Type: "bond", Name: "bond0", Members: []string{"fastest:2"}, BondMode: "active-backup", Links: []LayoutLink{{Mode: "link_up"}}}, + }} + p := resolvePlan(l, nics, &nics[0], []Subnet{labSubnet()}, nil, allocFrom("10.0.0.21")) + if got := strings.Join(p.Ifaces[1].Members, ","); got != "ens19,ens20" { + t.Errorf("bond members %q, want ens19,ens20", got) + } + if lk := p.Ifaces[1].Links[0]; lk.Subnet != "" || lk.Mode != "link_up" { + t.Errorf("a link_up link without a subnet stays without one: %+v", lk) + } +} + +func TestDnsmasqConfig(t *testing.T) { + rc := RackController{Name: "r1", TFTPRoot: "/srv/tftp", Interfaces: []RackIface{{Name: "eth0", Addrs: []string{"192.168.1.5/24"}}, {Name: "eth1", Addrs: []string{"10.0.0.1/24"}}}} + conf, ok := dnsmasqConfig(rc, []Subnet{labSubnet()}, "/var/lib/rollcall/dnsmasq") + if !ok { + t.Fatal("r1 serves 10.0.0.0/24") + } + for _, want := range []string{"interface=eth1\n", "dhcp-range=set:s0,10.0.0.100,10.0.0.200,255.255.255.0,12h", "dhcp-option=tag:s0,option:router,10.0.0.1", + "dhcp-option=tag:s0,option:dns-server,10.0.0.1", "domain=lab.local,10.0.0.0/24", "dhcp-authoritative", "tftp-root=/srv/tftp"} { + if !strings.Contains(conf, want) { + t.Errorf("dnsmasq conf lacks %q:\n%s", want, conf) + } + } + if strings.Contains(conf, "interface=eth0") { + t.Errorf("must not serve the management interface:\n%s", conf) + } + // two racks: each takes half of the pool, nobody is authoritative + sn := labSubnet() + sn.SecondaryRack = "r2" + c1, _ := dnsmasqConfig(rc, []Subnet{sn}, "/d") + c2, _ := dnsmasqConfig(RackController{Name: "r2", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"10.0.0.2/24"}}}}, []Subnet{sn}, "/d") + if !strings.Contains(c1, "10.0.0.100,10.0.0.150,") || !strings.Contains(c2, "10.0.0.151,10.0.0.200,") || strings.Contains(c1, "dhcp-authoritative") { + t.Errorf("split pool:\n%s\n%s", c1, c2) + } + if _, ok := dnsmasqConfig(RackController{Name: "other"}, []Subnet{labSubnet()}, "/d"); ok { + t.Errorf("a rack that serves nothing must not get a config") + } +} diff --git a/cmd/rollcall/rackctl.go b/cmd/rollcall/rackctl.go new file mode 100644 index 0000000..2b0e2b1 --- /dev/null +++ b/cmd/rollcall/rackctl.go @@ -0,0 +1,867 @@ +package main + +// Region side of the region/rack split, after MAAS. The region (rollcall +// serve) owns the database and computes everything; rack controllers sit next +// to the machines and do what has to happen on the machines' networks: DHCP +// and TFTP (dnsmasq), the per-MAC PXE files, BMC commands on a BMC network the +// region may not reach, and serving boot and OS images from a local cache. +// +// A rack controller pulls: it long-polls its desired state (PXE files, +// dnsmasq configuration, host files) and a queue of commands (ipmitool), so +// only rack → region connectivity is needed. The region host itself runs an +// embedded rack controller that applies the same state in-process, which is +// what a single-server installation uses. + +import ( + "bytes" + "context" + "crypto/subtle" + "encoding/json" + "errors" + "fmt" + "log" + "net" + "net/http" + "os" + "os/exec" + "path/filepath" + "sort" + "strconv" + "strings" + "sync" + "time" +) + +type RackIface struct { + Name string `json:"name"` + MAC string `json:"mac"` + Addrs []string `json:"addrs"` +} + +type RackController struct { + Name string `json:"name"` + URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1 + Kind string `json:"kind"` // embedded | remote + Version string `json:"version"` + Interfaces []RackIface `json:"interfaces"` + DHCPInterfaces []string `json:"dhcp_interfaces"` + TFTPRoot string `json:"tftp_root"` + LastSeen *time.Time `json:"last_seen"` + Registered time.Time `json:"registered"` + Online bool `json:"online"` + Machines int `json:"machines"` +} + +// rackState is everything a rack controller writes to disk. +type rackState struct { + Gen int64 `json:"gen"` + DefaultPXE string `json:"default_pxe"` + PXE map[string]string `json:"pxe"` // pxelinux.cfg file name (01-aa-bb-…) → content + DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here + HostsDHCP string `json:"hosts_dhcp"` + Hosts string `json:"hosts"` + BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP +} + +type rackJob struct { + ID string `json:"id"` + Kind string `json:"kind"` // ipmi + Address string `json:"address"` + Port int `json:"port"` + User string `json:"user"` + Password string `json:"password"` + Args []string `json:"args"` + done chan rackResult +} + +type rackResult struct { + ID string `json:"id"` + Output string `json:"output"` + Error string `json:"error"` +} + +// ---------------------------------------------------------------- store + +func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) { + rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`) + if err != nil { + return nil, err + } + defer rows.Close() + out := []RackController{} + for rows.Next() { + var r RackController + var ifs, dif []byte + if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil { + return nil, err + } + _ = json.Unmarshal(ifs, &r.Interfaces) + _ = json.Unmarshal(dif, &r.DHCPInterfaces) + out = append(out, r) + } + return out, rows.Err() +} + +func (s *Store) PutRackController(ctx context.Context, r RackController) error { + if r.Interfaces == nil { + r.Interfaces = []RackIface{} + } + if r.DHCPInterfaces == nil { + r.DHCPInterfaces = []string{} + } + ifs, _ := json.Marshal(r.Interfaces) + dif, _ := json.Marshal(r.DHCPInterfaces) + _, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen) VALUES ($1,$2,$3,$4,$5,$6,$7,now()) + ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now()`, + r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot) + return err +} + +func (s *Store) TouchRackController(ctx context.Context, name string) error { + _, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET last_seen=now() WHERE name=$1`, name) + return err +} + +func (s *Store) DeleteRackController(ctx context.Context, name string) error { + _, err := s.pool.Exec(ctx, `DELETE FROM rack_controllers WHERE name=$1`, name) + return err +} + +func (s *Store) SetRackController(ctx context.Context, id, rack string) error { + _, err := s.pool.Exec(ctx, `UPDATE machines SET rack_controller=nullif($2,'') WHERE id=$1`, id, rack) + return err +} + +func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error { + _, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4) + ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now()`, mac, machineID, rack, content) + return err +} + +func (s *Store) DeletePXE(ctx context.Context, mac string) error { + _, err := s.pool.Exec(ctx, `DELETE FROM pxe_files WHERE mac=$1`, mac) + return err +} + +func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, error) { + rows, err := s.pool.Query(ctx, `SELECT mac, content FROM pxe_files WHERE rack=$1`, rack) + if err != nil { + return nil, err + } + defer rows.Close() + out := map[string]string{} + for rows.Next() { + var mac, content string + if err := rows.Scan(&mac, &content); err != nil { + return nil, err + } + out[pxeFileName(mac)] = content + } + return out, rows.Err() +} + +func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") } + +// ---------------------------------------------------------------- hub: state generations and the command queue + +type rackHub struct { + mu sync.Mutex + gen int64 + changed chan struct{} + work map[string][]*rackJob + wake map[string]chan struct{} + flight map[string]*rackJob + kick chan struct{} // the embedded rack applies on this +} + +func newRackHub() *rackHub { + return &rackHub{gen: time.Now().UnixNano(), changed: make(chan struct{}), work: map[string][]*rackJob{}, + wake: map[string]chan struct{}{}, flight: map[string]*rackJob{}, kick: make(chan struct{}, 1)} +} + +func (h *rackHub) current() int64 { + h.mu.Lock() + defer h.mu.Unlock() + return h.gen +} + +// bump says the desired state of some rack changed. +func (h *rackHub) bump() { + h.mu.Lock() + h.gen++ + close(h.changed) + h.changed = make(chan struct{}) + h.mu.Unlock() + select { + case h.kick <- struct{}{}: + default: + } +} + +// waitChange blocks until the generation moves past since, or the wait is over. +func (h *rackHub) waitChange(ctx context.Context, since int64, wait time.Duration) int64 { + t := time.NewTimer(wait) + defer t.Stop() + for { + h.mu.Lock() + g, ch := h.gen, h.changed + h.mu.Unlock() + if g != since { + return g + } + select { + case <-ch: + case <-t.C: + return g + case <-ctx.Done(): + return g + } + } +} + +func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string, error) { + j.ID = randomToken(8) + j.done = make(chan rackResult, 1) + h.mu.Lock() + h.work[rack] = append(h.work[rack], j) + if w, ok := h.wake[rack]; ok { + close(w) + delete(h.wake, rack) + } + h.mu.Unlock() + t := time.NewTimer(90 * time.Second) + defer t.Stop() + select { + case r := <-j.done: + if r.Error != "" { + return r.Output, errors.New(r.Error) + } + return r.Output, nil + case <-t.C: + case <-ctx.Done(): + } + h.mu.Lock() + delete(h.flight, j.ID) + q := h.work[rack][:0] + for _, x := range h.work[rack] { + if x != j { + q = append(q, x) + } + } + h.work[rack] = q + h.mu.Unlock() + return "", fmt.Errorf("rack controller %s did not answer in time", rack) +} + +func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*rackJob { + deadline := time.Now().Add(wait) + for { + h.mu.Lock() + if q := h.work[rack]; len(q) > 0 { + h.work[rack] = nil + for _, j := range q { + h.flight[j.ID] = j + } + h.mu.Unlock() + return q + } + w, ok := h.wake[rack] + if !ok { + w = make(chan struct{}) + h.wake[rack] = w + } + h.mu.Unlock() + left := time.Until(deadline) + if left <= 0 { + return nil + } + t := time.NewTimer(left) + select { + case <-w: + case <-t.C: + case <-ctx.Done(): + t.Stop() + return nil + } + t.Stop() + } +} + +func (h *rackHub) finish(r rackResult) bool { + h.mu.Lock() + j, ok := h.flight[r.ID] + delete(h.flight, r.ID) + h.mu.Unlock() + if ok { + j.done <- r + } + return ok +} + +// ---------------------------------------------------------------- routing machines to racks + +func (a *App) racksSnapshot() map[string]RackController { + a.netMu.RLock() + defer a.netMu.RUnlock() + out := make(map[string]RackController, len(a.racks)) + for k, v := range a.racks { + out[k] = v + } + return out +} + +func (a *App) subnetsSnapshot() []Subnet { + a.netMu.RLock() + defer a.netMu.RUnlock() + return a.subnets +} + +// reloadNet refreshes the cached subnets and rack controllers after a change. +func (a *App) reloadNet(ctx context.Context) { + if sn, err := a.store.Subnets(ctx); err == nil { + a.netMu.Lock() + a.subnets = sn + a.netMu.Unlock() + } + a.reloadRacks(ctx) +} + +func (a *App) reloadRacks(ctx context.Context) { + rs, err := a.store.RackControllers(ctx) + if err != nil { + log.Printf("racks: %v", err) + return + } + m := map[string]RackController{} + for _, r := range rs { + m[r.Name] = r + } + a.netMu.Lock() + a.racks = m + a.netMu.Unlock() +} + +func rackOnline(r RackController, local string) bool { + return r.Name == local || (r.LastSeen != nil && time.Since(*r.LastSeen) < 90*time.Second) +} + +// rackOf names the rack controller serving a machine: the one it last came +// through, else the one serving the VLAN of its management or boot address, +// else the embedded one. +func (a *App) rackOf(m *Machine) string { + racks := a.racksSnapshot() + if m.RackController != "" { + if _, ok := racks[m.RackController]; ok { + return m.RackController + } + } + if subnets := a.subnetsSnapshot(); len(subnets) > 0 { + ips := []string{deref(m.MgmtIP)} + if b := bootNIC(machineNICs(m), m.MACs); b != nil { + ips = append(ips, b.IPv4...) + } + for _, ip := range ips { + if sn := subnetFor(subnets, ip); sn != nil && sn.PrimaryRack != "" { + if _, ok := racks[sn.PrimaryRack]; ok { + return sn.PrimaryRack + } + } + } + } + return a.cfg.RackName +} + +func (a *App) rackURL(name string) string { + if name == "" || name == a.cfg.RackName { + return a.cfg.ServerURL + } + if r, ok := a.racksSnapshot()[name]; ok && r.URL != "" { + return strings.TrimRight(r.URL, "/") + } + return a.cfg.ServerURL +} + +// urlFor is the base URL a machine uses for images and the node API. +func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) } + +// appendFor is commission_append with the server URL of the machine's rack. +func (a *App) appendFor(base string) string { + if base == a.cfg.ServerURL { + return a.cfg.CommissionAppend + } + return strings.ReplaceAll(a.cfg.CommissionAppend, a.cfg.ServerURL, base) +} + +// rackAddresses are the rack controllers' own addresses, never handed to machines. +func (a *App) rackAddresses() []string { + var out []string + for _, r := range a.racksSnapshot() { + for _, i := range r.Interfaces { + out = append(out, i.Addrs...) + } + } + return out +} + +// ---------------------------------------------------------------- BMC through the machine's rack + +var errNoBMC = errors.New("no BMC credentials stored") + +func ipmiExec(ctx context.Context, tool string, c *bmcCreds, args ...string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + if tool == "" { + tool = "ipmitool" + } + full := append([]string{"-I", "lanplus", "-H", c.Address, "-p", strconv.Itoa(c.Port), "-U", c.User, "-E"}, args...) + cmd := exec.CommandContext(ctx, tool, full...) + cmd.Env = []string{"IPMI_PASSWORD=" + c.Password, "PATH=/usr/sbin:/usr/bin:/sbin:/bin"} + var out bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &out + err := cmd.Run() + return strings.TrimSpace(out.String()), err +} + +// bmc runs an ipmitool command for a machine, on the rack controller that can reach its BMC. +func (a *App) bmc(ctx context.Context, m *Machine, args ...string) (string, error) { + c, err := a.credsFor(m) + if err != nil { + return "", fmt.Errorf("%w: %v", errNoBMC, err) + } + rack := a.rackOf(m) + if rack == a.cfg.RackName { + return ipmiExec(ctx, a.cfg.IPMITool, c, args...) + } + r, ok := a.racksSnapshot()[rack] + if !ok || !rackOnline(r, a.cfg.RackName) { + return "", fmt.Errorf("rack controller %s is offline", rack) + } + return a.hub.dispatch(ctx, rack, &rackJob{Kind: "ipmi", Address: c.Address, Port: c.Port, User: c.User, Password: c.Password, Args: args}) +} + +// ---------------------------------------------------------------- desired state + +const defaultPXETmpl = `# generated by rollcall for rack controller %s +# unknown machines land here; known ones get a per-MAC file (01-xx-xx-…) +DEFAULT menu.c32 +PROMPT 0 +TIMEOUT 80 +MENU TITLE rollcall · %s · unknown machine: commissioning in 8s + +LABEL commission + MENU LABEL Commission this machine (inventory + BMC account) + MENU DEFAULT + KERNEL rollcall/vmlinuz + INITRD rollcall/initrd.img + APPEND %s + +LABEL commission-debug + MENU LABEL Commission, but stay up afterwards (ssh root@) + KERNEL rollcall/vmlinuz + INITRD rollcall/initrd.img + APPEND %s rollcall.nopoweroff + +LABEL local + MENU LABEL Boot from local disk + LOCALBOOT 0 +` + +func (a *App) rackState(ctx context.Context, name string) (*rackState, error) { + gen := a.hub.current() + st := &rackState{Gen: gen} + base := a.rackURL(name) + app := a.appendFor(base) + st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app) + var err error + if st.PXE, err = a.store.PXEFiles(ctx, name); err != nil { + return nil, err + } + subnets, err := a.store.Subnets(ctx) + if err != nil { + return nil, err + } + rc := a.racksSnapshot()[name] + rc.Name = name + dnsDir := a.cfg.DnsmasqDir + if dnsDir == "" { + dnsDir = "/var/lib/rollcall/dnsmasq" + } + if conf, ok := dnsmasqConfig(rc, subnets, dnsDir); ok { + st.DnsmasqConf = conf + } + if st.HostsDHCP, st.Hosts, err = a.hostsFiles(ctx, name, subnets); err != nil { + return nil, err + } + st.BootFiles = []string{"rollcall/vmlinuz", "rollcall/initrd.img"} + if ents, err := os.ReadDir(a.cfg.ImagesDir); err == nil { + for _, e := range ents { + if e.IsDir() { + st.BootFiles = append(st.BootFiles, "images/"+e.Name()+"/vmlinuz", "images/"+e.Name()+"/initrd.img") + } + } + } + return st, nil +} + +// hostsFiles: static DHCP leases for the machines this rack serves, and names for every machine. +func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) { + ms, err := a.store.List(ctx) + if err != nil { + return "", "", err + } + var dhcp, hosts strings.Builder + dhcp.WriteString("# generated by rollcall\n") + hosts.WriteString("# generated by rollcall\n") + domainOf := func(ip string) string { + if sn := subnetFor(subnets, ip); sn != nil && sn.Domain != "" { + return sn.Domain + } + return str(a.cfg.ClusterVars["domain"]) + } + sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) }) + for _, m := range ms { + if m.NodeName == nil || len(m.MACs) == 0 { + continue + } + name := *m.NodeName + if ip := deref(m.MgmtIP); ip != "" { + if a.rackOf(m) == rack { + fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], ip, name) + } + if d := domainOf(ip); d != "" { + fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name) + } else { + fmt.Fprintf(&hosts, "%s %s\n", ip, name) + } + } + if ip := deref(m.IBIP); ip != "" { + fmt.Fprintf(&hosts, "%s %s-ib\n", ip, name) + } + } + return dhcp.String(), hosts.String(), nil +} + +// writeIfChanged replaces a file only when its content differs (path units watch these). +func writeIfChanged(p, content string, mode os.FileMode) (bool, error) { + if old, err := os.ReadFile(p); err == nil && string(old) == content { + return false, nil + } + if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { + return false, err + } + if err := os.WriteFile(p+".tmp", []byte(content), mode); err != nil { + return false, err + } + return true, os.Rename(p+".tmp", p) +} + +// applyRackState writes a rack's state: PXE files under tftp/pxelinux.cfg +// (stale per-MAC files removed), the dnsmasq configuration and host files. +func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err error) { + dir := filepath.Join(tftp, "pxelinux.cfg") + note := func(p string, ch bool, e error) { + if e != nil && err == nil { + err = e + } + if ch { + changed = append(changed, p) + } + } + if st.DefaultPXE != "" { + ch, e := writeIfChanged(filepath.Join(dir, "default"), st.DefaultPXE, 0o644) + note("default", ch, e) + } + for name, body := range st.PXE { + ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644) + note(name, ch, e) + } + if olds, e := filepath.Glob(filepath.Join(dir, "01-*")); e == nil { + for _, p := range olds { + if _, keep := st.PXE[filepath.Base(p)]; !keep && !strings.HasSuffix(p, ".tmp") { + note("-"+filepath.Base(p), true, os.Remove(p)) + } + } + } + if dnsDir != "" { + ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644) + note("hosts.dhcp", ch, e) + ch, e = writeIfChanged(filepath.Join(dnsDir, "hosts"), st.Hosts, 0o644) + note("hosts", ch, e) + conf := filepath.Join(dnsDir, "conf.d", "rollcall.conf") + if st.DnsmasqConf != "" { + ch, e = writeIfChanged(conf, st.DnsmasqConf, 0o644) + note("dnsmasq.conf", ch, e) + } else if _, e := os.Stat(conf); e == nil { + note("-dnsmasq.conf", true, os.Remove(conf)) + } + } + return changed, err +} + +// localRackLoop is the embedded rack controller: it applies this host's state on every change. +func (a *App) localRackLoop() { + apply := func() { + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + st, err := a.rackState(ctx, a.cfg.RackName) + if err != nil { + log.Printf("embedded rack: %v", err) + return + } + if ch, err := applyRackState(st, a.cfg.TFTPRoot, a.cfg.DnsmasqDir); err != nil { + log.Printf("embedded rack: %v", err) + } else if len(ch) > 0 { + log.Printf("embedded rack %s: wrote %s", a.cfg.RackName, strings.Join(ch, " ")) + } + _ = a.store.TouchRackController(ctx, a.cfg.RackName) + } + apply() + tick := time.NewTicker(60 * time.Second) + for { + select { + case <-a.hub.kick: + time.Sleep(150 * time.Millisecond) // let a burst of changes settle + select { + case <-a.hub.kick: + default: + } + case <-tick.C: + } + apply() + } +} + +func localInterfaces() []RackIface { + var out []RackIface + ifs, _ := net.Interfaces() + for _, i := range ifs { + if i.Flags&net.FlagLoopback != 0 { + continue + } + ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()} + addrs, _ := i.Addrs() + for _, ad := range addrs { + if ipn, ok := ad.(*net.IPNet); ok && ipn.IP.To4() != nil { + ri.Addrs = append(ri.Addrs, ipn.String()) + } + } + if len(ri.Addrs) > 0 { + out = append(out, ri) + } + } + return out +} + +// startRacks registers the embedded rack controller, carries over what an +// older version kept on disk or in config, and starts the local apply loop. +func (a *App) startRacks(ctx context.Context) error { + if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, + Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil { + return err + } + a.reloadNet(ctx) + // per-MAC files written before pxe_files existed + var n int + _ = a.store.pool.QueryRow(ctx, `SELECT count(*) FROM pxe_files`).Scan(&n) + if n == 0 { + files, _ := filepath.Glob(filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-*")) + ms, _ := a.store.List(ctx) + for _, f := range files { + b, err := os.ReadFile(f) + if err != nil { + continue + } + mac := strings.ReplaceAll(strings.TrimPrefix(filepath.Base(f), "01-"), "-", ":") + owner := "" + for _, m := range ms { + if hasTag(m.MACs, mac) { + owner = m.ID + } + } + _ = a.store.PutPXE(ctx, mac, owner, a.cfg.RackName, string(b)) + } + if len(files) > 0 { + log.Printf("imported %d existing PXE files", len(files)) + } + } + a.adoptIPs(ctx) + go a.localRackLoop() + return nil +} + +// adoptIPs records addresses machines already have (node pools, before a +// subnet was modeled) so IPAM never hands them out twice. +func (a *App) adoptIPs(ctx context.Context) { + subnets, err := a.store.Subnets(ctx) + if err != nil || len(subnets) == 0 { + return + } + ms, _ := a.store.List(ctx) + for _, m := range ms { + for _, ip := range []string{deref(m.MgmtIP), deref(m.IBIP)} { + if sn := subnetFor(subnets, ip); sn != nil { + iface := "" + if b := bootNIC(machineNICs(m), m.MACs); b != nil && ip == deref(m.MgmtIP) { + iface = b.Name + } + _, _ = a.store.ClaimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: m.ID, Iface: iface, Kind: "auto", Hostname: deref(m.NodeName)}) + } + } + } +} + +// ---------------------------------------------------------------- PXE files through the rack + +// writePXEBody records a machine's PXE file for its rack controller. +func (a *App) writePXEBody(m *Machine, body string) error { + rack := a.rackOf(m) + for _, mac := range m.MACs { + if err := a.store.PutPXE(context.Background(), normMAC(mac), m.ID, rack, body); err != nil { + return err + } + } + a.hub.bump() + return nil +} + +func (a *App) removePXE(m *Machine) { + for _, mac := range m.MACs { + _ = a.store.DeletePXE(context.Background(), normMAC(mac)) + } + a.hub.bump() +} + +// exportDnsmasq: host files and dnsmasq configuration follow from the state now. +func (a *App) exportDnsmasq(ctx context.Context) error { + a.hub.bump() + return nil +} + +// ---------------------------------------------------------------- HTTP: rack controllers + +func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack") + if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" { + fail(w, 401, errors.New("rack controller secret required")) + return + } + next(w, r, name) + } +} + +// fromRack names the rack controller a node request came through (it proxies them). +func (a *App) fromRack(r *http.Request) string { + sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack") + if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 { + return name + } + return a.cfg.RackName +} + +func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack string) { + var rc RackController + if err := json.NewDecoder(r.Body).Decode(&rc); err != nil { + fail(w, 400, err) + return + } + rc.Name, rc.Kind = rack, "remote" + if rack == a.cfg.RackName { + fail(w, 409, fmt.Errorf("%s is the embedded rack controller's name", rack)) + return + } + if rc.URL == "" { + fail(w, 400, errors.New("url is required")) + return + } + if err := a.store.PutRackController(r.Context(), rc); err != nil { + fail(w, 500, err) + return + } + a.reloadRacks(r.Context()) + a.hub.bump() + log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces)) + writeJSON(w, 200, obj{"ok": true, "gen": a.hub.current()}) +} + +func (a *App) handleRackState(w http.ResponseWriter, r *http.Request, rack string) { + if _, ok := a.racksSnapshot()[rack]; !ok { + fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack)) + return + } + since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64) + _ = a.store.TouchRackController(r.Context(), rack) + a.hub.waitChange(r.Context(), since, 25*time.Second) + _ = a.store.TouchRackController(r.Context(), rack) + go a.reloadRacks(context.Background()) // keep last_seen fresh for online checks + st, err := a.rackState(r.Context(), rack) + if err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, st) +} + +func (a *App) handleRackWork(w http.ResponseWriter, r *http.Request, rack string) { + jobs := a.hub.take(r.Context(), rack, 25*time.Second) + if jobs == nil { + jobs = []*rackJob{} + } + writeJSON(w, 200, jobs) +} + +func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack string) { + var res rackResult + if err := json.NewDecoder(r.Body).Decode(&res); err != nil { + fail(w, 400, err) + return + } + writeJSON(w, 200, obj{"accepted": a.hub.finish(res)}) +} + +func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) { + rs, err := a.store.RackControllers(r.Context()) + if err != nil { + fail(w, 500, err) + return + } + ms, _ := a.store.List(r.Context()) + for i := range rs { + rs[i].Online = rackOnline(rs[i], a.cfg.RackName) + for _, m := range ms { + if a.rackOf(m) == rs[i].Name { + rs[i].Machines++ + } + } + } + writeJSON(w, 200, rs) +} + +func (a *App) handleControllerDelete(w http.ResponseWriter, r *http.Request) { + name := r.PathValue("name") + if name == a.cfg.RackName { + fail(w, 409, errors.New("the embedded rack controller cannot be removed")) + return + } + if err := a.store.DeleteRackController(r.Context(), name); err != nil { + fail(w, 500, err) + return + } + a.reloadRacks(r.Context()) + a.hub.bump() + writeJSON(w, 200, obj{"deleted": name}) +} + +// handleRackDHCP shows a rack's generated dnsmasq configuration (admin). +func (a *App) handleRackDHCP(w http.ResponseWriter, r *http.Request) { + st, err := a.rackState(r.Context(), r.PathValue("name")) + if err != nil { + fail(w, 500, err) + return + } + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + if st.DnsmasqConf == "" { + fmt.Fprintf(w, "# rack controller %s serves no VLAN with dhcp on: rollcall leaves dnsmasq's configuration alone\n", r.PathValue("name")) + } + fmt.Fprint(w, st.DnsmasqConf, "\n# --- hosts.dhcp\n", st.HostsDHCP) +} diff --git a/cmd/rollcall/rackd.go b/cmd/rollcall/rackd.go new file mode 100644 index 0000000..21a2bdb --- /dev/null +++ b/cmd/rollcall/rackd.go @@ -0,0 +1,414 @@ +package main + +// `rollcall rack`: a rack controller next to the machines. It registers with +// the region, long-polls its desired state and writes it (PXE files, dnsmasq +// configuration, host files), runs BMC commands the region queues for it, +// serves boot and OS images from a local cache filled from the region on +// first use, and proxies the node API (commissioning, deploy jobs, overlays, +// phone-home) to the region, marking requests with its name. + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log" + "net/http" + "net/http/httputil" + "net/url" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "sync" + "time" +) + +type RackConfig struct { + Name string `json:"name"` // default: hostname + Region string `json:"region"` // the region's URL, e.g. http://10.0.0.234 + Secret string `json:"secret"` // the region's rack_secret + URL string `json:"url"` // how machines reach this rack, e.g. http://10.213.78.1 + Listen string `json:"listen"` // default ":80" + TFTPRoot string `json:"tftp_root"` // default /srv/tftp + DnsmasqDir string `json:"dnsmasq_dir"` // default /var/lib/rollcall/dnsmasq + CacheDir string `json:"cache_dir"` // default /var/cache/rollcall + IPMITool string `json:"ipmitool"` // default ipmitool + DHCPInterfaces []string `json:"dhcp_interfaces"` // default: the interfaces with an address in a served subnet +} + +type rackd struct { + cfg RackConfig + region *url.URL + hc *http.Client // long polls + fast *http.Client + gen int64 + + headMu sync.Mutex + heads map[string]headInfo + fetch sync.Mutex // one boot-file prefetch at a time +} + +type headInfo struct { + size int64 + mod time.Time + at time.Time +} + +func loadRackConfig(p string) (RackConfig, error) { + var c RackConfig + b, err := os.ReadFile(p) + if err != nil { + return c, err + } + if err := json.Unmarshal(b, &c); err != nil { + return c, fmt.Errorf("%s: %w", p, err) + } + if c.Name == "" { + c.Name, _ = os.Hostname() + } + if c.Listen == "" { + c.Listen = ":80" + } + if c.TFTPRoot == "" { + c.TFTPRoot = "/srv/tftp" + } + if c.DnsmasqDir == "" { + c.DnsmasqDir = "/var/lib/rollcall/dnsmasq" + } + if c.CacheDir == "" { + c.CacheDir = "/var/cache/rollcall" + } + if c.IPMITool == "" { + c.IPMITool = "ipmitool" + } + c.Region, c.URL = strings.TrimRight(c.Region, "/"), strings.TrimRight(c.URL, "/") + for _, k := range []struct{ v, n string }{{c.Region, "region"}, {c.Secret, "secret"}, {c.URL, "url"}} { + if k.v == "" { + return c, fmt.Errorf("%s: %s is required", p, k.n) + } + } + return c, nil +} + +func runRack(args []string) error { + fs := newFlags("rack") + cfgPath := fs.String("config", "/etc/rollcall/rack.json", "rack controller configuration") + _ = fs.Parse(args) + cfg, err := loadRackConfig(*cfgPath) + if err != nil { + return err + } + ru, err := url.Parse(cfg.Region) + if err != nil { + return fmt.Errorf("region: %w", err) + } + r := &rackd{cfg: cfg, region: ru, hc: &http.Client{Timeout: 45 * time.Second}, fast: &http.Client{Timeout: 30 * time.Second}, heads: map[string]headInfo{}} + for _, d := range []string{filepath.Join(cfg.TFTPRoot, "pxelinux.cfg"), cfg.CacheDir, filepath.Join(cfg.DnsmasqDir, "conf.d")} { + if err := os.MkdirAll(d, 0o755); err != nil { + return err + } + } + go r.loop() + mux := http.NewServeMux() + for _, p := range []string{"/rollcall/", "/images/", "/os/"} { + mux.HandleFunc("GET "+p, r.serveCached) + mux.HandleFunc("HEAD "+p, r.serveCached) + } + proxy := &httputil.ReverseProxy{ + Rewrite: func(pr *httputil.ProxyRequest) { + pr.SetURL(ru) + pr.SetXForwarded() + pr.Out.Header.Set("X-Rollcall-Rack", cfg.Name) + pr.Out.Header.Set("X-Rollcall-Rack-Secret", cfg.Secret) + }, + FlushInterval: 100 * time.Millisecond, + } + for _, p := range []string{"/api/v1/commission", "/api/v1/bmc-hint", "/api/v1/jobs/self", "/api/v1/jobs/self/progress", + "/api/v1/nodes/self/overlay", "/api/v1/phone-home/"} { + mux.Handle(p, proxy) + } + mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) { + writeJSON(w, 200, obj{"rack": cfg.Name, "gen": r.gen, "version": version}) + }) + srv := &http.Server{Addr: cfg.Listen, Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second} + log.Printf("rack controller %s: region %s, serving %s on %s (tftp %s, cache %s)", cfg.Name, cfg.Region, cfg.URL, cfg.Listen, cfg.TFTPRoot, cfg.CacheDir) + return srv.ListenAndServe() +} + +func (r *rackd) req(ctx context.Context, method, p string, body any) (*http.Request, error) { + var rd io.Reader + if body != nil { + b, _ := json.Marshal(body) + rd = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, r.cfg.Region+p, rd) + if err != nil { + return nil, err + } + req.Header.Set("X-Rollcall-Rack", r.cfg.Name) + req.Header.Set("X-Rollcall-Rack-Secret", r.cfg.Secret) + req.Header.Set("Content-Type", "application/json") + return req, nil +} + +func (r *rackd) call(c *http.Client, method, p string, body, out any) (int, error) { + req, err := r.req(context.Background(), method, p, body) + if err != nil { + return 0, err + } + resp, err := c.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + var e apiError + _ = json.Unmarshal(data, &e) + if e.Error == "" { + e.Error = strings.TrimSpace(string(data)) + } + return resp.StatusCode, fmt.Errorf("%s %s: %s", method, p, e.Error) + } + if out != nil { + return resp.StatusCode, json.Unmarshal(data, out) + } + return resp.StatusCode, nil +} + +func (r *rackd) register() error { + _, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version, + Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil) + return err +} + +func (r *rackd) loop() { + backoff := time.Second + for { + if err := r.register(); err != nil { + log.Printf("register: %v (retry in %s)", err, backoff) + time.Sleep(backoff) + backoff = min(backoff*2, 30*time.Second) + continue + } + break + } + log.Printf("registered with %s as %s", r.cfg.Region, r.cfg.Name) + go r.workLoop() + go func() { // interfaces change; registration also refreshes last_seen + for range time.Tick(5 * time.Minute) { + if err := r.register(); err != nil { + log.Printf("re-register: %v", err) + } + } + }() + for { + var st rackState + code, err := r.call(r.hc, "GET", "/api/v1/rack/state?since="+strconv.FormatInt(r.gen, 10), nil, &st) + if err != nil { + log.Printf("state: %v", err) + if code == 409 || code == 401 { + _ = r.register() + } + time.Sleep(3 * time.Second) + continue + } + if st.Gen == r.gen { + continue + } + ch, err := applyRackState(&st, r.cfg.TFTPRoot, r.cfg.DnsmasqDir) + if err != nil { + log.Printf("apply: %v", err) + } else if len(ch) > 0 { + log.Printf("state %d: wrote %s", st.Gen, strings.Join(ch, " ")) + } + r.gen = st.Gen + go r.prefetch(st.BootFiles) + } +} + +func (r *rackd) workLoop() { + for { + var jobs []rackJob + if _, err := r.call(r.hc, "GET", "/api/v1/rack/work", nil, &jobs); err != nil { + log.Printf("work: %v", err) + time.Sleep(3 * time.Second) + continue + } + for _, j := range jobs { + go func(j rackJob) { + res := rackResult{ID: j.ID} + switch j.Kind { + case "ipmi": + out, err := ipmiExec(context.Background(), r.cfg.IPMITool, &bmcCreds{Address: j.Address, Port: j.Port, User: j.User, Password: j.Password}, j.Args...) + res.Output = out + if err != nil { + res.Error = err.Error() + if out != "" { + res.Error = out + } + } + default: + res.Error = "unknown job kind " + j.Kind + } + log.Printf("job %s %s %s: %s", j.ID, j.Kind, strings.Join(j.Args, " "), oneLine(res.Output+" "+res.Error)) + if _, err := r.call(r.fast, "POST", "/api/v1/rack/result", res, nil); err != nil { + log.Printf("result %s: %v", j.ID, err) + } + }(j) + } + } +} + +// ---------------------------------------------------------------- image cache + +// head asks the region for a file's size and modification time (cached briefly). +func (r *rackd) head(rel string) (headInfo, error) { + r.headMu.Lock() + if h, ok := r.heads[rel]; ok && time.Since(h.at) < 30*time.Second { + r.headMu.Unlock() + return h, nil + } + r.headMu.Unlock() + req, _ := http.NewRequest("HEAD", r.cfg.Region+rel, nil) + resp, err := r.fast.Do(req) + if err != nil { + return headInfo{}, err + } + resp.Body.Close() + if resp.StatusCode != 200 { + return headInfo{}, fmt.Errorf("%s: %s", rel, resp.Status) + } + mod, _ := http.ParseTime(resp.Header.Get("Last-Modified")) + h := headInfo{size: resp.ContentLength, mod: mod, at: time.Now()} + r.headMu.Lock() + r.heads[rel] = h + r.headMu.Unlock() + return h, nil +} + +// fresh says whether the cached copy matches the region's file. OS images live +// under versioned paths and never change; the rest is checked by size and time. +func (r *rackd) fresh(rel string, fi os.FileInfo) bool { + if strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/") { + return true + } + h, err := r.head(rel) + if err != nil { + return true // region unreachable: serve what we have + } + return h.size == fi.Size() && h.mod.Unix() == fi.ModTime().Unix() +} + +func (r *rackd) localPath(rel string) (string, bool) { + clean := path.Clean("/" + rel) + if clean == "/" || strings.Contains(clean, "..") { + return "", false + } + return filepath.Join(r.cfg.CacheDir, filepath.FromSlash(clean)), true +} + +// fill downloads rel from the region into the cache, copying to w on the way when given. +func (r *rackd) fill(rel string, w http.ResponseWriter) error { + local, ok := r.localPath(rel) + if !ok { + return errors.New("bad path") + } + resp, err := http.Get(r.cfg.Region + rel) // no timeout: images are large + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + if w != nil { + w.WriteHeader(resp.StatusCode) + } + return fmt.Errorf("%s: %s", rel, resp.Status) + } + if err := os.MkdirAll(filepath.Dir(local), 0o755); err != nil { + return err + } + tmp, err := os.CreateTemp(filepath.Dir(local), ".fill-*") + if err != nil { + return err + } + defer os.Remove(tmp.Name()) + var dst io.Writer = tmp + if w != nil { + for _, h := range []string{"Content-Length", "Content-Type", "Last-Modified"} { + if v := resp.Header.Get(h); v != "" { + w.Header().Set(h, v) + } + } + w.WriteHeader(200) + dst = io.MultiWriter(tmp, w) + } + n, err := io.Copy(dst, resp.Body) + if cerr := tmp.Close(); err == nil { + err = cerr + } + if err != nil { + return err + } + if resp.ContentLength >= 0 && n != resp.ContentLength { + return fmt.Errorf("%s: short read %d/%d", rel, n, resp.ContentLength) + } + if mod, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil { + _ = os.Chtimes(tmp.Name(), mod, mod) + } + _ = os.Chmod(tmp.Name(), 0o644) + if err := os.Rename(tmp.Name(), local); err != nil { + return err + } + log.Printf("cached %s (%s)", rel, human(float64(n))) + return nil +} + +func (r *rackd) serveCached(w http.ResponseWriter, req *http.Request) { + rel := path.Clean(req.URL.Path) + local, ok := r.localPath(rel) + if !ok { + http.NotFound(w, req) + return + } + if fi, err := os.Stat(local); err == nil && !fi.IsDir() && r.fresh(rel, fi) { + http.ServeFile(w, req, local) + return + } + if req.Method == http.MethodHead { + h, err := r.head(rel) + if err != nil { + http.NotFound(w, req) + return + } + w.Header().Set("Content-Length", strconv.FormatInt(h.size, 10)) + w.Header().Set("Last-Modified", h.mod.UTC().Format(http.TimeFormat)) + return + } + if err := r.fill(rel, w); err != nil { + log.Printf("cache %s: %v", rel, err) + } +} + +// prefetch keeps kernels and initrds cached: TFTP cannot fetch on demand. +func (r *rackd) prefetch(files []string) { + r.fetch.Lock() + defer r.fetch.Unlock() + for _, f := range files { + rel := "/" + strings.TrimPrefix(f, "/") + local, ok := r.localPath(rel) + if !ok { + continue + } + if fi, err := os.Stat(local); err == nil && r.fresh(rel, fi) { + continue + } + if err := r.fill(rel, nil); err != nil { + log.Printf("prefetch %s: %v", rel, err) + } + } +} diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index 28ebc19..c8c0b31 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -11,10 +11,9 @@ import ( "io/fs" "log" "net/http" - "os" - "path/filepath" "regexp" "strings" + "sync" "time" ) @@ -25,6 +24,11 @@ type App struct { cfg *Config store *Store sealer *Sealer + hub *rackHub + + netMu sync.RWMutex + racks map[string]RackController + subnets []Subnet } type apiError struct { @@ -146,6 +150,33 @@ func (a *App) routes() http.Handler { } writeJSON(w, 200, jobs) })) + // network model and rack controllers + mux.HandleFunc("GET /api/v1/fabrics", a.adminAuth(a.handleFabrics)) + mux.HandleFunc("POST /api/v1/fabrics", a.adminAuth(a.handleFabrics)) + mux.HandleFunc("DELETE /api/v1/fabrics/{name}", a.adminAuth(a.handleFabricDelete)) + mux.HandleFunc("GET /api/v1/vlans", a.adminAuth(a.handleVLANs)) + mux.HandleFunc("POST /api/v1/vlans", a.adminAuth(a.handleVLANs)) + mux.HandleFunc("DELETE /api/v1/vlans/{fabric}/{vid}", a.adminAuth(a.handleVLANDelete)) + mux.HandleFunc("GET /api/v1/subnets", a.adminAuth(a.handleSubnets)) + mux.HandleFunc("POST /api/v1/subnets", a.adminAuth(a.handleSubnets)) + mux.HandleFunc("DELETE /api/v1/subnets/{cidr...}", a.adminAuth(a.handleSubnetDelete)) + mux.HandleFunc("POST /api/v1/ranges", a.adminAuth(a.handleRanges)) + mux.HandleFunc("DELETE /api/v1/ranges/{id}", a.adminAuth(a.handleRangeDelete)) + mux.HandleFunc("GET /api/v1/ips", a.adminAuth(a.handleIPs)) + mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs)) + mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete)) + mux.HandleFunc("GET /api/v1/layouts", a.adminAuth(a.handleLayouts)) + mux.HandleFunc("POST /api/v1/layouts", a.adminAuth(a.handleLayouts)) + mux.HandleFunc("DELETE /api/v1/layouts/{name}", a.adminAuth(a.handleLayoutDelete)) + mux.HandleFunc("GET /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork)) + mux.HandleFunc("POST /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork)) + mux.HandleFunc("GET /api/v1/controllers", a.adminAuth(a.handleControllers)) + mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete)) + mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP)) + mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister)) + mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState)) + mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork)) + mux.HandleFunc("POST /api/v1/rack/result", a.rackAuth(a.handleRackResult)) // physical location mux.HandleFunc("POST /api/v1/machines/{id}/location", a.adminAuth(a.handleLocation)) mux.HandleFunc("POST /api/v1/locations/import", a.adminAuth(a.handleLocationImport)) @@ -271,6 +302,8 @@ type Summary struct { ManualTags []string `json:"manual_tags"` AutoTags []string `json:"auto_tags"` Location Location `json:"location"` + Rack string `json:"rack_controller"` + NetLayout string `json:"net_layout"` } func num(v any) float64 { @@ -289,7 +322,8 @@ func summarize(m *Machine) Summary { IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: nodeCtx(m).GPUs, // real accelerators only Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt, OS: m.OS, OSVersion: m.OSVersion, DeployedAt: m.DeployedAt, - Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location} + Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location, + Rack: m.RackController, NetLayout: m.NetLayout} if m.BMC != nil { var b obj if json.Unmarshal(m.BMC, &b) == nil { @@ -325,10 +359,6 @@ LABEL local LOCALBOOT 0 ` -func (a *App) pxePath(mac string) string { - return filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-"+strings.ReplaceAll(normMAC(mac), ":", "-")) -} - func deref(p *string) string { if p == nil { return "" @@ -339,32 +369,10 @@ func deref(p *string) string { func (a *App) writePXE(m *Machine) error { man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial) body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser, - m.ID, man, prod, ser, a.cfg.CommissionAppend) + m.ID, man, prod, ser, a.appendFor(a.urlFor(m))) return a.writePXEBody(m, body) } -func (a *App) writePXEBody(m *Machine, body string) error { - for _, mac := range m.MACs { - p := a.pxePath(mac) - if err := os.WriteFile(p+".tmp", []byte(body), 0o644); err != nil { - return err - } - if err := os.Rename(p+".tmp", p); err != nil { - return err - } - log.Printf("wrote %s", p) - } - return nil -} - -func (a *App) removePXE(m *Machine) { - for _, mac := range m.MACs { - if err := os.Remove(a.pxePath(mac)); err == nil { - log.Printf("removed %s", a.pxePath(mac)) - } - } -} - // ---------------------------------------------------------------- handlers func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) { @@ -427,6 +435,7 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) { return } _ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail) + _ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r)) if full, err := a.store.Get(r.Context(), m.ID); err == nil { m = full // carries profile / node token when this is a re-commission of a stateless node } @@ -527,17 +536,7 @@ func (a *App) handleRecommission(w http.ResponseWriter, r *http.Request) { // optional: power-cycle it right away when the caller asks (?now=1) out := obj{"id": m.ID, "state": "recommission"} if r.URL.Query().Get("now") == "1" { - if c, err := a.credsFor(m); err == nil { - o, ierr := a.ipmi(r.Context(), c, "chassis", "bootdev", "pxe") - o2, ierr2 := a.ipmi(r.Context(), c, "chassis", "power", "cycle") - if ierr2 != nil && strings.Contains(o2, "off") { // cycle fails when off; power on instead - o2, ierr2 = a.ipmi(r.Context(), c, "chassis", "power", "on") - } - out["ipmi"] = strings.TrimSpace(o + "\n" + o2) - out["ipmi_ok"] = ierr == nil && ierr2 == nil - } else { - out["ipmi"] = err.Error() - } + out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m) } writeJSON(w, 200, out) } @@ -564,12 +563,11 @@ func (a *App) handlePower(w http.ResponseWriter, r *http.Request) { fail(w, 400, fmt.Errorf("unknown power action %q", action)) return } - c, err := a.credsFor(m) - if err != nil { + out, err := a.bmc(r.Context(), m, args...) + if errors.Is(err, errNoBMC) { fail(w, 409, err) return } - out, err := a.ipmi(r.Context(), c, args...) if action != "status" { _ = a.store.AddEvent(r.Context(), m.ID, "power "+action, out) } @@ -589,12 +587,11 @@ func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) { fail(w, 400, fmt.Errorf("device must be one of pxe, disk, cdrom, bios, none, safe")) return } - c, err := a.credsFor(m) - if err != nil { + out, err := a.bmc(r.Context(), m, "chassis", "bootdev", req.Device) + if errors.Is(err, errNoBMC) { fail(w, 409, err) return } - out, err := a.ipmi(r.Context(), c, "chassis", "bootdev", req.Device) _ = a.store.AddEvent(r.Context(), m.ID, "bootdev "+req.Device, out) writeJSON(w, 200, obj{"id": m.ID, "device": req.Device, "ok": err == nil, "output": out}) } @@ -614,7 +611,13 @@ func serve(cfgPath string) error { if err != nil { return fmt.Errorf("database: %w", err) } - app := &App{cfg: cfg, store: store, sealer: sealer} + app := &App{cfg: cfg, store: store, sealer: sealer, hub: newRackHub()} + ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second) + err = app.startRacks(ctx2) + cancel2() + if err != nil { + return fmt.Errorf("rack controllers: %w", err) + } srv := &http.Server{Addr: cfg.Listen, Handler: app.routes(), ReadHeaderTimeout: 10 * time.Second} log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "") return srv.ListenAndServe() diff --git a/cmd/rollcall/stateless.go b/cmd/rollcall/stateless.go index f757161..2c2988f 100644 --- a/cmd/rollcall/stateless.go +++ b/cmd/rollcall/stateless.go @@ -186,7 +186,29 @@ func parsePool(spec string) ([]string, error) { return out, nil } -func (a *App) allocate(ctx context.Context, nameFormat, wantName string) (name, mgmt, ib string, err error) { +// allocate picks a node name and addresses: from IPAM when the machine's boot +// NIC sits in a modeled subnet, else from the legacy node_pools. +func (a *App) allocate(ctx context.Context, m *Machine, nameFormat, wantName string) (name, mgmt, ib string, err error) { + if name, _, _, err = a.allocatePools(ctx, nameFormat, wantName, false); err != nil { + return + } + if b := bootNIC(machineNICs(m), m.MACs); b != nil { + for _, ip := range b.IPv4 { + if sn := subnetFor(a.subnetsSnapshot(), ip); sn != nil { + have, _ := a.store.IPs(ctx, sn.CIDR, m.ID) + for _, x := range have { + return name, x.IP, "", nil + } + mgmt, err = a.allocIP(ctx, *sn, m.ID, b.Name, name) + return + } + } + } + _, mgmt, ib, err = a.allocatePools(ctx, nameFormat, name, true) + return +} + +func (a *App) allocatePools(ctx context.Context, nameFormat, wantName string, pools bool) (name, mgmt, ib string, err error) { ms, err := a.store.List(ctx) if err != nil { return @@ -215,10 +237,13 @@ func (a *App) allocate(ctx context.Context, nameFormat, wantName string) (name, break } } - } else if usedName[name] { + } else if usedName[name] && !pools { err = fmt.Errorf("node name %s already in use", name) return } + if !pools { + return + } pick := func(pool string) (string, error) { spec := a.cfg.NodePools[pool] if spec == "" { @@ -441,7 +466,7 @@ func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) { } } } - return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, Vars: vars, Server: a.cfg.ServerURL, + return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, Vars: vars, Server: a.urlFor(m), Now: time.Now().Format(time.RFC3339)}, nil } @@ -614,11 +639,12 @@ LABEL local ` func (a *App) statelessAppend(m *Machine, p *Profile, token string) string { + base := a.urlFor(m) parts := []string{"boot=live", "components", - "fetch=" + a.cfg.ServerURL + "/images/" + p.Image + "/filesystem.squashfs", + "fetch=" + base + "/images/" + p.Image + "/filesystem.squashfs", "hostname=" + deref(m.NodeName), "live-config.nocomponents=user-setup,sudo", - "rollcall.server=" + a.cfg.ServerURL, "rollcall.node=" + token} + "rollcall.server=" + base, "rollcall.node=" + token} if p.Append != "" { parts = append(parts, p.Append) } @@ -628,7 +654,7 @@ func (a *App) statelessAppend(m *Machine, p *Profile, token string) string { func (a *App) writePXEStateless(m *Machine, p *Profile, token string) error { body := fmt.Sprintf(statelessTmpl, time.Now().Format(time.RFC3339), deref(m.NodeName), m.ID, p.Name, deref(m.NodeName), p.Name, deref(m.NodeName), p.Name, p.Image, p.Image, a.statelessAppend(m, p, token), - a.cfg.CommissionAppend) + a.appendFor(a.urlFor(m))) return a.writePXEBody(m, body) } @@ -651,51 +677,6 @@ func (a *App) writePXEAny(ctx context.Context, m *Machine) error { return a.writePXE(m) } -// exportDnsmasq writes dhcp-hostsfile and addn-hosts files; a systemd path unit HUPs dnsmasq. -func (a *App) exportDnsmasq(ctx context.Context) error { - if a.cfg.DnsmasqDir == "" { - return nil - } - ms, err := a.store.List(ctx) - if err != nil { - return err - } - domain := str(a.cfg.ClusterVars["domain"]) - var dhcp, hosts bytes.Buffer - fmt.Fprintf(&dhcp, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339)) - fmt.Fprintf(&hosts, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339)) - for _, m := range ms { - if m.NodeName == nil || len(m.MACs) == 0 { - continue - } - name := *m.NodeName - if m.MgmtIP != nil { - fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], *m.MgmtIP, name) - if domain != "" { - fmt.Fprintf(&hosts, "%s %s.%s %s\n", *m.MgmtIP, name, domain, name) - } else { - fmt.Fprintf(&hosts, "%s %s\n", *m.MgmtIP, name) - } - } - if m.IBIP != nil { - fmt.Fprintf(&hosts, "%s %s-ib\n", *m.IBIP, name) - } - } - if err := os.MkdirAll(a.cfg.DnsmasqDir, 0o755); err != nil { - return err - } - for fn, b := range map[string]*bytes.Buffer{"hosts.dhcp": &dhcp, "hosts": &hosts} { - p := filepath.Join(a.cfg.DnsmasqDir, fn) - if err := os.WriteFile(p+".tmp", b.Bytes(), 0o644); err != nil { - return err - } - if err := os.Rename(p+".tmp", p); err != nil { - return err - } - } - return nil -} - // ---------------------------------------------------------------- handlers func (a *App) handleProfiles(w http.ResponseWriter, r *http.Request) { @@ -769,7 +750,7 @@ func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) { // keep the existing name/IPs when re-assigning an already named node name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP) if name == "" || (req.Name != "" && req.Name != name) { - if name, mgmt, ib, err = a.allocate(r.Context(), p.NameFormat, req.Name); err != nil { + if name, mgmt, ib, err = a.allocate(r.Context(), m, p.NameFormat, req.Name); err != nil { fail(w, 409, err) return } @@ -805,18 +786,17 @@ func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) { // pxeBoot sets bootdev pxe and power-cycles (or powers on) through the BMC. func (a *App) pxeBoot(ctx context.Context, m *Machine) (string, bool) { - c, err := a.credsFor(m) - if err != nil { - return err.Error(), false + o1, e1 := a.bmc(ctx, m, "chassis", "bootdev", "pxe") + if errors.Is(e1, errNoBMC) { + return e1.Error(), false } - o1, e1 := a.ipmi(ctx, c, "chassis", "bootdev", "pxe") - st, _ := a.ipmi(ctx, c, "chassis", "power", "status") + st, _ := a.bmc(ctx, m, "chassis", "power", "status") var o2 string var e2 error if strings.Contains(st, "off") { - o2, e2 = a.ipmi(ctx, c, "chassis", "power", "on") + o2, e2 = a.bmc(ctx, m, "chassis", "power", "on") } else { - o2, e2 = a.ipmi(ctx, c, "chassis", "power", "cycle") + o2, e2 = a.bmc(ctx, m, "chassis", "power", "cycle") } _ = a.store.AddEvent(ctx, m.ID, "pxe boot", strings.TrimSpace(o1+"\n"+o2)) return strings.TrimSpace(o1 + "\n" + o2), e1 == nil && e2 == nil @@ -828,6 +808,7 @@ func (a *App) handleUnassign(w http.ResponseWriter, r *http.Request) { fail(w, 500, err) return } + _ = a.store.FreeMachineIPs(r.Context(), m.ID, "auto") if err := a.store.Unassign(r.Context(), m.ID); err != nil { fail(w, 500, err) return diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index 0818737..b43617c 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -100,6 +100,76 @@ CREATE TABLE IF NOT EXISTS racks ( comment text, PRIMARY KEY (building, room, name) ); +ALTER TABLE machines ADD COLUMN IF NOT EXISTS rack_controller text; +ALTER TABLE machines ADD COLUMN IF NOT EXISTS net_layout text; +ALTER TABLE tags ADD COLUMN IF NOT EXISTS net_layout text; +CREATE TABLE IF NOT EXISTS fabrics ( + name text PRIMARY KEY, + kind text NOT NULL DEFAULT 'ethernet', + comment text +); +CREATE TABLE IF NOT EXISTS vlans ( + id bigserial PRIMARY KEY, + fabric text NOT NULL REFERENCES fabrics(name) ON DELETE CASCADE, + vid int NOT NULL DEFAULT 0, + name text, + mtu int NOT NULL DEFAULT 1500, + dhcp boolean NOT NULL DEFAULT false, + primary_rack text, + secondary_rack text, + UNIQUE (fabric, vid) +); +CREATE TABLE IF NOT EXISTS subnets ( + cidr text PRIMARY KEY, + name text, + vlan_id bigint REFERENCES vlans(id) ON DELETE SET NULL, + purpose text NOT NULL DEFAULT 'provisioning', + gateway text, + dns jsonb NOT NULL DEFAULT '[]', + domain text, + comment text +); +CREATE TABLE IF NOT EXISTS ip_ranges ( + id bigserial PRIMARY KEY, + subnet text NOT NULL REFERENCES subnets(cidr) ON DELETE CASCADE, + kind text NOT NULL, + start_ip text NOT NULL, + end_ip text NOT NULL, + comment text +); +CREATE TABLE IF NOT EXISTS ip_addresses ( + ip text PRIMARY KEY, + subnet text NOT NULL REFERENCES subnets(cidr) ON DELETE CASCADE, + machine_id text, + iface text, + kind text NOT NULL, + hostname text, + created timestamptz NOT NULL DEFAULT now() +); +CREATE INDEX IF NOT EXISTS ip_addresses_machine ON ip_addresses(machine_id); +CREATE TABLE IF NOT EXISTS net_layouts ( + name text PRIMARY KEY, + spec jsonb NOT NULL, + comment text +); +CREATE TABLE IF NOT EXISTS rack_controllers ( + name text PRIMARY KEY, + url text, + kind text NOT NULL DEFAULT 'remote', + version text, + interfaces jsonb NOT NULL DEFAULT '[]', + dhcp_interfaces jsonb NOT NULL DEFAULT '[]', + tftp_root text, + last_seen timestamptz, + registered timestamptz NOT NULL DEFAULT now() +); +CREATE TABLE IF NOT EXISTS pxe_files ( + mac text PRIMARY KEY, + machine_id text, + rack text, + content text NOT NULL, + updated timestamptz NOT NULL DEFAULT now() +); ` var errNotFound = errors.New("not found") @@ -131,6 +201,8 @@ type Machine struct { Tags []string `json:"tags"` // manual AutoTags []string `json:"auto_tags"` // from tag match rules, recomputed at commissioning Location Location `json:"location"` + RackController string `json:"rack_controller"` + NetLayout string `json:"net_layout"` bmcSecret []byte nodeToken []byte hostKey []byte @@ -156,7 +228,8 @@ func openStore(ctx context.Context, url string) (*Store, error) { const machineCols = `id, uuid, serial, hostname, manufacturer, product, state, first_seen, last_seen, commissioned_at, macs, bmc, bmc_secret, inventory, profile, node_name, mgmt_ip, ib_ip, node_token, host_key, booted_at, os, os_version, deployed_at, tags, auto_tags, - coalesce(building,''), coalesce(room,''), coalesce(rack,''), coalesce(rack_unit,0), coalesce(asset_tag,'')` + coalesce(building,''), coalesce(room,''), coalesce(rack,''), coalesce(rack_unit,0), coalesce(asset_tag,''), + coalesce(rack_controller,''), coalesce(net_layout,'')` func scanMachine(row pgx.Row, full bool) (*Machine, error) { m := &Machine{} @@ -165,7 +238,8 @@ func scanMachine(row pgx.Row, full bool) (*Machine, error) { &m.FirstSeen, &m.LastSeen, &m.CommissionedAt, &macs, &m.BMC, &m.bmcSecret, &m.Inventory, &m.Profile, &m.NodeName, &m.MgmtIP, &m.IBIP, &m.nodeToken, &m.hostKey, &m.BootedAt, &m.OS, &m.OSVersion, &m.DeployedAt, &tags, &auto, - &m.Location.Building, &m.Location.Room, &m.Location.Rack, &m.Location.Unit, &m.Location.AssetTag) + &m.Location.Building, &m.Location.Room, &m.Location.Rack, &m.Location.Unit, &m.Location.AssetTag, + &m.RackController, &m.NetLayout) if errors.Is(err, pgx.ErrNoRows) { return nil, errNotFound } @@ -276,7 +350,7 @@ func (s *Store) UpsertCommissioned(ctx context.Context, m *Machine, secret []byt } // a node that already has a profile goes back to 'stateless', everything else to 'ready' _, err = tx.Exec(ctx, `UPDATE machines SET uuid=$2, serial=$3, hostname=$4, manufacturer=$5, product=$6, - state=CASE WHEN profile IS NULL THEN 'ready' ELSE 'stateless' END, booted_at=NULL, + state=CASE WHEN profile IS NULL THEN 'ready' ELSE 'stateless' END, booted_at=NULL, os=NULL, os_version=NULL, deployed_at=NULL, last_seen=now(), commissioned_at=now(), macs=$7, bmc=$8, bmc_secret=$9, inventory=$10 WHERE id=$1`, m.ID, m.UUID, m.Serial, m.Hostname, m.Manufacturer, m.Product, macs, m.BMC, secret, m.Inventory) if err != nil { @@ -308,6 +382,12 @@ func (s *Store) Delete(ctx context.Context, id string) error { if _, err := tx.Exec(ctx, `DELETE FROM jobs WHERE machine_id=$1`, id); err != nil { return err } + if _, err := tx.Exec(ctx, `DELETE FROM ip_addresses WHERE machine_id=$1`, id); err != nil { + return err + } + if _, err := tx.Exec(ctx, `DELETE FROM pxe_files WHERE machine_id=$1`, id); err != nil { + return err + } tag, err := tx.Exec(ctx, `DELETE FROM machines WHERE id=$1`, id) if err != nil { return err diff --git a/cmd/rollcall/tags.go b/cmd/rollcall/tags.go index b1558d2..40f663d 100644 --- a/cmd/rollcall/tags.go +++ b/cmd/rollcall/tags.go @@ -31,6 +31,7 @@ type TagDef struct { Match string `json:"match"` CloudConfig string `json:"cloud_config"` Overlays []string `json:"overlays"` + NetLayout string `json:"net_layout"` Created time.Time `json:"created"` Machines int `json:"machines,omitempty"` // filled by the list handler } @@ -40,7 +41,7 @@ var tagNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,63}$`) // ---------------------------------------------------------------- store func (s *Store) Tags(ctx context.Context) ([]TagDef, error) { - rows, err := s.pool.Query(ctx, `SELECT name, coalesce(comment,''), coalesce(match,''), coalesce(cloud_config,''), overlays, created FROM tags ORDER BY name`) + rows, err := s.pool.Query(ctx, `SELECT name, coalesce(comment,''), coalesce(match,''), coalesce(cloud_config,''), overlays, coalesce(net_layout,''), created FROM tags ORDER BY name`) if err != nil { return nil, err } @@ -49,7 +50,7 @@ func (s *Store) Tags(ctx context.Context) ([]TagDef, error) { for rows.Next() { var t TagDef var ov []byte - if err := rows.Scan(&t.Name, &t.Comment, &t.Match, &t.CloudConfig, &ov, &t.Created); err != nil { + if err := rows.Scan(&t.Name, &t.Comment, &t.Match, &t.CloudConfig, &ov, &t.NetLayout, &t.Created); err != nil { return nil, err } _ = json.Unmarshal(ov, &t.Overlays) @@ -66,9 +67,9 @@ func (s *Store) PutTag(ctx context.Context, t TagDef) error { t.Overlays = []string{} } ov, _ := json.Marshal(t.Overlays) - _, err := s.pool.Exec(ctx, `INSERT INTO tags(name, comment, match, cloud_config, overlays) VALUES ($1,$2,$3,$4,$5) - ON CONFLICT (name) DO UPDATE SET comment=$2, match=$3, cloud_config=$4, overlays=$5`, - t.Name, t.Comment, t.Match, t.CloudConfig, ov) + _, err := s.pool.Exec(ctx, `INSERT INTO tags(name, comment, match, cloud_config, overlays, net_layout) VALUES ($1,$2,$3,$4,$5,nullif($6,'')) + ON CONFLICT (name) DO UPDATE SET comment=$2, match=$3, cloud_config=$4, overlays=$5, net_layout=nullif($6,'')`, + t.Name, t.Comment, t.Match, t.CloudConfig, ov, t.NetLayout) return err } diff --git a/cmd/rollcall/tui.go b/cmd/rollcall/tui.go index 2416a8e..fd2b9a8 100644 --- a/cmd/rollcall/tui.go +++ b/cmd/rollcall/tui.go @@ -69,6 +69,7 @@ const ( pOverview = iota pMachines pRacks + pNetwork pJobs pEvents pTags @@ -77,8 +78,8 @@ const ( pCount ) -var pageLabels = [pCount]string{"Overview", "Machines", "Racks", "Jobs", "Events", "Tags", "Profiles", "OS images"} -var pageNames = [pCount]string{"overview", "machines", "racks", "jobs", "events", "tags", "profiles", "images"} +var pageLabels = [pCount]string{"Overview", "Machines", "Racks", "Network", "Jobs", "Events", "Tags", "Profiles", "OS images"} +var pageNames = [pCount]string{"overview", "machines", "racks", "network", "jobs", "events", "tags", "profiles", "images"} type tuiData struct { machines []Summary @@ -88,6 +89,9 @@ type tuiData struct { tags []TagDef profiles []Profile os []OSImage + subnets []Subnet + ctls []RackController + layouts []NetLayout at time.Time } @@ -98,6 +102,7 @@ type machineDetail struct { Inventory obj `json:"inventory"` Events []Event `json:"events"` Jobs []*Job `json:"jobs"` + Net obj `json:"-"` } type ( @@ -339,12 +344,17 @@ func (m tuiModel) fetch() tea.Cmd { get("/api/v1/tags", &d.tags) get("/api/v1/profiles", &d.profiles) get("/api/v1/os", &d.os) + get("/api/v1/subnets", &d.subnets) + get("/api/v1/controllers", &d.ctls) + get("/api/v1/layouts", &d.layouts) d.at = time.Now() var det *machineDetail if id != "" && err == nil { det = &machineDetail{} if e := c.do("GET", "/api/v1/machines/"+id, nil, det); e != nil { det = nil + } else { + _ = c.do("GET", "/api/v1/machines/"+id+"/network", nil, &det.Net) } } return tuiDataMsg{d: d, detail: det, detailID: id, err: err} @@ -650,6 +660,18 @@ func (m tuiModel) enter() (tea.Model, tea.Cmd) { if m.row < len(m.racks()) { m.rackOpen, m.rackRow = m.row, 0 } + case pNetwork: + if m.row < len(m.d.subnets) { + s := m.d.subnets[m.row] + var b strings.Builder + fmt.Fprintf(&b, "%s · %s · fabric %s vlan %d · mtu %d\ngateway %s · dns %s · domain %s\ndhcp: %s\n\n", orDash(s.Name), s.Purpose, orDash(s.Fabric), s.VID, s.MTU, + orDash(s.Gateway), orDash(strings.Join(s.DNS, ",")), orDash(s.Domain), pick(s.DHCP, "on, rack controller "+s.PrimaryRack+pick(s.SecondaryRack != "", " + "+s.SecondaryRack, ""), "off")) + for _, r := range s.Ranges { + fmt.Fprintf(&b, "%-9s %s – %s %s\n", r.Kind, r.Start, r.End, sDim.Render(r.Comment)) + } + fmt.Fprintf(&b, "\n%d of %d addresses in use", s.Used, s.Size) + m.dlg = &tuiDialog{title: "Subnet " + s.CIDR, info: b.String()} + } case pJobs: if m.row < len(m.d.jobs) { return m.open(m.d.jobs[m.row].MachineID) @@ -878,6 +900,7 @@ func (m tuiModel) actionMenu(ids []string) *tuiDialog { }}, {label: "Tags…", sep: true, run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.tagDialog(ids) }}, {label: "Location…", run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.locationDialog(ids) }}, + {label: "Network layout…", run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.layoutDialog(ids) }}, {label: "Re-commission on next boot", sep: true, run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.run("recommission", ids, apiPost("/recommission", nil, func(d obj) string { return "state " + str(d["state"]) })) }}, @@ -1043,6 +1066,22 @@ func (m tuiModel) locationDialog(ids []string) (tuiModel, tea.Cmd) { return m, nil } +func (m tuiModel) layoutDialog(ids []string) (tuiModel, tea.Cmd) { + opts, vals := []string{"(from tags, else the boot NIC on its subnet)"}, []string{""} + for _, l := range m.d.layouts { + opts = append(opts, l.Name+" "+sDim.Render(trunc(layoutSummary(l), 40))) + vals = append(vals, l.Name) + } + m.dlg = &tuiDialog{title: "Network layout · " + m.who(ids), note: "Applied at the next deployment; addresses come from the subnets the layout names.", + fields: []dlgField{{label: "Layout", kind: fkChoice, opts: opts, vals: vals}}, okLabel: "save", + onOK: func(m tuiModel, f []dlgField) (tuiModel, tea.Cmd) { + req := obj{"layout": field(f, "Layout")} + return m.run("network layout", ids, apiPost("/network", req, func(d obj) string { return str(d["layout"]) })) + }} + m.dlg.setFocus(0) + return m, nil +} + func (m tuiModel) releaseDialog(ids []string) (tuiModel, tea.Cmd) { m.dlg = &tuiDialog{title: "Release · " + m.who(ids), note: "The OS disk is wiped and the machine returns to ready.", fields: []dlgField{ @@ -1215,6 +1254,13 @@ func (m tuiModel) panel() (string, string, string, int) { k = side } return "Racks", m.racksBody(pw), k, m.sel0(3) + case pNetwork: + body, head := m.networkBody(pw) + sel := -1 + if m.inPanel { + sel = m.row + head + } + return "Network", body, pick(m.inPanel, "↑↓ subnet · Enter details · Esc back · r refresh · q quit", side), sel case pJobs: return "Jobs", m.jobsBody(pw), pick(m.inPanel, list, side), m.sel0(3) case pEvents: @@ -1249,6 +1295,8 @@ func (m tuiModel) rowCount() int { return len(m.filtered()) case pRacks: return len(m.racks()) + case pNetwork: + return len(m.d.subnets) case pJobs: return len(m.d.jobs) case pEvents: @@ -1581,6 +1629,9 @@ func (m tuiModel) overviewBody(pw int) string { failed++ } } + for _, c := range m.d.ctls { + note(!c.Online, sBad.Render("rack controller "+c.Name+" is offline (last seen "+agoP(c.LastSeen)+" ago)")) + } note(nobmc > 0, sBad.Render(fmt.Sprintf("%d machine(s) without a configured BMC", nobmc))) note(silent > 0, sWarn.Render(fmt.Sprintf("%d stateless node(s) silent for more than 15 minutes", silent))) note(failed > 0, sBad.Render(fmt.Sprintf("%d failed job(s) among the last 100", failed))) @@ -1718,6 +1769,46 @@ func (m tuiModel) detailPanel(pw int) (string, string, string, int) { kv("BMC", sBad.Render("none")) } + if d.Net != nil { + sec("Network · layout " + str(d.Net["layout"]) + " · rack controller " + str(d.Net["rack_controller"])) + plan, _ := d.Net["plan"].(obj) + for _, iv := range toList(plan["interfaces"]) { + i, _ := iv.(obj) + what := str(i["mac"]) + switch str(i["type"]) { + case "bond": + var ms []string + for _, x := range toList(i["members"]) { + ms = append(ms, str(x)) + } + what = "bond " + str(i["bond_mode"]) + " of " + strings.Join(ms, ",") + case "vlan": + what = "vlan " + str(i["vid"]) + " on " + str(i["parent"]) + } + for _, lv := range toList(i["links"]) { + l, _ := lv.(obj) + ip := str(l["ip"]) + if ip == "" && str(l["mode"]) == "auto" { + ip = sDim.Render("allocated at deploy") + } else if ip != "" { + ip = fmt.Sprintf("%s/%d", ip, int(num(l["prefix"]))) + } + gw := "" + if l["default_gateway"] == true { + gw = sDim.Render(" default via " + str(l["gateway"])) + } + w(" %-12s %-28s %-8s %-18s %s%s", str(i["name"]), truncateWidth(what, 28), str(l["mode"]), orDash(str(l["subnet"])), ip, gw) + what = "" + } + } + for _, x := range toList(plan["warnings"]) { + w(" %s", sWarn.Render("! "+str(x))) + } + if d.Net["modeled"] == false { + w(" %s", sDim.Render("no subnets defined: deployed machines use DHCP")) + } + } + job := s.Job if job == nil && len(d.Jobs) > 0 { job = d.Jobs[0] @@ -2013,3 +2104,53 @@ func tuiDump(args []string) error { fmt.Println(out) return nil } + +// networkBody: rack controllers, then the subnets (the selectable rows), then layouts. +// It returns the body and the line of the first subnet row. +func (m tuiModel) networkBody(pw int) (string, int) { + var b strings.Builder + b.WriteString(" " + sTitle.Render("Rack controllers") + "\n") + var rows [][]string + for _, c := range m.d.ctls { + st := sOK.Render("● online") + if !c.Online { + st = sBad.Render("● offline") + } + var ifs []string + for _, i := range c.Interfaces { + ifs = append(ifs, i.Name+" "+strings.Join(i.Addrs, ",")) + } + rows = append(rows, []string{c.Name, c.Kind, st, c.URL, fmt.Sprint(c.Machines), agoP(c.LastSeen), sDim.Render(strings.Join(ifs, " · "))}) + } + b.WriteString(table([]string{"Controller", "Kind", "State", "URL", "Machines", "Seen", "Interfaces"}, rows, -1, pw-2, false)) + b.WriteString("\n " + sTitle.Render("Subnets") + "\n") + head := strings.Count(b.String(), "\n") + 1 + rows = nil + for _, s := range m.d.subnets { + dhcp := sDim.Render("off") + if s.DHCP { + dhcp = sOK.Render("on") + " " + s.PrimaryRack + if s.SecondaryRack != "" { + dhcp += "+" + s.SecondaryRack + } + } + var rs []string + for _, r := range s.Ranges { + rs = append(rs, r.Kind[:1]+":"+lastOctets(r.Start)+"-"+lastOctets(r.End)) + } + pct := 0.0 + if s.Size > 0 { + pct = float64(s.Used) * 100 / float64(s.Size) + } + rows = append(rows, []string{s.CIDR, s.Purpose, fmt.Sprintf("%s/%d", orDash(s.Fabric), s.VID), dhcp, orDash(s.Gateway), + fmt.Sprintf("%s %d/%d", bar(pct, 8), s.Used, s.Size), strings.Join(rs, " ")}) + } + b.WriteString(table([]string{"Subnet", "Purpose", "Fabric/VLAN", "DHCP", "Gateway", "Used", "Ranges"}, rows, m.row, pw-2, m.inPanel)) + b.WriteString("\n " + sTitle.Render("Interface layouts") + "\n") + rows = nil + for _, l := range m.d.layouts { + rows = append(rows, []string{l.Name, fmt.Sprint(l.Machines), layoutSummary(l), l.Comment}) + } + b.WriteString(table([]string{"Layout", "Machines", "Interfaces", "Comment"}, rows, -1, pw-2, false)) + return b.String(), head +} diff --git a/cmd/rollcall/web/app.js b/cmd/rollcall/web/app.js index 37a0b85..9405ff5 100644 --- a/cmd/rollcall/web/app.js +++ b/cmd/rollcall/web/app.js @@ -14,7 +14,7 @@ let token = ''; try { token = localStorage.getItem('rollcall.token') || ''; } catch (e) { /* storage unavailable */ } // ?token=… on the URL signs in once and is removed from the address bar (for bookmarks/kiosks) { const u = new URL(location.href); const t = u.searchParams.get('token'); if (t) { token = t; try { localStorage.setItem('rollcall.token', t); } catch (e) { /* ignore */ } u.searchParams.delete('token'); history.replaceState(null, '', u.pathname + u.search + u.hash); } } -const S = { machines: [], tags: [], profiles: [], os: [], jobs: [], events: [], racks: [], sel: new Set(), q: '', states: new Set(), tagf: new Set(), loc: { building: '', room: '', rack: '' }, page: 0, per: 50, sort: 'name', dir: 1, loaded: false }; +const S = { machines: [], tags: [], profiles: [], os: [], jobs: [], events: [], racks: [], subnets: [], controllers: [], layouts: [], fabrics: [], sel: new Set(), q: '', states: new Set(), tagf: new Set(), loc: { building: '', room: '', rack: '' }, page: 0, per: 50, sort: 'name', dir: 1, loaded: false }; const locStr = l => { if (!l) return ''; const p = [l.building, l.room, l.rack].filter(Boolean).join(' / '); return p + (l.unit ? ` U${l.unit}` : ''); }; let timer = null; @@ -59,9 +59,12 @@ function dialog(title, fields, note, okLabel = 'OK') { async function refresh(full) { if (!token) return askToken(); try { - const [machines, tags, profiles, os, racks] = await Promise.all([api('GET', '/api/v1/machines'), api('GET', '/api/v1/tags'), api('GET', '/api/v1/profiles'), api('GET', '/api/v1/os'), api('GET', '/api/v1/racks')]); - Object.assign(S, { machines, tags, profiles, os, racks, loaded: true }); + const [machines, tags, profiles, os, racks, subnets, controllers, layouts, fabrics] = await Promise.all([api('GET', '/api/v1/machines'), api('GET', '/api/v1/tags'), api('GET', '/api/v1/profiles'), api('GET', '/api/v1/os'), api('GET', '/api/v1/racks'), + api('GET', '/api/v1/subnets'), api('GET', '/api/v1/controllers'), api('GET', '/api/v1/layouts'), api('GET', '/api/v1/fabrics')]); + Object.assign(S, { machines, tags, profiles, os, racks, subnets, controllers, layouts, fabrics, loaded: true }); $('#n-racks').textContent = racks.length || ''; + const off = controllers.filter(c => !c.online).length; + $('#n-net').textContent = off ? `${off} offline` : (subnets.length || ''); const page = route().page; if (full || page === 'overview' || page === 'jobs') S.jobs = await api('GET', '/api/v1/jobs'); if (full || page === 'overview') S.events = await api('GET', '/api/v1/events'); @@ -107,6 +110,7 @@ function render() { case 'machine': if (!detailCache[r.id]) { main.innerHTML = '

loading…

'; loadDetail(r.id); } else { main.innerHTML = pageMachine(detailCache[r.id], r.tab); wireMachine(detailCache[r.id]); } break; case 'jobs': main.innerHTML = pageJobs(); wireJobs(); break; case 'racks': main.innerHTML = pageRacks(); wireRacks(); break; + case 'network': main.innerHTML = pageNetwork(); wireNetwork(); break; case 'tags': main.innerHTML = pageTags(); wireTags(); break; case 'profiles': main.innerHTML = pageProfiles(); wireProfiles(); break; case 'images': main.innerHTML = pageImages(); break; @@ -133,6 +137,7 @@ function pageOverview() {

Active jobs

${jobsTable(active, true)}

Attention

${kv([ + ['Rack controllers offline', (() => { const off = S.controllers.filter(c => !c.online); return off.length ? `${off.map(c => esc(c.name)).join(', ')} show` : `none (${S.controllers.length})`; })()], ['BMC not configured', bmcBad ? `${bmcBad} show` : 'none'], ['Stateless nodes silent > 15 min', stale ? `${stale}` : 'none'], ['Failed jobs (last 100)', (() => { const n = S.jobs.filter(j => j.state === 'failed').length; return n ? `${n} show` : 'none'; })()], @@ -223,7 +228,7 @@ function bulkBar() { - ${menuBtn('More', [['recommission', 'Re-commission (next boot)'], ['recommission-now', 'Re-commission now (BMC)'], ['release', 'Release (wipe disk)…'], ['unassign', 'Unassign profile'], ['hr'], ['bootdev-pxe', 'Boot device: PXE'], ['bootdev-disk', 'Boot device: disk']], 'bmore')} + ${menuBtn('More', [['layout', 'Network layout…'], ['recommission', 'Re-commission (next boot)'], ['recommission-now', 'Re-commission now (BMC)'], ['release', 'Release (wipe disk)…'], ['unassign', 'Unassign profile'], ['hr'], ['bootdev-pxe', 'Boot device: PXE'], ['bootdev-disk', 'Boot device: disk']], 'bmore')}
`; } function menuBtn(label, items, id) { return `
${items.map(([v, l]) => v === 'hr' ? '
' : ``).join('')}
`; } @@ -341,6 +346,12 @@ const act = { case 'unassign': if (confirm(`Remove the stateless profile from ${n} machine(s)? Names and IPs are released.`)) await each(ids, 'unassign', async id => (await api('POST', `/api/v1/machines/${id}/unassign`)).state); break; case 'bootdev-pxe': await act.bootdev(ids, 'pxe'); break; case 'bootdev-disk': await act.bootdev(ids, 'disk'); break; + case 'layout': { + const one = ids.length === 1 ? S.machines.find(m => m.id === ids[0]) : null; + const v = await dialog(`Network layout of ${one ? nameOf(one) : n + ' machines'}`, [{ name: 'layout', label: 'Layout', type: 'select', value: one ? one.net_layout : '', options: [['', '(from tags, else the boot NIC on its subnet)'], ...S.layouts.map(l => [l.name, `${l.name} · ${l.comment || layoutSummary(l)}`])] }], 'Applied at the next deployment; addresses come from the subnets the layout names.', 'Save'); + if (v) await each(ids, 'network layout', async id => { const r = await api('POST', `/api/v1/machines/${id}/network`, { layout: v.layout }); delete netCache[id]; return r.layout; }); + break; + } case 'cancel-job': await each(ids, 'cancel job', async id => `job ${(await api('POST', `/api/v1/machines/${id}/jobs/cancel`)).job} cancelled`); break; case 'delete': if (confirm(`Delete ${n} machine record(s)? They are commissioned again as new machines on their next PXE boot.`)) { await each(ids, 'delete', async id => { await api('DELETE', `/api/v1/machines/${id}`); delete detailCache[id]; return 'deleted'; }); S.sel.clear(); nav('machines'); } break; } @@ -370,6 +381,7 @@ function pageMachine(d, tab) { ${card('Role', kv([ ['State', pill(s.state)], ['Node name', esc(s.node_name)], ['Management IP', s.mgmt_ip ? `${esc(s.mgmt_ip)}` : ''], ['IB IP', s.ib_ip ? `${esc(s.ib_ip)}` : ''], + ['Rack controller', s.rack_controller ? `${esc(s.rack_controller)} network` : ''], ['Stateless profile', s.profile ? `${esc(s.profile)} · booted ${esc(s.booted_at ? when(s.booted_at) : 'not yet')}` : ''], ['OS on disk', s.os ? `${esc(s.os)} ${esc(s.os_version || '')} · deployed ${esc(when(s.deployed_at))}` : ''], ['Phoned home', s.deployed_at ? (s.booted_at ? `${esc(when(s.booted_at))}` : 'not yet') : ''], @@ -395,7 +407,7 @@ function pageMachine(d, tab) { } else if (tab === 'network') { const hcas = (inv.infiniband || {}).hcas || []; const lldpRows = []; for (const [ifn, ents] of Object.entries(inv.lldp || {})) for (const e of (Array.isArray(ents) ? ents : [ents])) { const ch = e.chassis || {}; const name = Object.keys(ch)[0] || '?'; const c = ch[name] || {}; lldpRows.push([esc(ifn), esc(name), esc((c.descr || '').slice(0, 60)), esc((e.port || {}).id && (e.port.id.value || e.port.id)), esc((e.port || {}).descr), esc((e.vlan || {})['vlan-id'] || (e.vlan || {}).value)]); } - body = `
+ body = netPlanCard(s.id) + `
${card(`NICs · ${(inv.network || []).length}`, table(['If', 'MAC', 'Driver', 'FW', 'Speed', 'Link', 'PCI', 'NUMA', 'VFs', 'IPv4'], (inv.network || []).map(n => [esc(n.name), `${esc(n.mac)}`, esc(n.driver), esc(n.firmware_version), esc(n.speed), n.link === true ? 'up' : n.link === false ? 'down' : '', `${esc(n.pci)}`, esc(n.numa_node), esc(n.sriov_totalvfs), esc((n.ipv4 || []).join(' '))])), true)} ${card(`InfiniBand · ${hcas.length} HCAs`, hcas.length ? table(['HCA', 'Port', 'Layer', 'State', 'Rate', 'LID', 'Node GUID', 'FW', 'Board'], hcas.flatMap(h => (h.ports || []).map(p => [esc(h.name), esc(p.port), esc(p.link_layer), esc(p.state), esc(p.rate), esc(p.lid), `${esc(h.node_guid)}`, esc(h.fw_ver), esc(h.board_id)]))) : 'none', true)} ${card('LLDP neighbours (switch ports)', table(['If', 'Switch', 'Descr', 'Port', 'Port descr', 'VLAN'], lldpRows), true)} @@ -416,7 +428,7 @@ function pageMachine(d, tab) { ${menuBtn('Boot device', [['pxe', 'PXE'], ['disk', 'Disk'], ['bios', 'BIOS setup'], ['cdrom', 'CD-ROM']], 'dboot')} - ${menuBtn('More', [['recommission', 'Re-commission (next boot)'], ['recommission-now', 'Re-commission now (BMC)'], ['release', 'Release (wipe disk)…'], ['unassign', 'Unassign profile'], ...(hasJob ? [['cancel-job', 'Cancel active job']] : []), ['hr'], ['delete', 'Delete record']], 'dmore')} + ${menuBtn('More', [['layout', 'Network layout…'], ['recommission', 'Re-commission (next boot)'], ['recommission-now', 'Re-commission now (BMC)'], ['release', 'Release (wipe disk)…'], ['unassign', 'Unassign profile'], ...(hasJob ? [['cancel-job', 'Cancel active job']] : []), ['hr'], ['delete', 'Delete record']], 'dmore')}
${esc([sys['system-manufacturer'], sys['system-product-name']].filter(Boolean).join(' '))}S/N ${esc(s.serial || '')}${hwSummary(m)}${s.mgmt_ip ? `${esc(s.mgmt_ip)}` : ''}seen ${ago(s.last_seen)} ago
${tabs.map(t => `${t === 'bmc' ? 'BMC' : t[0].toUpperCase() + t.slice(1)}`).join('')}
@@ -424,6 +436,7 @@ function pageMachine(d, tab) { } function wireMachine(d) { const id = d.summary.id, ids = [id]; + if (route().tab === 'network') wireNetPlan(id); wireMenus(); $$('#dpower [data-item]').forEach(b => b.onclick = () => act.power(ids, b.dataset.item)); $$('#dboot [data-item]').forEach(b => b.onclick = () => act.bootdev(ids, b.dataset.item)); @@ -489,7 +502,7 @@ function wireRacks() { // ---------------------------------------------------------------- tags function pageTags() { return `

Tags

single out machines for special treatment: auto rules from the inventory, cloud-init fragments at deployment, overlays for stateless nodes
- ${table(['Tag', 'Machines', 'Match rule', 'Cloud-config', 'Overlays', 'Comment', ''], S.tags.map(t => [`${esc(t.name)}`, `${t.machines || 0}`, `${esc(t.match || '')}`, t.cloud_config ? `${t.cloud_config.split('\n').length} lines` : '', esc((t.overlays || []).join(', ')), esc(t.comment || ''), ` `]))} + ${table(['Tag', 'Machines', 'Match rule', 'Cloud-config', 'Overlays', 'Layout', 'Comment', ''], S.tags.map(t => [`${esc(t.name)}`, `${t.machines || 0}`, `${esc(t.match || '')}`, t.cloud_config ? `${t.cloud_config.split('\n').length} lines` : '', esc((t.overlays || []).join(', ')), esc(t.net_layout || ''), esc(t.comment || ''), ` `]))}

Match rules are Go templates over .Node (GPUs, IBGUIDs, CPUs, Sockets, MemMB, Disks, NVMe, PCIVendors, Manufacturer, Product, OS, Tags), e.g. {{ gt .Node.GPUs 0 }}, {{ has .Node.PCIVendors "15b3" }}, {{ matches "(?i)PowerEdge R7" .Node.Product }}. Rules run at commissioning and whenever a tag is saved.

`; } async function editTag(t) { @@ -498,10 +511,11 @@ async function editTag(t) { { name: 'comment', label: 'Comment', value: t ? t.comment : '' }, { name: 'match', label: 'Match rule', value: t ? t.match : '', placeholder: '{{ gt .Node.GPUs 0 }} (empty = manual only)' }, { name: 'overlays', label: 'Overlays', value: t ? (t.overlays || []).join(',') : '', placeholder: 'gpu-slurm', help: 'extra overlays for stateless nodes with this tag' }, + { name: 'net_layout', label: 'Network layout', type: 'select', value: t ? t.net_layout : '', options: [['', '(none)'], ...S.layouts.map(l => [l.name, l.name])], help: 'interfaces of deployed machines with this tag' }, { name: 'cloud_config', label: 'Cloud-config', type: 'textarea', value: t ? t.cloud_config : '', placeholder: '#cloud-config\npackages: [nvidia-driver]\nruncmd:\n - [ sh, -c, "..." ]' }, ], 'The cloud-config fragment is merged into the user-data of every deployment of a tagged machine. A fragment starting with #! runs as a script.', 'Save'); if (!v) return; - try { await api('POST', '/api/v1/tags', { name: v.name.trim(), comment: v.comment, match: v.match, overlays: v.overlays.split(',').map(s => s.trim()).filter(Boolean), cloud_config: v.cloud_config }); toast(`tag ${v.name} saved; auto tags re-evaluated`); refresh(true); } catch (e) { toast(e.message, true); } + try { await api('POST', '/api/v1/tags', { name: v.name.trim(), comment: v.comment, match: v.match, overlays: v.overlays.split(',').map(s => s.trim()).filter(Boolean), cloud_config: v.cloud_config, net_layout: v.net_layout }); toast(`tag ${v.name} saved; auto tags re-evaluated`); refresh(true); } catch (e) { toast(e.message, true); } } function wireTags() { $('#newtag').onclick = () => editTag(null); @@ -545,3 +559,127 @@ function pageImages() { // ---------------------------------------------------------------- go schedule(); refresh(true); + +// ---------------------------------------------------------------- network: a machine's plan +const netCache = {}; +function layoutSummary(l) { + return (l.interfaces || []).map(i => i.type === 'bond' ? `${i.name || 'bond0'}(${(i.members || []).join('+')})` : i.type === 'vlan' ? `${i.parent}.${i.vid}` : (i.select || 'boot')).join(' '); +} +function netPlanCard(id) { + const n = netCache[id]; + if (!n) { api('GET', `/api/v1/machines/${encodeURIComponent(id)}/network`).then(r => { netCache[id] = r; if (route().page === 'machine' && route().id === id) render(); }).catch(e => toast(e.message, true)); return '

Network configuration

loading…
'; } + const p = n.plan || {}; + const rows = []; + for (const i of (p.interfaces || [])) { + const what = i.type === 'bond' ? `bond ${esc(i.bond_mode)} of ${esc((i.members || []).join(', '))}` : i.type === 'vlan' ? `vlan ${i.vid} on ${esc(i.parent)}` : `${esc(i.mac || '')}${i.ib ? ' IB' : ''}`; + const links = i.links && i.links.length ? i.links : [{}]; + links.forEach((l, k) => rows.push([k ? '' : `${esc(i.name)}`, k ? '' : what, k ? '' : esc(i.mtu || ''), `${esc(l.subnet || '')}`, esc(l.mode || '—'), + l.ip ? `${esc(l.ip)}/${l.prefix}` : (l.mode === 'auto' ? 'allocated at deploy' : ''), l.default_gateway ? `default via ${esc(l.gateway)}` : ''])); + } + const pins = (n.addresses || []).map(a => `${esc(a.ip)} ${esc(a.kind)}${a.iface ? ' · ' + esc(a.iface) : ''}${a.kind === 'static' ? ` ×` : ''}`).join(' '); + return `

Network configuration

+ ${n.modeled ? '' : '

No subnets are defined: deployed machines use DHCP. Define them on the Network page.

'} + ${kv([['Layout', `${esc(p.layout)} `], ['Rack controller', `${esc(n.rack_controller)} `], ['Addresses', (pins || 'none') + ' ']])} + ${table(['Interface', 'What', 'MTU', 'Subnet', 'Mode', 'Address', 'Route'], rows)} + ${(p.warnings || []).length ? `

${p.warnings.map(esc).join('
')}

` : ''} +
network-config (cloud-init v2, written into the image at deployment)
${esc(JSON.stringify(n.network_config, null, 2))}
`; +} +function wireNetPlan(id) { + const post = async (body, label) => { try { const r = await api('POST', `/api/v1/machines/${id}/network`, body); netCache[id] = r; toast(label); render(); } catch (e) { toast(e.message, true); } }; + const l = $('#n-layout'); if (l) l.onclick = () => act.more([id], 'layout'); + const rk = $('#n-rack'); if (rk) rk.onclick = async () => { + const v = await dialog('Rack controller', [{ name: 'rack', label: 'Serves this machine', type: 'select', value: (netCache[id] || {}).rack_controller, options: S.controllers.map(c => [c.name, `${c.name} · ${c.url}${c.online ? '' : ' (offline)'}`]) }], 'Normally set by where the machine last PXE-booted; change it when a machine moved to another segment.', 'Save'); + if (v) await post({ rack_controller: v.rack }, 'rack controller ' + v.rack); + }; + const pin = $('#n-pin'); if (pin) pin.onclick = async () => { + const ifs = (((netCache[id] || {}).plan || {}).interfaces || []).map(i => [i.name, i.name]); + const v = await dialog('Pin an address', [{ name: 'iface', label: 'Interface', type: 'select', options: ifs }, { name: 'ip', label: 'Address', placeholder: '10.0.0.60' }], 'A pinned address is used instead of one from the subnet\'s pool.', 'Pin'); + if (v && v.ip) await post({ static: [{ iface: v.iface, ip: v.ip.trim() }] }, 'pinned ' + v.ip); + }; + $$('[data-unpin]').forEach(a => a.onclick = ev => { ev.preventDefault(); post({ unstatic: [a.dataset.unpin] }, 'unpinned ' + a.dataset.unpin); }); +} + +// ---------------------------------------------------------------- network page +function pageNetwork() { + const ctl = table(['Controller', 'Kind', 'State', 'URL', 'Machines', 'Last seen', 'Interfaces', ''], S.controllers.map(c => [`${esc(c.name)}`, esc(c.kind), c.online ? 'online' : 'offline', `${esc(c.url)}`, c.machines, + `${ago(c.last_seen)}`, `${esc((c.interfaces || []).map(i => i.name + ' ' + (i.addrs || []).join(',')).join(' · '))}`, `${c.kind === 'remote' ? ` ` : ''}`])); + const sub = table(['Subnet', 'Purpose', 'Fabric / VLAN', 'DHCP', 'Gateway', 'DNS', 'Usage', 'Ranges', ''], S.subnets.map(s => { + const pct = s.size ? Math.round(s.used * 100 / s.size) : 0; + const ranges = (s.ranges || []).map(r => `${esc(r.kind)} ${esc(r.start)}–${esc(r.end.split('.').pop())} ×`).join(' '); + return [`${esc(s.cidr)}${s.name ? `
${esc(s.name)}` : ''}`, esc(s.purpose), `${esc(s.fabric || '—')} / ${s.vid || 'untagged'}${s.mtu && s.mtu !== 1500 ? ` · mtu ${s.mtu}` : ''}`, + s.dhcp ? `on · ${esc(s.primary_rack)}${s.secondary_rack ? ' + ' + esc(s.secondary_rack) : ''}` : 'off', `${esc(s.gateway || '—')}`, + `${esc((s.dns || []).join(', '))}${s.domain ? `
${esc(s.domain)}` : ''}`, `
${s.used} / ${s.size}`, + ranges + ` `, ` `]; + })); + const lay = table(['Layout', 'Machines', 'Interfaces', 'Comment', ''], S.layouts.map(l => [`${esc(l.name)}`, l.machines || 0, `${esc(layoutSummary(l))}`, esc(l.comment || ''), ` `])); + return `

Network

fabrics, VLANs and subnets (MAAS-style); rack controllers serve DHCP/PXE on their VLANs +
+

Rack controllers

${ctl}

Add one next to a segment: deploy/install-rack.sh with /etc/rollcall/rack.json (region URL, rack_secret, its own URL). It registers itself.

+

Subnets

${sub}
+

Interface layouts

${lay}

A layout applies to a machine directly (More → Network layout…) or through a tag (Tags → layout). Without one, the boot NIC gets an address on its subnet.

`; +} +const layoutExample = JSON.stringify({ interfaces: [ + { type: 'physical', select: 'boot', links: [{ subnet: 'boot', mode: 'auto', default_gateway: true }] }, + { type: 'bond', name: 'bond0', members: ['fastest:2'], bond_mode: '802.3ad', mtu: 9000, links: [{ mode: 'link_up' }] }, + { type: 'vlan', parent: 'bond0', vid: 200, links: [{ subnet: '10.20.0.0/24', mode: 'auto' }] }, + { type: 'physical', select: 'ib:0', links: [{ subnet: '10.30.0.0/24', mode: 'auto' }] }] }, null, 2); +async function editSubnet(s) { + const fabrics = S.fabrics.map(f => [f.name, f.name]); + const v = await dialog(s ? `Subnet ${s.cidr}` : 'New subnet', [ + { name: 'cidr', label: 'CIDR', value: s ? s.cidr : '', placeholder: '10.50.0.0/24' }, { name: 'name', label: 'Name', value: s ? s.name : '' }, + { name: 'fabric', label: 'Fabric', value: s ? s.fabric : (S.fabrics[0] || {}).name, placeholder: 'fabric-0', help: 'a new name creates the fabric', list: 'dl-fabrics' }, + { name: 'vid', label: 'VLAN id', value: s ? String(s.vid) : '0', help: '0 = untagged' }, + { name: 'purpose', label: 'Purpose', type: 'select', value: s ? s.purpose : 'provisioning', options: ['provisioning', 'mgmt', 'bmc', 'storage', 'ib', 'other'].map(x => [x, x]) }, + { name: 'gateway', label: 'Gateway', value: s ? s.gateway : '' }, { name: 'dns', label: 'DNS servers', value: s ? (s.dns || []).join(',') : '', placeholder: 'comma separated' }, + { name: 'domain', label: 'Domain', value: s ? s.domain : '' }, { name: 'mtu', label: 'VLAN MTU', value: s ? String(s.mtu || 1500) : '1500' }, + { name: 'dhcp', label: 'Serve DHCP/PXE', type: 'checkbox', value: s ? s.dhcp : false }, + { name: 'rack', label: 'Rack controller', type: 'select', value: s ? s.primary_rack : '', options: S.controllers.map(c => [c.name, `${c.name}${c.online ? '' : ' (offline)'}`]) }, + ], 'Ranges are added afterwards: dynamic (PXE pool), reserved (never handed out), static (where machines get addresses).', 'Save'); + if (!v) return; + try { + const fabric = v.fabric.trim(); + if (fabric && !S.fabrics.some(f => f.name === fabric)) await api('POST', '/api/v1/fabrics', { name: fabric }); + const vid = Number(v.vid) || 0; + await api('POST', '/api/v1/subnets', { cidr: v.cidr.trim(), name: v.name, fabric, vid, purpose: v.purpose, gateway: v.gateway.trim(), dns: v.dns.split(',').map(x => x.trim()).filter(Boolean), domain: v.domain.trim() }); + if (fabric) await api('POST', '/api/v1/vlans', { fabric, vid, mtu: Number(v.mtu) || 1500, dhcp: v.dhcp, primary_rack: v.dhcp ? v.rack : '' }); + toast(`subnet ${v.cidr} saved`); refresh(true); + } catch (e) { toast(e.message, true); } +} +async function editLayout(l) { + const v = await dialog(l ? `Layout ${l.name}` : 'New layout', [{ name: 'name', label: 'Name', value: l ? l.name : '', placeholder: 'gpu-bond' }, { name: 'comment', label: 'Comment', value: l ? l.comment : '' }, + { name: 'spec', label: 'Interfaces (JSON)', type: 'textarea', rows: 14, value: l ? JSON.stringify({ interfaces: l.interfaces }, null, 2) : layoutExample }], + 'Selectors: boot, name:X, mac:X, fastest:N, ib:N, all. Link modes: auto (from the subnet), static (with ip), dhcp, link_up. Subnets by CIDR, name, or "boot".', 'Save'); + if (!v) return; + let spec; try { spec = JSON.parse(v.spec); } catch (e) { return toast('interfaces: invalid JSON', true); } + try { await api('POST', '/api/v1/layouts', { name: v.name.trim(), comment: v.comment, interfaces: spec.interfaces || spec }); for (const k in netCache) delete netCache[k]; toast(`layout ${v.name} saved`); refresh(true); } catch (e) { toast(e.message, true); } +} +function wireNetwork() { + let dl = $('#dl-fabrics'); if (!dl) { dl = document.createElement('datalist'); dl.id = 'dl-fabrics'; document.body.appendChild(dl); } + dl.innerHTML = S.fabrics.map(f => `