Compare commits
No files matched your search
@@ -30,6 +30,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: cmd/rollcall/go.mod
|
||||
@@ -37,7 +39,7 @@ jobs:
|
||||
|
||||
- name: Build
|
||||
working-directory: cmd/rollcall
|
||||
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w" -o ../../deploy/rollcall .
|
||||
run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags="-s -w -X main.version=$(git describe --tags --always --dirty)" -o ../../deploy/rollcall .
|
||||
|
||||
- name: SSH setup
|
||||
env:
|
||||
|
||||
@@ -10,6 +10,8 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
- uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version-file: cmd/rollcall/go.mod
|
||||
@@ -22,8 +24,34 @@ jobs:
|
||||
for arch in amd64 arm64; do
|
||||
CGO_ENABLED=0 GOOS=linux GOARCH=$arch go build -trimpath \
|
||||
-ldflags="-s -w -X main.version=${GITHUB_REF_NAME}" -o rollcall-linux-$arch .
|
||||
# install bundle: the binary with deploy/ (bootstrap.sh, units) and agent/ (image build)
|
||||
b=rollcall-${GITHUB_REF_NAME}-linux-$arch
|
||||
mkdir -p "$b"
|
||||
cp rollcall-linux-$arch "$b/rollcall"
|
||||
cp -r ../../deploy ../../agent ../../README.md ../../INSTALL.md ../../CHANGELOG.md "$b/"
|
||||
tar -czf "$b.tar.gz" "$b"
|
||||
rm -rf "$b"
|
||||
done
|
||||
sha256sum rollcall-linux-* > SHA256SUMS
|
||||
sha256sum rollcall-linux-* rollcall-*.tar.gz > SHA256SUMS
|
||||
|
||||
- name: Release notes from CHANGELOG.md
|
||||
env:
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -eu
|
||||
ver=${TAG#v}
|
||||
# the section "## [X.Y.Z]" up to the next "## " heading
|
||||
awk -v v="$ver" '
|
||||
/^## \[/ { inside = ($0 ~ "^## \\[" v "\\]") }
|
||||
inside && !/^## \[/ { print }
|
||||
' CHANGELOG.md > notes.md
|
||||
if ! grep -q '[^[:space:]]' notes.md; then
|
||||
echo "::error::CHANGELOG.md has no section '## [$ver]' for tag $TAG; add it before tagging" >&2
|
||||
exit 1
|
||||
fi
|
||||
prev=$(git describe --tags --abbrev=0 "$TAG^" 2>/dev/null || true)
|
||||
{ echo; echo "---"; if [ -n "$prev" ]; then echo "Commits: $prev...$TAG"; git log --no-merges --format='- %s' "$prev..$TAG"; else git log --no-merges --format='- %s' "$TAG"; fi; } >> notes.md
|
||||
echo "== release notes"; cat notes.md
|
||||
|
||||
- name: Create release and upload assets
|
||||
env:
|
||||
@@ -33,10 +61,10 @@ jobs:
|
||||
working-directory: cmd/rollcall
|
||||
run: |
|
||||
set -eu
|
||||
body=$(printf '{"tag_name":"%s","name":"%s","draft":false,"prerelease":false}' "$TAG" "$TAG")
|
||||
body=$(python3 -c 'import json,sys; print(json.dumps({"tag_name": sys.argv[1], "name": sys.argv[1], "body": open("../../notes.md").read(), "draft": False, "prerelease": "-" in sys.argv[1]}))' "$TAG")
|
||||
id=$(curl -sfS -X POST -H "Authorization: token $TOKEN" -H "Content-Type: application/json" \
|
||||
-d "$body" "$API/releases" | python3 -c 'import json,sys; print(json.load(sys.stdin)["id"])')
|
||||
for f in rollcall-linux-amd64 rollcall-linux-arm64 SHA256SUMS; do
|
||||
for f in rollcall-linux-amd64 rollcall-linux-arm64 rollcall-*.tar.gz SHA256SUMS; do
|
||||
curl -sfS -X POST -H "Authorization: token $TOKEN" \
|
||||
-F "attachment=@$f" "$API/releases/$id/assets?name=$f" >/dev/null
|
||||
echo "uploaded $f"
|
||||
|
||||
@@ -4,3 +4,4 @@
|
||||
*.key
|
||||
/site/dist/
|
||||
.DS_Store
|
||||
__pycache__/
|
||||
+127
@@ -0,0 +1,127 @@
|
||||
# Changelog
|
||||
|
||||
All notable changes to Rollcall. The format follows [Keep a Changelog](https://keepachangelog.com/en/1.1.0/);
|
||||
the release workflow uses the section of the pushed tag as the release notes and refuses to
|
||||
publish a tag that has no section here.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.2.0] - 2026-10-28
|
||||
|
||||
Second release: install from scratch, network model and region/rack split, external IPAM
|
||||
(NetBox), a certificate authority with delegation, many power drivers, IPv6, and a web UI and
|
||||
console for fleets.
|
||||
|
||||
### Added
|
||||
- Machine tags (MAAS-style): manual tags per machine, auto tags from match rules over the
|
||||
inventory (`{{ gt .Node.GPUs 0 }}`), re-evaluated at commissioning and when a tag changes.
|
||||
- Tags carry a cloud-init fragment merged into deployment user-data (`deploy/tags/gpu-nvidia.yaml`
|
||||
installs the NVIDIA driver on GPU nodes) and extra overlays for stateless nodes.
|
||||
- `rollcall tag`, `rollcall tags`, `rollcall list -tag`, `deploy tag:NAME` / `power tag:NAME`
|
||||
fan-out; tags column, editor and definitions panel in the web UI; `?tag=` filter on the API.
|
||||
- Web UI redesigned for fleets: sidebar navigation (Overview, Machines, Jobs, Tags, Profiles,
|
||||
OS images), state/tag filter chips, search, sorting and paging, multi-select with bulk
|
||||
power/deploy/assign/tag/re-commission/release, machine detail with tabs, dialog-driven
|
||||
actions, fleet-wide `/api/v1/events` and `/api/v1/jobs`, `?token=` sign-in for bookmarks.
|
||||
- Physical location per machine (building, room, rack, unit, asset tag): `rollcall location`,
|
||||
CSV import matched by id/serial/MAC/node name with dry run, rack definitions with height,
|
||||
`list -rack/-room/-building`, location column and chips in the UI, Racks page with
|
||||
elevation views, bulk "Set location…", `.Node.Rack` etc. in templates and tag rules.
|
||||
- Terminal console: a bare `rollcall` in a terminal opens a full-screen UI in the llmgw style
|
||||
(Bubble Tea): overview, machines with filter/selection/bulk actions, machine detail, racks
|
||||
with elevation, jobs, events, tags, profiles, OS images; connect screen that saves
|
||||
`~/.config/rollcall/client.json`, which the CLI also reads; `rollcall tui -dump` renders one
|
||||
screen. First Go tests (console rendering and keys).
|
||||
- Network model after MAAS: fabrics, VLANs, subnets, IP ranges (dynamic/reserved/static) and
|
||||
IPAM; addresses already in use are adopted. `rollcall fabric|vlan|subnet|range|ip`.
|
||||
- Interface layouts (physical, bond, VLAN; NICs by selector) per machine or per tag; deployment
|
||||
writes cloud-init `network-config` and udev name rules into the image; release frees addresses.
|
||||
`rollcall layout`, `rollcall net show|set`.
|
||||
- Region/rack split: `rollcall rack` (dnsmasq state, per-MAC PXE files, BMC commands, image cache,
|
||||
node API proxy) long-polls the region; the region runs an embedded rack controller.
|
||||
`deploy/install-rack.sh`, `rollcall controllers`, `rollcall dhcp`.
|
||||
- dnsmasq configuration generated from the model, restarted only when it changes and rolled back
|
||||
if dnsmasq does not come back.
|
||||
- Web UI: Network page (rack controllers, subnets with ranges and usage, layouts), network tab per
|
||||
machine, layout action; console: Network page, network section, layout action.
|
||||
- Certificate authority on the region (llmgw-style): HTTPS listener with an auto-renewed region
|
||||
certificate, `admin_http` policy, `/ca.pem`; rack controllers enroll with one-time join codes
|
||||
(`rollcall controller join-code`, `rollcall rack join`) and use mutual TLS, renewing their
|
||||
certificate themselves; removal revokes; deployed machines and overlays trust the CA;
|
||||
`rollcall ca show|issue`; the console trusts a CA on first connect after showing its fingerprint.
|
||||
- Power drivers: ipmi, redfish, idrac, ilo, supermicro, xcc, openbmc, proxmox, manual, run on the
|
||||
rack controller; Redfish auto-detection after commissioning; `rollcall bmc`, BMC editing in the
|
||||
web UI and console. Lab Redfish emulator with iDRAC/iLO/generic profiles.
|
||||
- IPv6: subnets, IPAM, dual-stack default layout, per-family default routes, dnsmasq RA/DHCPv6,
|
||||
AAAA host records, IPv6 listen addresses and URLs, IPv6 in the commissioning inventory.
|
||||
- External IPAM authority (NetBox driver): allocation registers each address in NetBox first and
|
||||
skips addresses others hold there; ownership by tag + `rollcall_machine` custom field; releases
|
||||
and a periodic reconcile create, fix and delete only Rollcall-owned records; `dns_name` per address
|
||||
for DNS sync; delegated prefixes (`rollcall ipam prefixes|import`); `rollcall ipam [sync -dry]`,
|
||||
`/api/v1/ipam`.
|
||||
- Web UI: network topology drawing (region, external IPAM, rack controllers, one bus per VLAN with
|
||||
its subnets, machines wired to every VLAN they have an address on; large groups collapse) on the
|
||||
Network page and, compact, on the Overview. Overview redesigned: fleet state bar, per-rack counts,
|
||||
attention list incl. IPAM sync, capacity and free pool, OS, hardware, BMC drivers, locations, tags,
|
||||
recent deployments.
|
||||
- CA delegation: new roots allow one level of intermediates; `rollcall ca intermediate NAME -permit
|
||||
DOMAIN,CIDR [-csr FILE]` signs a name-constrained intermediate (for an ACME server such as Caddy's
|
||||
`acme_server`); `rollcall ca rotate -yes` replaces a root that cannot delegate; tag fragment
|
||||
`deploy/tags/acme-cert.yaml` fetches and renews a node certificate with certbot.
|
||||
- Rack controllers update themselves to the region's version: checksum- and version-checked
|
||||
download over mutual TLS, installed by a root path unit with automatic rollback;
|
||||
`rack_binaries` for other architectures, `auto_update` to opt out; versions in
|
||||
`rollcall controllers` and the web UI.
|
||||
- Addresses on secondary subnets with a gateway get their own routing table and a source rule in
|
||||
`network-config`, so replies from a storage or tenant VLAN leave through that VLAN's gateway.
|
||||
- With `admin_http` other than `allow`, browsers that open the web UI over HTTP are redirected to
|
||||
HTTPS (nodes keep HTTP for images and the agent).
|
||||
- Admin client certificates: `rollcall admin-cert request NAME` (key stays on the client, CSR only),
|
||||
`list`, `revoke`; the region accepts them as admin over HTTPS; CLI/console use them from
|
||||
`client.json`, the web UI works without a token when the browser presents one (`.p12` written).
|
||||
- `deploy/bootstrap.sh`: a region on a fresh Debian 13 host. Day 0 needs nothing about the network:
|
||||
packages, PostgreSQL, TFTP tree, config, services and a verified commissioning image, the region
|
||||
reachable on its management address with no DHCP or subnets. Run again with `--iface` once the
|
||||
provisioning network is known: server address updated, subnet with ranges, DHCP, optional NAT.
|
||||
- `rollcall net discover`: per interface link, speed, addresses and implied subnets, switch/port/VLAN
|
||||
from LLDP, DHCP servers that answer, neighbours, and what to do with it.
|
||||
- `commission_append` follows `server_url` when left empty.
|
||||
- `rollcall doctor`: checks configuration, database, listeners, certificates, TFTP files, image
|
||||
checksums, dnsmasq, DNS as machines see it, clock, foreign DHCP servers on the provisioning
|
||||
network (DHCPDISCOVER probe), rack controllers, OS images and IPAM, with what to do.
|
||||
- `rollcall version`; CD-built binary carries `git describe`.
|
||||
- Release notes from `CHANGELOG.md`.
|
||||
|
||||
### Fixed
|
||||
- RHEL-family deployments (Rocky, Alma, CentOS) did not trust the Rollcall CA: cloud-init's
|
||||
`ca_certs` ran `update-ca-certificates` there and failed. The CA now goes into the image's trust
|
||||
anchors and a `bootcmd` rebuilds the store with the distribution's tool.
|
||||
- A deploy whose power command was lost (rack or region restarting) stayed pending forever and the
|
||||
event looked like a success: power errors are now part of the event (`pxe boot failed`), and a
|
||||
job the machine has not started within `job_start_timeout` (default 30m, room for servers that
|
||||
take 4-5 minutes to POST) fails with a hint.
|
||||
- CentOS Stream 9 image URL (the 502 was a missing file name).
|
||||
- Rack controllers' dnsmasq had no upstream DNS on Debian (it looks for a resolvconf file that is
|
||||
usually missing) and refused every name outside the cluster domain; the generated configuration
|
||||
now forwards to systemd-resolved's stub (keeping split DNS) or the host's resolv.conf servers.
|
||||
- Virtual BMC addresses are configured in netplan on the lab PXE server (they were lost on a
|
||||
network reconfigure, which made every BMC call fail).
|
||||
- Commissioning console no longer shows a failing autologin (`noautologin`).
|
||||
|
||||
## [0.1.0] - 2026-10-27
|
||||
|
||||
First tagged release.
|
||||
|
||||
### Added
|
||||
- Commissioning: Debian 13 live image with firmware, IB and NVMe tooling; hardware inventory
|
||||
(DMI, DIMMs, CPUs, disks, NVMe, NICs, InfiniBand, GPUs, PCI, LLDP, TPM); in-band BMC account
|
||||
creation with out-of-band hints for VMs; secrets sealed with AES-256-GCM; ipmitool server-side.
|
||||
- Single static Go binary: HTTP API, embedded web UI, CLI, PostgreSQL schema.
|
||||
- Stateless (Warewulf-style) nodes: profiles, node names and IP pools, rendered overlays, dnsmasq
|
||||
export, Slurm compute image and controller sync.
|
||||
- OS deployment: image catalog with nightly upstream sync and vendor checksum verification
|
||||
(Debian 13, Ubuntu 24.04, Rocky 9, AlmaLinux 9, CentOS Stream 9), streamed `zstd | dd` writes
|
||||
with sha256 and live progress, cloud-init NoCloud seed, SELinux relabel, UEFI boot entry,
|
||||
phone-home, release/wipe jobs.
|
||||
- Gitea Actions: CI, deploy to the PXE server, website publish, release on tags.
|
||||
- Lab: Proxmox SDN subnet, `pve-vbmc` IPMI emulator for VMs.
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
# Installing Rollcall at a new site
|
||||
|
||||
This follows a site from nothing to a deployed machine. It was written from a
|
||||
clean run: a fresh Debian 13 VM became the region, discovered its interfaces,
|
||||
was given a provisioning network, and an empty machine commissioned itself and
|
||||
was deployed with Debian 13.
|
||||
|
||||
The network does not have to be decided before you start. Day 0 installs the
|
||||
region on its management address; the provisioning network is added when it
|
||||
exists.
|
||||
|
||||
## What you need
|
||||
|
||||
| | |
|
||||
|---|---|
|
||||
| Region host | Debian 13, 2 CPUs, 4 GB RAM, 30 GB disk (plus ~1 GB per OS image), root |
|
||||
| Network access | a default route; the internet or a mirror for packages and cloud images |
|
||||
| Later: provisioning network | an interface of this host (or of a rack controller) on the VLAN where machines PXE-boot |
|
||||
| Later: BMC network | the rack controller must reach the machines' BMCs (IPMI, Redfish) |
|
||||
| Ports | DHCP 67/udp and TFTP 69/udp on the provisioning network; HTTP 80 for machines; HTTPS 443 for people and rack controllers |
|
||||
|
||||
The release bundle `rollcall-<version>-linux-amd64.tar.gz` holds the binary,
|
||||
`deploy/` and `agent/`. From a checkout, `deploy/bootstrap.sh` builds the binary
|
||||
itself when Go is installed.
|
||||
|
||||
## Day 0: install
|
||||
|
||||
```sh
|
||||
tar xzf rollcall-v0.2.0-linux-amd64.tar.gz && cd rollcall-v0.2.0-linux-amd64
|
||||
sudo ./deploy/bootstrap.sh --image-url https://gitea.example/OWNER/rollcall/releases/download/v0.2.0
|
||||
```
|
||||
|
||||
- **Packages, database, TFTP.** PostgreSQL with its own role and a random password,
|
||||
dnsmasq, the BIOS and UEFI syslinux files, lldpd.
|
||||
- **Configuration.** `/etc/rollcall/config.json` with generated tokens. The admin
|
||||
API only answers over HTTPS from other hosts (`admin_http: loopback`).
|
||||
- **Commissioning image.** Downloaded and checked against `SHA256SUMS`. For a
|
||||
private repository set `IMAGE_TOKEN`. Without a release to download from,
|
||||
`--image-url http://EXISTING-REGION/rollcall` copies it from another region,
|
||||
and `--build-image` builds it here with live-build (about 20 minutes).
|
||||
- **SSH keys.** The keys of the user running sudo go to root on deployed
|
||||
machines; add more with `--ssh-key KEY_OR_FILE`.
|
||||
|
||||
At the end the region answers on its management address: the web UI at
|
||||
`https://ADDRESS`, no DHCP, no subnets. Check it:
|
||||
|
||||
```sh
|
||||
sudo rollcall doctor
|
||||
```
|
||||
|
||||
On day 0 the only warning should be that the host serves no DHCP yet.
|
||||
|
||||
## Find the provisioning network
|
||||
|
||||
```sh
|
||||
sudo rollcall net discover
|
||||
```
|
||||
|
||||
```
|
||||
INTERFACE KIND LINK SPEED ADDRESSES SWITCH / PORT / VLAN DHCP SERVERS NEIGH
|
||||
eth0* physical up - 10.0.0.116/24 - 10.0.0.1 4
|
||||
eth1 physical up - 10.214.0.1/24 - none 0
|
||||
|
||||
eth0 uplink: management network with its own DHCP (10.0.0.1); keep it for the region's access
|
||||
eth1 candidate provisioning network 10.214.0.0/24: bootstrap.sh --iface eth1
|
||||
```
|
||||
|
||||
For each interface: link and speed, addresses and the subnet they imply, the
|
||||
switch port and VLAN that LLDP reports, and DHCP servers that answer a probe.
|
||||
The last column matters most: a DHCP server already on the provisioning VLAN
|
||||
competes with Rollcall for the machines' PXE boot.
|
||||
|
||||
| What discover shows | What to do |
|
||||
|---|---|
|
||||
| An interface with an address and no DHCP | the provisioning network: next step |
|
||||
| Link up, no address, LLDP shows the VLAN | give the interface its address on that VLAN first |
|
||||
| Another DHCP server on the provisioning VLAN | the network team turns it off for that VLAN, or keeps it and points it at this host (next-server, or a DHCP relay), then `--no-dhcp` |
|
||||
| No link | cable it |
|
||||
|
||||
## Add the provisioning network
|
||||
|
||||
```sh
|
||||
sudo ./deploy/bootstrap.sh --iface eth1 --os debian-13
|
||||
```
|
||||
|
||||
- **Server address.** The host's address on `eth1` becomes the server address
|
||||
machines use. The management address stays in the certificate, so the web UI
|
||||
keeps working there.
|
||||
- **Subnet.** Modelled with ranges: reserved `.1-.20`, static `.50-.99` (where
|
||||
deployed machines get their addresses), dynamic `.100-.200` (PXE and commissioning).
|
||||
- **DHCP and PXE** on `eth1`, unless `--no-dhcp`.
|
||||
- **Routing.** Without `--gateway`, this host routes for the machines: forwarding
|
||||
plus masquerade out of the uplink. Give `--gateway IP` when the network has a router.
|
||||
- **First OS image.** `--os debian-13` imports it now; a nightly timer keeps the
|
||||
catalog current.
|
||||
|
||||
The same can be done in the web UI (Network, New subnet) or the CLI
|
||||
(`rollcall subnet add`, `range add`, `vlan add ... -dhcp`), or imported from
|
||||
NetBox (`rollcall ipam import`). Several provisioning VLANs, bonds and VLAN
|
||||
layouts per machine are described in the README, under Network.
|
||||
|
||||
`rollcall doctor` should now report no failures. It also sends a DHCP
|
||||
DISCOVER on the provisioning interface and fails if any other server answers.
|
||||
|
||||
## First machines
|
||||
|
||||
Set a machine to PXE-boot on the provisioning network and power it on. An
|
||||
unknown machine commissions itself: it boots the live image, reports its
|
||||
hardware, and appears as `ready`.
|
||||
|
||||
```sh
|
||||
sudo rollcall list
|
||||
sudo rollcall deploy MACHINE debian-13 -hostname node01 -boot -watch
|
||||
```
|
||||
|
||||
`-boot` power-cycles the machine through its BMC. Set the BMC first if
|
||||
commissioning did not find it (`rollcall bmc MACHINE ...`, or the machine's page
|
||||
in the web UI). Without a BMC, power the machine on by hand after `deploy`.
|
||||
|
||||
A deployed machine gets the address from the static range, the domain, a route,
|
||||
DNS through the rack controller, root SSH keys and trust in the Rollcall CA.
|
||||
|
||||
## Admin access from your laptop
|
||||
|
||||
```sh
|
||||
# on the laptop, once, with the token (sudo jq -r .admin_token /etc/rollcall/config.json on the region)
|
||||
ROLLCALL_URL=https://REGION ROLLCALL_TOKEN=... ROLLCALL_CA=ca.pem rollcall admin-cert request mylaptop
|
||||
```
|
||||
|
||||
The key is made on the laptop; only a certificate request travels. The CLI then
|
||||
works without the token. The `.p12` it writes goes into the browser's keychain,
|
||||
and the web UI signs in by itself. Trust the CA (`/ca.pem`) in the browser once.
|
||||
|
||||
## More segments: rack controllers
|
||||
|
||||
A provisioning VLAN the region has no interface on gets its own rack controller:
|
||||
|
||||
```sh
|
||||
# on the region
|
||||
sudo rollcall controller join-code -name hall-b -addr REGION:443
|
||||
# on the rack host (Debian 13, an interface on that VLAN), from the bundle
|
||||
sudo ./rollcall rack join CODE -url http://RACK-ADDRESS-ON-THAT-VLAN
|
||||
sudo sh deploy/install-rack.sh
|
||||
```
|
||||
|
||||
Rack controllers talk to the region over mutual TLS and update themselves to the
|
||||
region's version.
|
||||
|
||||
## Integrations
|
||||
|
||||
- **NetBox (or Infoblox) as the address authority.** `ipam` in `config.json`: see README, External IPAM.
|
||||
- **Certificates for machines.** Delegate a name-constrained intermediate to an ACME
|
||||
server: see README, Certificate authority.
|
||||
|
||||
## Upgrades
|
||||
|
||||
Replace the binary on the region (or let CD do it) and restart `rollcall`. Rack
|
||||
controllers follow by themselves. `rollcall doctor` afterwards.
|
||||
@@ -44,7 +44,7 @@ unknown server ──PXE──▶ dnsmasq (DHCP+TFTP) ──▶ pxelinux default
|
||||
|---|---|
|
||||
| `cmd/rollcall/` | Go server + CLI + embedded web UI (`web/`) |
|
||||
| `agent/` | `rollcall-agent.py` (runs inside the live image), its systemd unit, `build-image.sh` (live-build recipe) |
|
||||
| `deploy/` | `install.sh`, `rollcall.service`, `config.example.json`, `pxelinux-default`, `os-sources.json`, sync/dnsmasq/controller units, `overlays/` |
|
||||
| `deploy/` | `install.sh` (region), `install-rack.sh` + `rack.example.json` (rack controller), units, `config.example.json`, `os-sources.json`, `tags/`, `overlays/` |
|
||||
| `images/` | stateless root image recipes (`build-rootfs.sh`, `compute-debian13/`) |
|
||||
| `lab/` | Proxmox test lab: `pve-vbmc` (virtual IPMI BMC for VMs backed by the Proxmox API), cloud-init for the PXE server VM |
|
||||
| `site/` | Project website (static) |
|
||||
@@ -52,6 +52,8 @@ unknown server ──PXE──▶ dnsmasq (DHCP+TFTP) ──▶ pxelinux default
|
||||
|
||||
## Server
|
||||
|
||||
New site: **[INSTALL.md](INSTALL.md)** (`deploy/bootstrap.sh`, from day 0 without a network plan to the first deployed machine). The steps below are the manual equivalent.
|
||||
|
||||
```sh
|
||||
# build
|
||||
cd cmd/rollcall && CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o rollcall .
|
||||
@@ -72,6 +74,30 @@ cd deploy && sudo ./install.sh
|
||||
command line baked into generated PXE files), `bmc_hints` (out-of-band BMC
|
||||
mapping for machines without in-band IPMI, e.g. VMs).
|
||||
|
||||
### Console (same binary)
|
||||
|
||||
A bare `rollcall` in a terminal opens a full-screen console, built like the
|
||||
llmgw one (Bubble Tea): the fleet down the left (Overview, Machines, Racks,
|
||||
Jobs, Events) and provisioning (Tags, Profiles, OS images), the chosen page on
|
||||
the right, keys along the bottom. It is keyboard only, so it works on serial
|
||||
consoles and BMC KVMs.
|
||||
|
||||
* Machines: `/` filter (name, serial, MAC, IP, rack, tag), `s` cycles the state
|
||||
filter, `Space` selects, `*` selects everything shown, `Enter` opens a
|
||||
machine, `a` opens the actions for the selection or the current row: power,
|
||||
boot device, deploy OS, assign/unassign profile, tags, location,
|
||||
re-commission, release, cancel job, delete. Destructive ones ask first.
|
||||
* Racks: `Enter` shows the elevation; `Enter` on a unit opens that machine.
|
||||
* Jobs and events open the machine they belong to; tags, profiles and OS
|
||||
images show their definition. `r` refreshes (it also refreshes on its own,
|
||||
every 3 s while a job runs), `?` lists the keys, `q` quits.
|
||||
|
||||
On the server as root it reads `/etc/rollcall/config.json`. Anywhere else it
|
||||
asks for the server URL and admin token once and keeps them in
|
||||
`~/.config/rollcall/client.json` (mode 0600), which the CLI then uses as well.
|
||||
In a pipe or a script a bare `rollcall` prints its help as before.
|
||||
`rollcall tui -dump 140x40 -page machines -plain` renders one screen to stdout.
|
||||
|
||||
### CLI (same binary)
|
||||
|
||||
```sh
|
||||
@@ -225,6 +251,317 @@ up for ssh). A deployed machine's PXE entry defaults to local disk, so boot orde
|
||||
can stay "network first" on BIOS hosts; on UEFI hosts the agent adds a boot entry
|
||||
for the installed loader. `rollcall recommission` or `release` take it back.
|
||||
|
||||
## Tags
|
||||
|
||||
Tags single out machines for special treatment, as in MAAS. A tag has a name
|
||||
and optionally a **match rule**, a **cloud-config fragment** and **overlays**.
|
||||
|
||||
```sh
|
||||
rollcall tags add gpu -match '{{ gt .Node.GPUs 0 }}' -cloud-config deploy/tags/gpu-nvidia.yaml -comment "NVIDIA nodes"
|
||||
rollcall tags add rack-3 -comment "rack 3, row B" # manual only
|
||||
rollcall tag m-2ef1dab828c8 +rack-3 +burn-in -staging # manual tags on one machine
|
||||
rollcall list -tag gpu
|
||||
rollcall deploy tag:gpu rocky-9 -boot # every gpu machine
|
||||
rollcall power tag:rack-3 off
|
||||
```
|
||||
|
||||
* **Auto tags** come from the match rule, a Go template over `.Node` (GPUs,
|
||||
IB GUIDs, CPUs, memory, disks, NVMe, PCI vendor ids, manufacturer, product,
|
||||
OS, existing tags). They are recomputed at every commissioning and whenever a
|
||||
tag definition is saved; manual tags are never touched by rules.
|
||||
* **At deployment** the fragments of the machine's tags are appended to the
|
||||
generated user-data as MIME parts (`packages`, `runcmd`, `write_files` merge
|
||||
additively; a `#!` fragment runs as a script). `deploy/tags/gpu-nvidia.yaml`
|
||||
installs the NVIDIA driver on Debian, Ubuntu and the RHEL family and reboots.
|
||||
* **For stateless nodes** a tag's overlays are applied after the profile's, so a
|
||||
`gpu` tag can add `gres.conf` lines or driver modules without a second profile.
|
||||
|
||||
## Physical location
|
||||
|
||||
Every machine can carry building / room / rack / rack unit and an asset tag.
|
||||
|
||||
```sh
|
||||
rollcall location m-2ef1dab828c8 -building DC1 -room "Hall A" -rack R12 -unit 24 -asset A-0042
|
||||
rollcall location tag:gpu -building DC1 -room "Hall B" # bulk by tag
|
||||
rollcall location import racks.csv -dry # from a DCIM / asset export
|
||||
rollcall location racks | rack R12 -building DC1 -room "Hall A" -height 48
|
||||
rollcall list -rack R12
|
||||
```
|
||||
|
||||
The CSV needs a header; rows are matched by `id`, `serial`, `mac` or `node_name`
|
||||
and may carry `building, room, rack, unit, asset_tag`. Racks are created on
|
||||
first use (42U) and can be resized. The web UI shows a location column with
|
||||
building › room › rack chips, a **Racks** page with elevation views (machines
|
||||
drawn at their U, coloured by state) and bulk "Set location…". Templates and
|
||||
tag rules see `.Node.Building`, `.Node.Room`, `.Node.Rack`, `.Node.Unit`, so a
|
||||
rack can drive a tag (`{{ eq .Node.Rack "R12" }}`) or a Slurm `topology.conf`
|
||||
overlay.
|
||||
|
||||
## Network and rack controllers
|
||||
|
||||
### Model
|
||||
|
||||
As in MAAS: **fabric → VLAN → subnet → IP ranges**.
|
||||
|
||||
* A **fabric** is a switched layer-2 domain (`ethernet` or `infiniband`).
|
||||
* A **VLAN** has a VID (0 = untagged; a partition key on IB), an MTU, and
|
||||
says whether rollcall serves DHCP/PXE on it and which **rack controller**
|
||||
does (a second one splits the dynamic pool with it).
|
||||
* A **subnet** has a purpose (provisioning, mgmt, bmc, storage, ib, other), a
|
||||
gateway, DNS servers and a domain.
|
||||
* **Ranges** inside a subnet are `dynamic` (the DHCP pool PXE and
|
||||
commissioning use), `reserved` (never handed out: routers, switches,
|
||||
controllers) and `static` (where machines get their addresses; without one,
|
||||
anything outside dynamic/reserved).
|
||||
* **IPAM** records every address handed out (`rollcall ip list`), pinned ones
|
||||
(`rollcall net set ID -static ens18=10.0.0.60`) and reserved single
|
||||
addresses (`rollcall ip reserve 10.0.0.5 -name switch-1`). Addresses a machine
|
||||
already had are adopted when its subnet is defined.
|
||||
|
||||
```sh
|
||||
rollcall fabric add hall-a
|
||||
rollcall subnet add 10.50.0.0/24 -fabric hall-a -vid 0 -purpose provisioning -gateway 10.50.0.1 -dns 10.50.0.1 -domain dc.example
|
||||
rollcall range add 10.50.0.0/24 reserved 10.50.0.1 10.50.0.20
|
||||
rollcall range add 10.50.0.0/24 static 10.50.0.50 10.50.0.199
|
||||
rollcall range add 10.50.0.0/24 dynamic 10.50.0.200 10.50.0.250
|
||||
rollcall vlan add hall-a 0 -dhcp -rack hall-a-rack1 # this rack controller serves DHCP/PXE here
|
||||
rollcall subnet add 10.60.0.0/24 -fabric hall-a -vid 200 -purpose storage
|
||||
rollcall dhcp hall-a-rack1 # the dnsmasq configuration it generates
|
||||
```
|
||||
|
||||
### Machine interfaces
|
||||
|
||||
A **layout** describes interfaces in terms that fit many machines: NICs are
|
||||
chosen by selectors (`boot`, `name:X`, `mac:X`, `fastest:N`, `ib:N`, `all`),
|
||||
not by MAC. Interfaces are `physical`, `bond` (802.3ad by default) or `vlan`;
|
||||
each link names a subnet (CIDR, name, or `boot`) and a mode: `auto` (an address
|
||||
from the subnet), `static` (a given one), `dhcp` or `link_up`.
|
||||
|
||||
```json
|
||||
{"interfaces": [
|
||||
{"type": "physical", "select": "boot", "links": [{"subnet": "boot", "mode": "auto", "default_gateway": true}]},
|
||||
{"type": "bond", "name": "bond0", "members": ["fastest:2"], "mtu": 9000, "links": [{"mode": "link_up"}]},
|
||||
{"type": "vlan", "parent": "bond0", "vid": 200, "links": [{"subnet": "10.60.0.0/24", "mode": "auto"}]},
|
||||
{"type": "physical", "select": "ib:0", "links": [{"subnet": "ib0", "mode": "auto"}]}
|
||||
]}
|
||||
```
|
||||
|
||||
`rollcall layout add gpu-bond layout.json`, then either per machine
|
||||
(`rollcall net set ID -layout gpu-bond`) or per tag (`rollcall tags add gpu
|
||||
… -layout gpu-bond`). Without a layout the boot NIC gets an `auto` address on
|
||||
its subnet; without any subnet model, deployed machines use DHCP as before.
|
||||
|
||||
At deployment the layout is resolved against the machine's inventory,
|
||||
addresses are allocated, and two files go into the image next to the
|
||||
cloud-init seed: `network-config` (cloud-init v2; cloud-init renders it for
|
||||
netplan, NetworkManager or ifupdown) and a udev rule that pins each NIC's name
|
||||
to its MAC, because RHEL-family cloud images boot with `net.ifnames=0`.
|
||||
Release returns the allocated addresses. `rollcall net show ID` previews it all.
|
||||
|
||||
Verified in the lab: Debian 13 and Ubuntu 24.04 (netplan) and Rocky 9
|
||||
(NetworkManager), with a static boot NIC and with an active-backup bond
|
||||
carrying VLAN 200.
|
||||
|
||||
### Region and rack controllers
|
||||
|
||||
`rollcall serve` is the **region**: database, API, web UI, IPAM, image store.
|
||||
**Rack controllers** sit next to the machines, one or two per network segment
|
||||
(hall, building, site):
|
||||
|
||||
* they serve DHCP and TFTP with dnsmasq on the VLANs assigned to them; the
|
||||
configuration, per-MAC PXE files and host files are computed by the region;
|
||||
* they run BMC commands (BMC networks are rarely routed to the region);
|
||||
* they serve boot and OS images from a local cache, filled from the region on
|
||||
first use (kernels and initrds are prefetched, TFTP cannot wait);
|
||||
* they pass the node API (commissioning, deploy jobs, overlays, phone-home)
|
||||
through to the region and tell it which rack a machine came through.
|
||||
|
||||
A rack controller only needs to reach the region: it long-polls its desired
|
||||
state and a command queue. The region host runs an **embedded** rack
|
||||
controller (`rack_name`, default its hostname), which is all a single-server
|
||||
installation needs. Remote ones:
|
||||
|
||||
```sh
|
||||
rollcall controller join-code -name hall-b -addr 10.0.0.234:443 # on the region: one word, one use, 60 min
|
||||
# on the rack host (Debian/Ubuntu), from deploy/ with the binary next to it:
|
||||
./rollcall rack join rollcall-join1-... -url http://10.213.78.1 # writes /etc/rollcall/rack.json + certificate
|
||||
sh install-rack.sh # dnsmasq, syslinux, cache dirs, rollcall-rack.service
|
||||
rollcall controllers # on the region: it shows up online
|
||||
rollcall vlan add hall-b 0 -dhcp -rack hall-b
|
||||
```
|
||||
|
||||
dnsmasq is restarted only when its generated configuration changes, and only
|
||||
if it comes back; otherwise the last working configuration is restored (a
|
||||
hand-written DHCP configuration for the same interface is the usual cause:
|
||||
`install.sh` points it out). A machine that moved to another segment gets
|
||||
`rollcall net set ID -rack NAME`.
|
||||
|
||||
The lab runs both: `pxe-server` (embedded, 10.213.77.0/24) and `pxe-rack2`
|
||||
(VM 213, remote, 10.213.78.0/24 on SDN vnet `pxenet2`), with node002 behind
|
||||
pxe-rack2 and its virtual BMC on pxe-rack2. Rack ↔ region traffic is mutual
|
||||
TLS (next section), so the BMC passwords of queued commands are encrypted.
|
||||
|
||||
## Certificate authority and TLS
|
||||
|
||||
The region is the installation's certificate authority, as the llmgw control
|
||||
plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and
|
||||
`ca-key.pem` (ECDSA P-256, ten years), created on first start.
|
||||
|
||||
* **HTTPS.** The region serves API, UI and images on `tls_listen` (default
|
||||
`:443`) with a certificate for every name and address it has plus
|
||||
`tls_names`; it is reissued when they change or a month before it ends.
|
||||
`admin_http` decides what plain HTTP may still do with the admin API:
|
||||
`allow` (default, for compatibility), `loopback` (the CLI on the region
|
||||
host) or `deny`. Nodes keep booting over HTTP: firmware and the live image
|
||||
cannot check a certificate before they have one.
|
||||
* **Rack controllers** join with a code from `rollcall controller join-code`:
|
||||
one word carrying the first half of the CA's SHA-256, a one-time secret, an
|
||||
expiry and the region's address. `rollcall rack join` checks the region
|
||||
against the fingerprint, sends a CSR with the secret and gets a
|
||||
client+server certificate (`rack:<name>`, a year). From then on every rack
|
||||
call is mutual TLS and the region identifies the rack by its certificate;
|
||||
`rollcall controller rm` revokes it at once. Racks renew a month before
|
||||
expiry. The old `rack_secret` is still accepted when configured.
|
||||
* **Updates.** Racks follow the region's version: the region names the
|
||||
binary (version, architecture, SHA-256) in each rack's state and serves it
|
||||
over the rack's mutual-TLS channel. The rack stages it in
|
||||
`/var/lib/rollcall/update` after checking the checksum and that it reports
|
||||
the expected version; `rollcall-rack-update.path` (root) installs it,
|
||||
restarts the rack and puts the previous binary back if the rack does not
|
||||
register again within a minute (`last-result` says what happened). Racks on
|
||||
another architecture get `rack_binaries/rollcall-linux-<arch>`;
|
||||
`"auto_update": false` in `rack.json` opts out. `rollcall controllers`
|
||||
shows each rack's version.
|
||||
* **Admin certificates.** Instead of the admin token, a person or laptop can
|
||||
hold a client certificate: `rollcall admin-cert request NAME` (with the
|
||||
token once) makes a key locally, sends only a CSR and gets
|
||||
`admin:NAME` back; the CLI and console use it from `client.json`, and the
|
||||
`.p12` it writes goes into the browser's keychain, after which the web UI
|
||||
signs in by itself. `rollcall admin-cert list | revoke NAME` (revocation
|
||||
takes effect within 30 seconds). Certificates are checked against the CA,
|
||||
the client-auth usage and the serials the region recorded.
|
||||
* **Clients.** On the region host the CLI reads the CA from `ca_dir`;
|
||||
elsewhere `ROLLCALL_CA=ca.pem`, or the console shows the CA fingerprint on
|
||||
first connect and keeps the CA in `~/.config/rollcall/client.json` once you
|
||||
confirm it (`rollcall ca show` prints it on the region). `/ca.pem` serves
|
||||
the certificate; the web login links to it.
|
||||
* **Machines** trust it: deployments write the CA into the trust anchors of
|
||||
both families (`/usr/local/share/ca-certificates`, `/etc/pki/ca-trust/source/anchors`)
|
||||
and rebuild the store at every boot (`update-ca-certificates` or
|
||||
`update-ca-trust`; cloud-init's `ca_certs` fails on RHEL-family systems);
|
||||
stateless overlays get it as `.CA`. `rollcall ca issue NAME
|
||||
-host h1,h2 [-client]` issues leaves for anything else (Slurm REST, a
|
||||
registry, a web front end).
|
||||
* **Delegation.** The root may sign one level of intermediate CAs, each
|
||||
limited by name constraints to the domains and address ranges it serves:
|
||||
`rollcall ca intermediate caddy -permit int.example.com,10.213.0.0/16
|
||||
-csr caddy.csr` signs a CSR (the key stays on the host that made it). An ACME
|
||||
server under that intermediate (Caddy's `acme_server`, step-ca) then issues
|
||||
short-lived certificates to nodes with certbot or acme.sh; everything that
|
||||
trusts the Rollcall CA trusts them, and nothing outside the permitted names
|
||||
can be issued. Roots created before this (path length 0) cannot delegate:
|
||||
`rollcall ca rotate -yes` replaces the root, keeping the old one as a
|
||||
backup; rack controllers then join again and clients trust the new
|
||||
fingerprint. `deploy/tags/acme-cert.yaml` is a tag that fetches a node
|
||||
certificate at first boot and renews it.
|
||||
|
||||
## Power drivers
|
||||
|
||||
Every machine's BMC record names a driver; commands run on the rack controller
|
||||
that reaches the BMC network.
|
||||
|
||||
| Driver | For |
|
||||
|---|---|
|
||||
| `ipmi` | IPMI 2.0 over LAN (ipmitool lanplus), any vendor; settings `cipher` (17 on newer BMCs), `privilege`, `interface` |
|
||||
| `redfish` | DMTF Redfish, any vendor: system discovered, ResetType chosen from what the BMC allows, boot override once (with If-Match where the BMC wants it) |
|
||||
| `idrac` | Dell iDRAC 8/9 (Redfish, `System.Embedded.1`) |
|
||||
| `ilo` | HPE iLO 4 (2.30+), 5, 6 (Redfish; no PowerCycle, so ForceRestart; PushPowerButton for soft off on older firmware) |
|
||||
| `supermicro`, `xcc`, `openbmc` | Supermicro, Lenovo XClarity, OpenBMC (Redfish) |
|
||||
| `proxmox` | Proxmox VE VMs: API token (user = token id, password = secret), settings `node`, `vmid` |
|
||||
| `manual` | no power control; actions say so |
|
||||
|
||||
After commissioning, a BMC still on IPMI is asked whether it speaks Redfish
|
||||
with the same account; if it does, the driver for its vendor takes over
|
||||
(event `bmc driver`). An operator's choice is never overridden.
|
||||
|
||||
```sh
|
||||
rollcall bmc drivers
|
||||
rollcall bmc m-1234 -driver idrac -address 10.0.5.21 -user root -password - -verify < pw
|
||||
rollcall bmc m-1234 -driver redfish -set system=/redfish/v1/Systems/1 -set verify_tls=true
|
||||
rollcall bmc m-1234 -probe # detect Redfish again
|
||||
```
|
||||
|
||||
The lab has `lab/pve-vbmc/pve_redfish.py`, a Redfish service per Proxmox VM
|
||||
with iDRAC, iLO and generic profiles, next to the IPMI emulator.
|
||||
|
||||
## IPv6
|
||||
|
||||
Subnets can be IPv6 (`rollcall subnet add fd00:50::/64 ...`). IPAM hands out
|
||||
addresses from static ranges (walked from their start), deployments get them
|
||||
in `network-config` with a separate default route per family, and the default
|
||||
layout links every subnet of the boot NIC's VLAN, so an IPv6 prefix next to
|
||||
the IPv4 one makes machines dual-stack. Interfaces with planned addresses
|
||||
ignore router advertisements. dnsmasq serves router advertisements and
|
||||
DHCPv6 on IPv6 subnets with dhcp on; the rack offers itself as default router
|
||||
only when it is the subnet's gateway. Host files carry AAAA records, rack
|
||||
controllers report their IPv6 addresses, the region certificate covers them,
|
||||
and region, racks and clients work over IPv6 (`https://[fd00::1]`).
|
||||
|
||||
PXE stays IPv4: syslinux cannot netboot over IPv6, so the provisioning VLAN
|
||||
needs an IPv4 subnet for commissioning and deployment.
|
||||
|
||||
Verified in the lab: Debian 13 (with a bond and VLAN), Rocky 9 and AlmaLinux 9
|
||||
dual-stack on two segments, AAAA lookups through dnsmasq, the admin CLI over
|
||||
`https://[fdc3:...]`.
|
||||
|
||||
## External IPAM (NetBox; Infoblox-style)
|
||||
|
||||
Rollcall works on its own, but at a site with an established network plan the
|
||||
address authority is somewhere else. With `ipam` in `config.json` every
|
||||
address Rollcall hands out is registered there first:
|
||||
|
||||
```json
|
||||
"ipam": {
|
||||
"provider": "netbox",
|
||||
"url": "https://netbox.int.example.com",
|
||||
"token_file": "/etc/rollcall/netbox.token",
|
||||
"ca_file": "/etc/rollcall/netbox-ca.pem",
|
||||
"dns_zone": "int.example.com",
|
||||
"sync_interval": "5m"
|
||||
}
|
||||
```
|
||||
|
||||
- **Allocation asks the authority.** An address that exists in NetBox and is
|
||||
not Rollcall's is skipped and the next candidate is tried, so Rollcall never
|
||||
collides with a printer, a VIP or another team's reservation. If NetBox is
|
||||
unreachable, allocation fails instead of guessing.
|
||||
- **Ownership is explicit.** Rollcall's records carry the tag `rollcall` and the
|
||||
machine ID in the custom field `rollcall_machine` (create both in NetBox;
|
||||
on Infoblox these are an extensible attribute). Records without both are
|
||||
never changed or deleted, only reported.
|
||||
- **Releases reconcile.** Releasing, unassigning or deleting a machine frees
|
||||
addresses locally and triggers a reconcile, which also runs every
|
||||
`sync_interval`: missing records are created, names and owners fixed, and
|
||||
Rollcall-owned records that are no longer in use deleted.
|
||||
- **DNS follows the record.** `dns_name` is `<host>.<zone>` for a machine's
|
||||
provisioning/mgmt address and `<host>-<iface>.<zone>` elsewhere (a subnet's
|
||||
`domain` replaces the zone when it lies inside it; `pxe.local` stays local). A NetBox-to-DNS sync (PowerDNS here, a host
|
||||
record on Infoblox) publishes it; Rollcall writes one object per address.
|
||||
- **Prefixes are delegated in NetBox.** Tag a prefix `rollcall`;
|
||||
`rollcall ipam prefixes` lists them and `rollcall ipam import` creates the
|
||||
missing subnets (VLAN and rack are then set as usual).
|
||||
|
||||
```
|
||||
rollcall ipam # provider, last reconcile
|
||||
rollcall ipam sync -dry # what would change
|
||||
rollcall ipam sync
|
||||
rollcall ipam prefixes | import
|
||||
```
|
||||
|
||||
The provider is an interface (reserve, owned records, update, delete,
|
||||
delegated prefixes); the NetBox driver is the first. An Infoblox WAPI driver
|
||||
maps them to `record:host`/`fixedaddress` with extensible attributes.
|
||||
|
||||
## Lab
|
||||
|
||||
`lab/` documents a Proxmox SDN test setup: an isolated vnet, a PXE server VM
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -262,6 +262,8 @@ def network():
|
||||
"numa_node": readf(f"{sysd}/device/numa_node") or None,
|
||||
"ipv4": [a["local"] + "/" + str(a["prefixlen"]) for a in addrs.get(ifn, {}).get("addr_info", [])
|
||||
if a.get("family") == "inet"],
|
||||
"ipv6": [a["local"] + "/" + str(a["prefixlen"]) for a in addrs.get(ifn, {}).get("addr_info", [])
|
||||
if a.get("family") == "inet6" and a.get("scope") == "global"],
|
||||
}
|
||||
if l.get("link_type") == "ether":
|
||||
nic.update(ethtool_info(ifn))
|
||||
|
||||
@@ -0,0 +1,219 @@
|
||||
package main
|
||||
|
||||
// Admin client certificates: an alternative to the admin token. A browser or
|
||||
// CLI that presents a certificate from the installation's CA with the name
|
||||
// "admin:<name>", made for client authentication and with a serial the region
|
||||
// recorded (and not revoked), is an admin. The key is made on the client; only
|
||||
// a CSR travels. Issuing needs admin rights once (the token, or another admin
|
||||
// certificate); revoking takes effect within the cache period (30 s).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
const adminCertSchema = `
|
||||
CREATE TABLE IF NOT EXISTS admin_certs (
|
||||
serial text PRIMARY KEY,
|
||||
name text NOT NULL,
|
||||
expires timestamptz NOT NULL,
|
||||
created timestamptz NOT NULL DEFAULT now(),
|
||||
issued_by text,
|
||||
revoked timestamptz
|
||||
);
|
||||
`
|
||||
|
||||
type AdminCert struct {
|
||||
Serial string `json:"serial"`
|
||||
Name string `json:"name"`
|
||||
Expires time.Time `json:"expires"`
|
||||
Created time.Time `json:"created"`
|
||||
IssuedBy string `json:"issued_by"`
|
||||
Revoked *time.Time `json:"revoked,omitempty"`
|
||||
}
|
||||
|
||||
func (s *Store) AddAdminCert(ctx context.Context, c AdminCert) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO admin_certs(serial, name, expires, issued_by) VALUES ($1,$2,$3,$4)`, c.Serial, c.Name, c.Expires, c.IssuedBy)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) AdminCerts(ctx context.Context) ([]AdminCert, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT serial, name, expires, created, coalesce(issued_by,''), revoked FROM admin_certs ORDER BY created`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []AdminCert{}
|
||||
for rows.Next() {
|
||||
var c AdminCert
|
||||
if err := rows.Scan(&c.Serial, &c.Name, &c.Expires, &c.Created, &c.IssuedBy, &c.Revoked); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RevokeAdminCerts revokes by serial or by name (all of that name); returns how many.
|
||||
func (s *Store) RevokeAdminCerts(ctx context.Context, ref string) (int64, error) {
|
||||
tag, err := s.pool.Exec(ctx, `UPDATE admin_certs SET revoked=now() WHERE revoked IS NULL AND (serial=$1 OR name=$1)`, strings.ToLower(ref))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return tag.RowsAffected(), nil
|
||||
}
|
||||
|
||||
func (s *Store) validAdminSerials(ctx context.Context) (map[string]string, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT serial, name FROM admin_certs WHERE revoked IS NULL AND expires > now()`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]string{}
|
||||
for rows.Next() {
|
||||
var serial, name string
|
||||
if err := rows.Scan(&serial, &name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[serial] = name
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
type adminCertCache struct {
|
||||
mu sync.Mutex
|
||||
at time.Time
|
||||
serial map[string]string
|
||||
}
|
||||
|
||||
func (a *App) adminCertsCached(ctx context.Context) map[string]string {
|
||||
a.adminCC.mu.Lock()
|
||||
defer a.adminCC.mu.Unlock()
|
||||
if a.adminCC.serial != nil && time.Since(a.adminCC.at) < 30*time.Second || a.store == nil {
|
||||
return a.adminCC.serial
|
||||
}
|
||||
m, err := a.store.validAdminSerials(ctx)
|
||||
if err != nil {
|
||||
log.Printf("admin certificates: %v", err)
|
||||
return a.adminCC.serial // keep the last good view on a database hiccup
|
||||
}
|
||||
a.adminCC.serial, a.adminCC.at = m, time.Now()
|
||||
return m
|
||||
}
|
||||
|
||||
func (a *App) dropAdminCertCache() {
|
||||
a.adminCC.mu.Lock()
|
||||
a.adminCC.serial = nil
|
||||
a.adminCC.mu.Unlock()
|
||||
}
|
||||
|
||||
// peerAdmin is the admin name of a verified client certificate, "" otherwise.
|
||||
func (a *App) peerAdmin(r *http.Request) string {
|
||||
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 || len(r.TLS.PeerCertificates) == 0 {
|
||||
return ""
|
||||
}
|
||||
c := r.TLS.PeerCertificates[0]
|
||||
name, ok := strings.CutPrefix(c.Subject.CommonName, "admin:")
|
||||
if !ok || name == "" {
|
||||
return ""
|
||||
}
|
||||
clientAuth := false
|
||||
for _, u := range c.ExtKeyUsage {
|
||||
clientAuth = clientAuth || u == x509.ExtKeyUsageClientAuth
|
||||
}
|
||||
if !clientAuth {
|
||||
return ""
|
||||
}
|
||||
if a.adminCertsCached(r.Context())[c.SerialNumber.Text(16)] != name {
|
||||
return ""
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
// who describes the admin making a request, for the log.
|
||||
func (a *App) who(r *http.Request) string {
|
||||
if n := a.peerAdmin(r); n != "" {
|
||||
return "admin:" + n
|
||||
}
|
||||
return "token from " + r.RemoteAddr
|
||||
}
|
||||
|
||||
var adminNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,62}$`)
|
||||
|
||||
func (a *App) handleAdminCerts(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
cs, err := a.store.AdminCerts(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, cs)
|
||||
case http.MethodPost:
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
CSR string `json:"csr"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
req.Name = strings.ToLower(strings.TrimSpace(req.Name))
|
||||
if !adminNameRe.MatchString(req.Name) {
|
||||
fail(w, 400, errors.New("name: lower-case letters, digits, . _ - (e.g. the laptop's or person's name)"))
|
||||
return
|
||||
}
|
||||
blk, _ := pem.Decode([]byte(req.CSR))
|
||||
if blk == nil || blk.Type != "CERTIFICATE REQUEST" {
|
||||
fail(w, 400, errors.New("csr: a PEM CERTIFICATE REQUEST is required (the key stays on the client)"))
|
||||
return
|
||||
}
|
||||
csr, err := x509.ParseCertificateRequest(blk.Bytes)
|
||||
if err == nil {
|
||||
err = csr.CheckSignature()
|
||||
}
|
||||
if err != nil {
|
||||
fail(w, 400, fmt.Errorf("csr: %w", err))
|
||||
return
|
||||
}
|
||||
certPEM, cert, err := a.ca.sign(csr.PublicKey, "admin:"+req.Name, nil, x509.ExtKeyUsageClientAuth)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
by := a.who(r)
|
||||
if err := a.store.AddAdminCert(r.Context(), AdminCert{Serial: cert.SerialNumber.Text(16), Name: req.Name, Expires: cert.NotAfter, IssuedBy: by}); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.dropAdminCertCache()
|
||||
log.Printf("admin certificate %q (serial %s, until %s) issued by %s", req.Name, cert.SerialNumber.Text(16), cert.NotAfter.Format("2006-01-02"), by)
|
||||
writeJSON(w, 200, obj{"name": req.Name, "serial": cert.SerialNumber.Text(16), "expires": cert.NotAfter, "cert": string(certPEM), "ca": string(a.ca.PEM)})
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleAdminCertRevoke(w http.ResponseWriter, r *http.Request) {
|
||||
ref := r.PathValue("ref")
|
||||
n, err := a.store.RevokeAdminCerts(r.Context(), ref)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
if n == 0 {
|
||||
fail(w, 404, fmt.Errorf("no valid admin certificate %q", ref))
|
||||
return
|
||||
}
|
||||
a.dropAdminCertCache()
|
||||
log.Printf("admin certificate %q revoked (%d) by %s", ref, n, a.who(r))
|
||||
writeJSON(w, 200, obj{"revoked": n})
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
pkcs12 "software.sslmate.com/src/go-pkcs12"
|
||||
)
|
||||
|
||||
// clientCertificate: the admin client certificate from ROLLCALL_CERT/ROLLCALL_KEY
|
||||
// or client.json, nil when there is none.
|
||||
func clientCertificate() *tls.Certificate {
|
||||
certFile, keyFile := os.Getenv("ROLLCALL_CERT"), os.Getenv("ROLLCALL_KEY")
|
||||
if certFile == "" {
|
||||
if cc, ok := loadClientConf(); ok {
|
||||
certFile, keyFile = cc.Cert, cc.Key
|
||||
}
|
||||
}
|
||||
if certFile == "" || keyFile == "" {
|
||||
return nil
|
||||
}
|
||||
pair, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "warning: admin certificate %s: %v\n", certFile, err)
|
||||
return nil
|
||||
}
|
||||
return &pair
|
||||
}
|
||||
|
||||
// rollcall admin-cert request NAME | list | revoke NAME|SERIAL
|
||||
func (c *client) adminCertCmd(args []string) error {
|
||||
sub := ""
|
||||
if len(args) > 0 {
|
||||
sub = args[0]
|
||||
}
|
||||
switch sub {
|
||||
case "", "list":
|
||||
var cs []AdminCert
|
||||
if err := c.do("GET", "/api/v1/admin-certs", nil, &cs); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-20s %-34s %-11s %-11s %s\n", "NAME", "SERIAL", "EXPIRES", "STATE", "ISSUED BY")
|
||||
for _, x := range cs {
|
||||
st := "valid"
|
||||
if x.Revoked != nil {
|
||||
st = "revoked"
|
||||
} else if time.Now().After(x.Expires) {
|
||||
st = "expired"
|
||||
}
|
||||
fmt.Printf("%-20s %-34s %-11s %-11s %s\n", x.Name, x.Serial, x.Expires.Format("2006-01-02"), st, x.IssuedBy)
|
||||
}
|
||||
return nil
|
||||
case "revoke":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
var res struct {
|
||||
Revoked int `json:"revoked"`
|
||||
}
|
||||
if err := c.do("DELETE", "/api/v1/admin-certs/"+args[1], nil, &res); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("revoked %d certificate(s) for %s\n", res.Revoked, args[1])
|
||||
return nil
|
||||
case "request":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
return c.requestAdminCert(strings.ToLower(args[1]))
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *client) requestAdminCert(name string) error {
|
||||
if !strings.HasPrefix(c.base, "https://") {
|
||||
return errors.New("request the certificate over HTTPS (ROLLCALL_URL=https://..., or connect the console to https://)")
|
||||
}
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
der, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{Subject: pkix.Name{CommonName: "admin:" + name}}, key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var res struct {
|
||||
Serial string `json:"serial"`
|
||||
Expires time.Time `json:"expires"`
|
||||
Cert string `json:"cert"`
|
||||
CA string `json:"ca"`
|
||||
}
|
||||
if err := c.do("POST", "/api/v1/admin-certs", obj{"name": name, "csr": string(pemEncode("CERTIFICATE REQUEST", der))}, &res); err != nil {
|
||||
return err
|
||||
}
|
||||
dir := filepath.Dir(clientConfPath())
|
||||
if err := os.MkdirAll(dir, 0o700); err != nil {
|
||||
return err
|
||||
}
|
||||
certFile := filepath.Join(dir, "admin-"+name+".pem")
|
||||
keyFile := filepath.Join(dir, "admin-"+name+"-key.pem")
|
||||
p12File := filepath.Join(dir, "admin-"+name+".p12")
|
||||
caFile := filepath.Join(dir, "rollcall-ca.pem")
|
||||
kder, _ := x509.MarshalECPrivateKey(key)
|
||||
if err := os.WriteFile(keyFile, pemEncode("EC PRIVATE KEY", kder), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(certFile, []byte(res.Cert), 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(caFile, []byte(res.CA), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
// a PKCS#12 bundle for the browser / macOS keychain (3DES: every keychain imports it)
|
||||
blk, _ := pem.Decode([]byte(res.Cert))
|
||||
leaf, err := x509.ParseCertificate(blk.Bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cab, _ := pem.Decode([]byte(res.CA))
|
||||
ca, err := x509.ParseCertificate(cab.Bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pass := randomToken(9)
|
||||
p12, err := pkcs12.LegacyDES.Encode(key, leaf, []*x509.Certificate{ca}, pass)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(p12File, p12, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
// the CLI and console use it from now on
|
||||
cc, _ := loadClientConf()
|
||||
if cc.URL == "" {
|
||||
cc.URL = c.base
|
||||
}
|
||||
if cc.CA == "" || strings.TrimRight(cc.URL, "/") == strings.TrimRight(c.base, "/") {
|
||||
cc.CA = res.CA // the region's current CA (it may have been rotated since)
|
||||
}
|
||||
cc.Cert, cc.Key = certFile, keyFile
|
||||
if err := saveClientConf(cc); err != nil {
|
||||
return err
|
||||
}
|
||||
// prove it: a request with the certificate alone
|
||||
pair, err := tls.LoadX509KeyPair(certFile, keyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
probe := &client{base: c.base, http: newHTTPClient(res.CA), ca: res.CA}
|
||||
probe.http.Transport.(*httpTransport).TLSClientConfig.Certificates = []tls.Certificate{pair}
|
||||
verdict := "works without the token"
|
||||
if err := probe.do("GET", "/api/v1/admin-certs", nil, nil); err != nil {
|
||||
verdict = "NOT accepted yet: " + err.Error()
|
||||
}
|
||||
fmt.Printf(`admin certificate "admin:%s" serial %s, valid until %s: %s
|
||||
%s
|
||||
%s (key, stays here)
|
||||
%s (for the browser, password: %s)
|
||||
%s (the installation's CA)
|
||||
client.json now uses the certificate; the token can be removed from it.
|
||||
macOS browser:
|
||||
security import %s -k ~/Library/Keychains/login.keychain-db -P %s
|
||||
security add-trusted-cert -r trustRoot -k ~/Library/Keychains/login.keychain-db %s
|
||||
then open %s and pick "admin:%s" when the browser asks for a certificate.
|
||||
`, name, res.Serial, res.Expires.Format("2006-01-02"), verdict, certFile, keyFile, p12File, pass, caFile, p12File, pass, caFile, c.base, name)
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,787 @@
|
||||
package main
|
||||
|
||||
// The region is the certificate authority of its installation, the way the
|
||||
// llmgw control plane is: one root every part trusts, leaves issued from it,
|
||||
// identity in the subject.
|
||||
//
|
||||
// - the region serves its API and UI over HTTPS with a certificate from it;
|
||||
// - rack controllers join with a one-time code (CA fingerprint, secret,
|
||||
// expiry, address in one word) and get a client+server certificate whose
|
||||
// common name is their name; from then on they talk to the region over
|
||||
// mutual TLS and the shared rack_secret is not needed;
|
||||
// - deployed machines trust it (anchors in the image + bootcmd), stateless overlays get
|
||||
// it as .CA, and `rollcall ca issue` hands out leaves for anything else.
|
||||
//
|
||||
// Nodes still boot over plain HTTP: PXE firmware and the live image have no
|
||||
// way to check a certificate before they have one.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"math/big"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
caValidity = 10 * 365 * 24 * time.Hour
|
||||
leafValidity = 365 * 24 * time.Hour
|
||||
backdate = time.Hour
|
||||
renewBefore = 30 * 24 * time.Hour
|
||||
joinPrefix = "rollcall-join1-"
|
||||
)
|
||||
|
||||
type CA struct {
|
||||
Cert *x509.Certificate
|
||||
Key *ecdsa.PrivateKey
|
||||
PEM []byte
|
||||
}
|
||||
|
||||
func randomSerial() (*big.Int, error) {
|
||||
return rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128))
|
||||
}
|
||||
|
||||
func pemEncode(kind string, der []byte) []byte {
|
||||
return pem.EncodeToMemory(&pem.Block{Type: kind, Bytes: der})
|
||||
}
|
||||
|
||||
func newCA(cn string) (*CA, error) {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
serial, err := randomSerial()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tmpl := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: cn, Organization: []string{"rollcall"}},
|
||||
NotBefore: time.Now().Add(-backdate), NotAfter: time.Now().Add(caValidity),
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, BasicConstraintsValid: true, IsCA: true, MaxPathLen: 1}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cert, err := x509.ParseCertificate(der)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &CA{Cert: cert, Key: key, PEM: pemEncode("CERTIFICATE", der)}, nil
|
||||
}
|
||||
|
||||
func parseCertPEM(b []byte) (*x509.Certificate, error) {
|
||||
blk, _ := pem.Decode(b)
|
||||
if blk == nil {
|
||||
return nil, errors.New("not PEM")
|
||||
}
|
||||
return x509.ParseCertificate(blk.Bytes)
|
||||
}
|
||||
|
||||
func parseKeyPEM(b []byte) (*ecdsa.PrivateKey, error) {
|
||||
blk, _ := pem.Decode(b)
|
||||
if blk == nil {
|
||||
return nil, errors.New("not PEM")
|
||||
}
|
||||
return x509.ParseECPrivateKey(blk.Bytes)
|
||||
}
|
||||
|
||||
// loadOrCreateCA keeps the authority in dir as ca.pem / ca-key.pem.
|
||||
func loadOrCreateCA(dir, cn string) (*CA, error) {
|
||||
cp, kp := filepath.Join(dir, "ca.pem"), filepath.Join(dir, "ca-key.pem")
|
||||
if cb, err := os.ReadFile(cp); err == nil {
|
||||
kb, err := os.ReadFile(kp)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
cert, err := parseCertPEM(cb)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", cp, err)
|
||||
}
|
||||
key, err := parseKeyPEM(kb)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", kp, err)
|
||||
}
|
||||
return &CA{Cert: cert, Key: key, PEM: cb}, nil
|
||||
}
|
||||
ca, err := newCA(cn)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.MkdirAll(dir, 0o750); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
kder, _ := x509.MarshalECPrivateKey(ca.Key)
|
||||
if err := os.WriteFile(kp, pemEncode("EC PRIVATE KEY", kder), 0o600); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.WriteFile(cp, ca.PEM, 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
log.Printf("created certificate authority %q in %s (fingerprint %s)", cn, dir, ca.Fingerprint())
|
||||
return ca, nil
|
||||
}
|
||||
|
||||
func certFingerprint(c *x509.Certificate) [32]byte { return sha256.Sum256(c.Raw) }
|
||||
|
||||
// Fingerprint is the SHA-256 of the CA certificate, as people compare it.
|
||||
func (ca *CA) Fingerprint() string { return fpString(certFingerprint(ca.Cert)) }
|
||||
|
||||
func fpString(sum [32]byte) string {
|
||||
h := hex.EncodeToString(sum[:])
|
||||
var p []string
|
||||
for i := 0; i < len(h); i += 2 {
|
||||
p = append(p, strings.ToUpper(h[i:i+2]))
|
||||
}
|
||||
return "sha256:" + strings.Join(p, ":")
|
||||
}
|
||||
|
||||
func splitHosts(hosts []string) (ips []net.IP, dns []string) {
|
||||
seen := map[string]bool{}
|
||||
for _, h := range hosts {
|
||||
h = strings.Trim(strings.TrimSpace(h), "[]")
|
||||
if h == "" || seen[h] {
|
||||
continue
|
||||
}
|
||||
seen[h] = true
|
||||
if ip := net.ParseIP(h); ip != nil {
|
||||
ips = append(ips, ip)
|
||||
} else {
|
||||
dns = append(dns, h)
|
||||
}
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// sign issues a certificate for pub with the given name, hosts and usages.
|
||||
func (ca *CA) sign(pub crypto.PublicKey, cn string, hosts []string, usage ...x509.ExtKeyUsage) ([]byte, *x509.Certificate, error) {
|
||||
serial, err := randomSerial()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
tmpl := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: cn, Organization: []string{"rollcall"}},
|
||||
NotBefore: time.Now().Add(-backdate), NotAfter: time.Now().Add(leafValidity),
|
||||
KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: usage}
|
||||
tmpl.IPAddresses, tmpl.DNSNames = splitHosts(hosts)
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Cert, pub, ca.Key)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
cert, err := x509.ParseCertificate(der)
|
||||
return pemEncode("CERTIFICATE", der), cert, err
|
||||
}
|
||||
|
||||
// canDelegate: the root may sign intermediate CAs (roots before path length 1 cannot).
|
||||
func (ca *CA) canDelegate() bool {
|
||||
return !(ca.Cert.MaxPathLen == 0 && ca.Cert.MaxPathLenZero)
|
||||
}
|
||||
|
||||
// parsePermits splits "int.example.com,10.213.0.0/16" into DNS and IP constraints.
|
||||
func parsePermits(list string) (dns []string, nets []*net.IPNet, err error) {
|
||||
for _, p := range strings.Split(list, ",") {
|
||||
p = strings.TrimSpace(p)
|
||||
if p == "" {
|
||||
continue
|
||||
}
|
||||
if _, n, e := net.ParseCIDR(p); e == nil {
|
||||
nets = append(nets, n)
|
||||
continue
|
||||
}
|
||||
if net.ParseIP(p) != nil {
|
||||
return nil, nil, fmt.Errorf("%s: give an address range as CIDR", p)
|
||||
}
|
||||
dns = append(dns, strings.TrimPrefix(strings.ToLower(p), "*."))
|
||||
}
|
||||
return dns, nets, nil
|
||||
}
|
||||
|
||||
// intermediate signs a CA certificate for pub that may issue only below the
|
||||
// permitted DNS names and inside the permitted address ranges, one level deep.
|
||||
func (ca *CA) intermediate(pub crypto.PublicKey, cn string, dns []string, nets []*net.IPNet, validity time.Duration) ([]byte, *x509.Certificate, error) {
|
||||
if !ca.canDelegate() {
|
||||
return nil, nil, errors.New("this root was created with path length 0 and cannot sign intermediate CAs; `rollcall ca rotate` replaces it with one that can")
|
||||
}
|
||||
serial, err := randomSerial()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if validity <= 0 || validity > caValidity {
|
||||
validity = 365 * 24 * time.Hour
|
||||
}
|
||||
if na := ca.Cert.NotAfter; time.Now().Add(validity).After(na) {
|
||||
validity = time.Until(na)
|
||||
}
|
||||
tmpl := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: cn, Organization: []string{"rollcall"}},
|
||||
NotBefore: time.Now().Add(-backdate), NotAfter: time.Now().Add(validity),
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, BasicConstraintsValid: true, IsCA: true, MaxPathLenZero: true,
|
||||
PermittedDNSDomainsCritical: len(dns)+len(nets) > 0, PermittedDNSDomains: dns, PermittedIPRanges: nets}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Cert, pub, ca.Key)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
cert, err := x509.ParseCertificate(der)
|
||||
return pemEncode("CERTIFICATE", der), cert, err
|
||||
}
|
||||
|
||||
// issue makes a key and a certificate for it.
|
||||
func (ca *CA) issue(cn string, hosts []string, usage ...x509.ExtKeyUsage) (certPEM, keyPEM []byte, err error) {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
certPEM, _, err = ca.sign(&key.PublicKey, cn, hosts, usage...)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
kder, _ := x509.MarshalECPrivateKey(key)
|
||||
return certPEM, pemEncode("EC PRIVATE KEY", kder), nil
|
||||
}
|
||||
|
||||
// hostAddresses: names and addresses this machine answers on (no link-local).
|
||||
func hostAddresses() []string {
|
||||
h, _ := os.Hostname()
|
||||
out := []string{"localhost", "127.0.0.1", "::1"}
|
||||
if h != "" {
|
||||
out = append(out, h)
|
||||
}
|
||||
ifs, _ := net.Interfaces()
|
||||
for _, i := range ifs {
|
||||
addrs, _ := i.Addrs()
|
||||
for _, a := range addrs {
|
||||
if ipn, ok := a.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() {
|
||||
out = append(out, ipn.IP.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// regionCert keeps dir/region.pem valid for every name and address the region has.
|
||||
func regionCert(ca *CA, dir string, extra []string) (tls.Certificate, error) {
|
||||
cp, kp := filepath.Join(dir, "region.pem"), filepath.Join(dir, "region-key.pem")
|
||||
want := append(hostAddresses(), extra...)
|
||||
if cb, err := os.ReadFile(cp); err == nil {
|
||||
if cert, err := parseCertPEM(cb); err == nil && time.Until(cert.NotAfter) > renewBefore && cert.CheckSignatureFrom(ca.Cert) == nil && coversHosts(cert, want) {
|
||||
if pair, err := tls.LoadX509KeyPair(cp, kp); err == nil {
|
||||
pair.Certificate = append(pair.Certificate, ca.Cert.Raw)
|
||||
return pair, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
h, _ := os.Hostname()
|
||||
certPEM, keyPEM, err := ca.issue("rollcall region "+h, want, x509.ExtKeyUsageServerAuth)
|
||||
if err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
if err := os.WriteFile(kp, keyPEM, 0o600); err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
if err := os.WriteFile(cp, certPEM, 0o644); err != nil {
|
||||
return tls.Certificate{}, err
|
||||
}
|
||||
log.Printf("issued the region's TLS certificate for %s", strings.Join(want, " "))
|
||||
pair, err := tls.X509KeyPair(certPEM, keyPEM)
|
||||
if err != nil {
|
||||
return pair, err
|
||||
}
|
||||
pair.Certificate = append(pair.Certificate, ca.Cert.Raw) // clients that pin the CA find it in the chain
|
||||
return pair, nil
|
||||
}
|
||||
|
||||
func coversHosts(c *x509.Certificate, hosts []string) bool {
|
||||
for _, h := range hosts {
|
||||
if c.VerifyHostname(strings.Trim(h, "[]")) != nil {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- join codes
|
||||
|
||||
type joinCode struct {
|
||||
Fingerprint [16]byte // first half of the CA certificate's SHA-256
|
||||
Secret [16]byte
|
||||
Expires int64
|
||||
Addr string // host:port of the region's HTTPS listener
|
||||
}
|
||||
|
||||
func (c joinCode) String() string {
|
||||
b := make([]byte, 0, 36+len(c.Addr))
|
||||
b = append(b, c.Fingerprint[:]...)
|
||||
b = append(b, c.Secret[:]...)
|
||||
b = binary.BigEndian.AppendUint32(b, uint32(c.Expires))
|
||||
b = append(b, c.Addr...)
|
||||
return joinPrefix + base64.RawURLEncoding.EncodeToString(b)
|
||||
}
|
||||
|
||||
func parseJoinCode(s string) (joinCode, error) {
|
||||
raw, ok := strings.CutPrefix(strings.TrimSpace(s), joinPrefix)
|
||||
if !ok {
|
||||
return joinCode{}, errors.New("not a join code: it starts " + joinPrefix)
|
||||
}
|
||||
b, err := base64.RawURLEncoding.DecodeString(raw)
|
||||
if err != nil || len(b) < 36+3 {
|
||||
return joinCode{}, errors.New("the join code is damaged: copy it whole")
|
||||
}
|
||||
var c joinCode
|
||||
copy(c.Fingerprint[:], b[:16])
|
||||
copy(c.Secret[:], b[16:32])
|
||||
c.Expires = int64(binary.BigEndian.Uint32(b[32:36]))
|
||||
c.Addr = string(b[36:])
|
||||
if _, _, err := net.SplitHostPort(c.Addr); err != nil {
|
||||
return joinCode{}, errors.New("the join code is damaged: copy it whole")
|
||||
}
|
||||
if time.Now().Unix() > c.Expires {
|
||||
return joinCode{}, fmt.Errorf("the join code expired at %s; ask for a new one", time.Unix(c.Expires, 0).Format(time.RFC3339))
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// pinnedTLS trusts a server whose chain carries the CA with this fingerprint.
|
||||
func pinnedTLS(fp [16]byte) *tls.Config {
|
||||
return &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: true, //nolint:gosec // verified below against the pinned CA
|
||||
VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error {
|
||||
var leaf *x509.Certificate
|
||||
for i, r := range raw {
|
||||
c, err := x509.ParseCertificate(r)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if i == 0 {
|
||||
leaf = c
|
||||
}
|
||||
sum := sha256.Sum256(r)
|
||||
if c.IsCA && string(sum[:16]) == string(fp[:]) {
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(c)
|
||||
_, err := leaf.Verify(x509.VerifyOptions{Roots: pool})
|
||||
return err
|
||||
}
|
||||
}
|
||||
return errors.New("the server's certificate is not from the authority this code is for")
|
||||
}}
|
||||
}
|
||||
|
||||
func (s *Store) PutJoinToken(ctx context.Context, hash, name string, expires time.Time) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO join_tokens(hash, name, expires) VALUES ($1,nullif($2,''),$3)`, hash, name, expires)
|
||||
return err
|
||||
}
|
||||
|
||||
// UseJoinToken consumes a token; it returns the name it was bound to.
|
||||
func (s *Store) UseJoinToken(ctx context.Context, hash string) (string, error) {
|
||||
var name string
|
||||
err := s.pool.QueryRow(ctx, `UPDATE join_tokens SET used=now() WHERE hash=$1 AND used IS NULL AND expires > now() RETURNING coalesce(name,'')`, hash).Scan(&name)
|
||||
if err != nil {
|
||||
return "", errors.New("unknown, used or expired join code")
|
||||
}
|
||||
return name, nil
|
||||
}
|
||||
|
||||
func (s *Store) SetRackCert(ctx context.Context, name, serial string, expires time.Time) error {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET cert_serial=$2, cert_expires=$3 WHERE name=$1`, name, serial, expires)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) RackCertSerial(ctx context.Context, name string) string {
|
||||
var serial string
|
||||
_ = s.pool.QueryRow(ctx, `SELECT coalesce(cert_serial,'') FROM rack_controllers WHERE name=$1`, name).Scan(&serial)
|
||||
return serial
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- HTTP
|
||||
|
||||
// peerRack is the rack controller a mutual-TLS request comes from, if any:
|
||||
// a verified client certificate whose serial is the one on record.
|
||||
func (a *App) peerRack(r *http.Request) string {
|
||||
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 || len(r.TLS.PeerCertificates) == 0 {
|
||||
return ""
|
||||
}
|
||||
c := r.TLS.PeerCertificates[0]
|
||||
name := strings.TrimPrefix(c.Subject.CommonName, "rack:")
|
||||
if name == c.Subject.CommonName { // only rack certificates
|
||||
return ""
|
||||
}
|
||||
if a.store.RackCertSerial(r.Context(), name) != c.SerialNumber.Text(16) {
|
||||
return ""
|
||||
}
|
||||
return name
|
||||
}
|
||||
|
||||
func (a *App) handleCAPEM(w http.ResponseWriter, r *http.Request) {
|
||||
if a.ca == nil {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/x-pem-file")
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="rollcall-ca.pem"`)
|
||||
w.Header().Set("X-Rollcall-CA-Fingerprint", a.ca.Fingerprint())
|
||||
_, _ = w.Write(a.ca.PEM)
|
||||
}
|
||||
|
||||
// handleJoinCode makes a one-time code for a rack controller (admin).
|
||||
func (a *App) handleJoinCode(w http.ResponseWriter, r *http.Request) {
|
||||
if a.ca == nil || a.cfg.TLSListen == "" {
|
||||
fail(w, 409, errors.New("the region has no TLS listener (tls_listen)"))
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Name string `json:"name"`
|
||||
TTL int `json:"ttl_minutes"`
|
||||
Addr string `json:"addr"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
if req.TTL <= 0 {
|
||||
req.TTL = 60
|
||||
}
|
||||
addr := req.Addr
|
||||
if addr == "" {
|
||||
host := r.Host
|
||||
if h, _, err := net.SplitHostPort(host); err == nil {
|
||||
host = h
|
||||
}
|
||||
if ip := net.ParseIP(strings.Trim(host, "[]")); (ip != nil && ip.IsLoopback()) || host == "localhost" {
|
||||
fail(w, 400, errors.New("say where rack controllers reach the region's HTTPS: addr (CLI -addr host:443)"))
|
||||
return
|
||||
}
|
||||
_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
|
||||
addr = net.JoinHostPort(strings.Trim(host, "[]"), port)
|
||||
}
|
||||
var c joinCode
|
||||
sum := certFingerprint(a.ca.Cert)
|
||||
copy(c.Fingerprint[:], sum[:16])
|
||||
_, _ = rand.Read(c.Secret[:])
|
||||
c.Expires = time.Now().Add(time.Duration(req.TTL) * time.Minute).Unix()
|
||||
c.Addr = addr
|
||||
h := sha256.Sum256(c.Secret[:])
|
||||
if err := a.store.PutJoinToken(r.Context(), hex.EncodeToString(h[:]), req.Name, time.Unix(c.Expires, 0)); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, obj{"code": c.String(), "expires": time.Unix(c.Expires, 0), "addr": addr, "name": req.Name, "ca_fingerprint": a.ca.Fingerprint()})
|
||||
}
|
||||
|
||||
type enrollRequest struct {
|
||||
Secret string `json:"secret"` // base64 of the code's secret
|
||||
Name string `json:"name"`
|
||||
CSR string `json:"csr"`
|
||||
Rack RackController `json:"rack"`
|
||||
}
|
||||
|
||||
func (a *App) signRackCSR(csrPEM, name string, rc RackController) ([]byte, *x509.Certificate, error) {
|
||||
blk, _ := pem.Decode([]byte(csrPEM))
|
||||
if blk == nil {
|
||||
return nil, nil, errors.New("csr is not PEM")
|
||||
}
|
||||
csr, err := x509.ParseCertificateRequest(blk.Bytes)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := csr.CheckSignature(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
hosts := []string{name}
|
||||
if u := strings.TrimPrefix(strings.TrimPrefix(rc.URL, "http://"), "https://"); u != "" {
|
||||
if h, _, err := net.SplitHostPort(u); err == nil {
|
||||
hosts = append(hosts, h)
|
||||
} else {
|
||||
hosts = append(hosts, strings.Trim(u, "/[]"))
|
||||
}
|
||||
}
|
||||
for _, i := range rc.Interfaces {
|
||||
for _, ad := range i.Addrs {
|
||||
hosts = append(hosts, stripPrefix(ad))
|
||||
}
|
||||
}
|
||||
return a.ca.sign(csr.PublicKey, "rack:"+name, hosts, x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth)
|
||||
}
|
||||
|
||||
// handleRackEnroll: a rack controller presents a join code's secret and a CSR
|
||||
// over TLS; it gets a certificate and the CA. No admin token, no client cert.
|
||||
func (a *App) handleRackEnroll(w http.ResponseWriter, r *http.Request) {
|
||||
if r.TLS == nil || a.ca == nil {
|
||||
fail(w, 400, errors.New("enrollment needs HTTPS"))
|
||||
return
|
||||
}
|
||||
var req enrollRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
sec, err := base64.RawURLEncoding.DecodeString(req.Secret)
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
h := sha256.Sum256(sec)
|
||||
bound, err := a.store.UseJoinToken(r.Context(), hex.EncodeToString(h[:]))
|
||||
if err != nil {
|
||||
fail(w, 401, err)
|
||||
return
|
||||
}
|
||||
name := req.Name
|
||||
if bound != "" {
|
||||
name = bound
|
||||
}
|
||||
if !tagNameRe.MatchString(name) || name == a.cfg.RackName {
|
||||
fail(w, 400, fmt.Errorf("bad rack controller name %q", name))
|
||||
return
|
||||
}
|
||||
certPEM, cert, err := a.signRackCSR(req.CSR, name, req.Rack)
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
rc := req.Rack
|
||||
rc.Name, rc.Kind = name, "remote"
|
||||
if err := a.store.PutRackController(r.Context(), rc); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
if err := a.store.SetRackCert(r.Context(), name, cert.SerialNumber.Text(16), cert.NotAfter); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.reloadRacks(r.Context())
|
||||
a.hub.bump()
|
||||
log.Printf("rack controller %s enrolled from %s, certificate until %s", name, r.RemoteAddr, cert.NotAfter.Format("2006-01-02"))
|
||||
writeJSON(w, 200, obj{"name": name, "cert": string(certPEM), "ca": string(a.ca.PEM)})
|
||||
}
|
||||
|
||||
// handleRackRenew issues a new certificate to a rack controller that still has a valid one.
|
||||
func (a *App) handleRackRenew(w http.ResponseWriter, r *http.Request, rack string) {
|
||||
if a.peerRack(r) != rack {
|
||||
fail(w, 401, errors.New("renewal needs the current certificate"))
|
||||
return
|
||||
}
|
||||
var req enrollRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
rc := a.racksSnapshot()[rack]
|
||||
certPEM, cert, err := a.signRackCSR(req.CSR, rack, rc)
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := a.store.SetRackCert(r.Context(), rack, cert.SerialNumber.Text(16), cert.NotAfter); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
log.Printf("rack controller %s renewed its certificate, until %s", rack, cert.NotAfter.Format("2006-01-02"))
|
||||
writeJSON(w, 200, obj{"name": rack, "cert": string(certPEM), "ca": string(a.ca.PEM)})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- CLI: rollcall ca (on the region host)
|
||||
|
||||
func caCmd(args []string) error {
|
||||
cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(args) == 0 || args[0] == "show" {
|
||||
cb, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("no CA yet (it is created when rollcall serve starts): %w", err)
|
||||
}
|
||||
c, err := parseCertPEM(cb)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s\n subject %s\n valid until %s\n fingerprint %s\n", filepath.Join(cfg.CADir, "ca.pem"), c.Subject.CommonName,
|
||||
c.NotAfter.Format("2006-01-02"), fpString(certFingerprint(c)))
|
||||
if rb, err := os.ReadFile(filepath.Join(cfg.CADir, "region.pem")); err == nil {
|
||||
if rc, err := parseCertPEM(rb); err == nil {
|
||||
names := append([]string{}, rc.DNSNames...)
|
||||
for _, ip := range rc.IPAddresses {
|
||||
names = append(names, ip.String())
|
||||
}
|
||||
sort.Strings(names)
|
||||
fmt.Printf("region certificate until %s for %s\n", rc.NotAfter.Format("2006-01-02"), strings.Join(names, " "))
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
switch {
|
||||
case args[0] == "intermediate" && len(args) >= 2:
|
||||
return caIntermediateCmd(cfg, args[1], args[2:])
|
||||
case args[0] == "rotate":
|
||||
return caRotateCmd(cfg, args[1:])
|
||||
case args[0] != "issue" || len(args) < 2:
|
||||
usage()
|
||||
}
|
||||
fs := newFlags("ca issue")
|
||||
hosts := fs.String("host", "", "comma-separated DNS names and addresses")
|
||||
client := fs.Bool("client", false, "client certificate (identifies its holder) instead of a server one")
|
||||
out := fs.String("out", ".", "directory for NAME.pem and NAME-key.pem")
|
||||
_ = fs.Parse(args[2:])
|
||||
ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var hs []string
|
||||
if *hosts != "" {
|
||||
hs = strings.Split(*hosts, ",")
|
||||
}
|
||||
usage := x509.ExtKeyUsageServerAuth
|
||||
if *client {
|
||||
usage = x509.ExtKeyUsageClientAuth
|
||||
} else if len(hs) == 0 {
|
||||
return errors.New("a server certificate needs -host")
|
||||
}
|
||||
certPEM, keyPEM, err := ca.issue(args[1], hs, usage)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(*out, args[1]+"-key.pem"), keyPEM, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(*out, args[1]+".pem"), append(certPEM, ca.PEM...), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("wrote %s.pem (with the CA) and %s-key.pem in %s, valid until %s\n", args[1], args[1], *out, time.Now().Add(leafValidity).Format("2006-01-02"))
|
||||
return nil
|
||||
}
|
||||
|
||||
// rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days N] [-out DIR]
|
||||
func caIntermediateCmd(cfg *Config, name string, args []string) error {
|
||||
fs := newFlags("ca intermediate")
|
||||
permit := fs.String("permit", "", "comma-separated DNS domains and CIDRs it may issue for (required)")
|
||||
csrFile := fs.String("csr", "", "sign this CSR (the key stays where it was made); without it a key is generated")
|
||||
days := fs.Int("days", 365, "validity in days")
|
||||
out := fs.String("out", ".", "directory for NAME.pem (+ NAME-key.pem without -csr)")
|
||||
unconstrained := fs.Bool("unconstrained", false, "allow an intermediate without name constraints")
|
||||
_ = fs.Parse(args)
|
||||
dns, nets, err := parsePermits(*permit)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(dns)+len(nets) == 0 && !*unconstrained {
|
||||
return errors.New("an intermediate needs -permit (e.g. -permit int.example.com,10.213.0.0/16); -unconstrained to override")
|
||||
}
|
||||
ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var pub crypto.PublicKey
|
||||
var keyPEM []byte
|
||||
if *csrFile != "" {
|
||||
b, err := os.ReadFile(*csrFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
blk, _ := pem.Decode(b)
|
||||
if blk == nil {
|
||||
return fmt.Errorf("%s: not PEM", *csrFile)
|
||||
}
|
||||
csr, err := x509.ParseCertificateRequest(blk.Bytes)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if err := csr.CheckSignature(); err != nil {
|
||||
return fmt.Errorf("CSR signature: %w", err)
|
||||
}
|
||||
pub = csr.PublicKey
|
||||
} else {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pub = &key.PublicKey
|
||||
kder, _ := x509.MarshalECPrivateKey(key)
|
||||
keyPEM = pemEncode("EC PRIVATE KEY", kder)
|
||||
}
|
||||
certPEM, cert, err := ca.intermediate(pub, name, dns, nets, time.Duration(*days)*24*time.Hour)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if keyPEM != nil {
|
||||
if err := os.WriteFile(filepath.Join(*out, name+"-key.pem"), keyPEM, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(*out, name+".pem"), certPEM, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
var permits []string
|
||||
permits = append(permits, dns...)
|
||||
for _, n := range nets {
|
||||
permits = append(permits, n.String())
|
||||
}
|
||||
fmt.Printf("wrote %s.pem in %s: intermediate CA %q until %s, may issue for %s\nroot: %s (%s)\n", name, *out, name,
|
||||
cert.NotAfter.Format("2006-01-02"), orDash(strings.Join(permits, " ")), filepath.Join(cfg.CADir, "ca.pem"), ca.Fingerprint())
|
||||
return nil
|
||||
}
|
||||
|
||||
// rollcall ca rotate -yes: a new root (the old one is kept as a backup). Racks
|
||||
// must join again and clients trust the new fingerprint.
|
||||
func caRotateCmd(cfg *Config, args []string) error {
|
||||
fs := newFlags("ca rotate")
|
||||
yes := fs.Bool("yes", false, "really replace the root")
|
||||
_ = fs.Parse(args)
|
||||
old, err := loadOrCreateCA(cfg.CADir, "rollcall CA")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !*yes {
|
||||
return fmt.Errorf("this replaces the root %s (delegates: %v); rack controllers then join again and clients trust the new one. Run with -yes", old.Fingerprint(), old.canDelegate())
|
||||
}
|
||||
bak := cfg.CADir + ".bak-" + time.Now().Format("20060102-150405")
|
||||
if err := os.Rename(cfg.CADir, bak); err != nil {
|
||||
return err
|
||||
}
|
||||
ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA")
|
||||
if err != nil {
|
||||
_ = os.Rename(bak, cfg.CADir)
|
||||
return err
|
||||
}
|
||||
if st, err := os.Stat(bak); err == nil {
|
||||
_ = os.Chmod(cfg.CADir, st.Mode().Perm())
|
||||
if own, ok := fileOwner(bak); ok {
|
||||
_ = chownTree(cfg.CADir, own[0], own[1])
|
||||
}
|
||||
}
|
||||
fmt.Printf("new root %s\nold root kept in %s\nnext: restart rollcall; on each rack controller `rollcall rack join` with a new join code; clients trust the new fingerprint\n", ca.Fingerprint(), bak)
|
||||
return nil
|
||||
}
|
||||
|
||||
// fileOwner returns uid, gid of path (unix).
|
||||
func fileOwner(path string) ([2]int, bool) {
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return [2]int{}, false
|
||||
}
|
||||
if s, ok := st.Sys().(*syscall.Stat_t); ok {
|
||||
return [2]int{int(s.Uid), int(s.Gid)}, true
|
||||
}
|
||||
return [2]int{}, false
|
||||
}
|
||||
|
||||
func chownTree(root string, uid, gid int) error {
|
||||
return filepath.Walk(root, func(p string, _ os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Lchown(p, uid, gid)
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,236 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestCAAndRegionCert(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ca, err := loadOrCreateCA(dir, "test CA")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again, err := loadOrCreateCA(dir, "test CA")
|
||||
if err != nil || again.Fingerprint() != ca.Fingerprint() {
|
||||
t.Fatalf("reload gives another CA: %v", err)
|
||||
}
|
||||
pair, err := regionCert(ca, dir, []string{"rollcall.example", "fd00:213:77::1"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
leaf, _ := x509.ParseCertificate(pair.Certificate[0])
|
||||
for _, h := range []string{"rollcall.example", "fd00:213:77::1", "127.0.0.1", "localhost"} {
|
||||
if err := leaf.VerifyHostname(h); err != nil {
|
||||
t.Errorf("region certificate does not cover %s: %v", h, err)
|
||||
}
|
||||
}
|
||||
if len(pair.Certificate) != 2 {
|
||||
t.Errorf("the chain should carry the CA for pinning clients")
|
||||
}
|
||||
// unchanged names: the same certificate is reused
|
||||
p2, _ := regionCert(ca, dir, []string{"rollcall.example", "fd00:213:77::1"})
|
||||
if string(p2.Certificate[0]) != string(pair.Certificate[0]) {
|
||||
t.Errorf("certificate reissued without a reason")
|
||||
}
|
||||
}
|
||||
|
||||
func TestJoinCodeAndPinnedTLS(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
ca, _ := loadOrCreateCA(dir, "test CA")
|
||||
pair, _ := regionCert(ca, dir, nil)
|
||||
srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) }))
|
||||
srv.TLS = &tls.Config{Certificates: []tls.Certificate{pair}}
|
||||
srv.StartTLS()
|
||||
defer srv.Close()
|
||||
|
||||
var c joinCode
|
||||
sum := certFingerprint(ca.Cert)
|
||||
copy(c.Fingerprint[:], sum[:16])
|
||||
c.Secret[0] = 7
|
||||
c.Expires = time.Now().Add(time.Hour).Unix()
|
||||
c.Addr = strings.TrimPrefix(srv.URL, "https://")
|
||||
got, err := parseJoinCode(c.String())
|
||||
if err != nil || got != c {
|
||||
t.Fatalf("round trip: %+v %v", got, err)
|
||||
}
|
||||
ok := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(c.Fingerprint)}}
|
||||
if resp, err := ok.Get(srv.URL); err != nil {
|
||||
t.Fatalf("pinned CA rejected: %v", err)
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
}
|
||||
other, _ := newCA("another")
|
||||
osum := sha256.Sum256(other.Cert.Raw)
|
||||
var wrong [16]byte
|
||||
copy(wrong[:], osum[:16])
|
||||
bad := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(wrong)}}
|
||||
if _, err := bad.Get(srv.URL); err == nil {
|
||||
t.Errorf("a server from another authority was accepted")
|
||||
}
|
||||
expired := c
|
||||
expired.Expires = time.Now().Add(-time.Minute).Unix()
|
||||
if _, err := parseJoinCode(expired.String()); err == nil {
|
||||
t.Errorf("an expired code was accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSignRackCSR(t *testing.T) {
|
||||
ca, _ := newCA("test CA")
|
||||
a := &App{ca: ca, cfg: &Config{}}
|
||||
_, csr, err := newCSR("rack:hall-b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, cert, err := a.signRackCSR(csr, "hall-b", RackController{URL: "http://10.213.78.1", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"fd00:213:78::1/64"}}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(ca.Cert)
|
||||
if _, err := cert.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil {
|
||||
t.Errorf("rack certificate is no client certificate: %v", err)
|
||||
}
|
||||
if cert.Subject.CommonName != "rack:hall-b" || cert.VerifyHostname("10.213.78.1") != nil || cert.VerifyHostname("fd00:213:78::1") != nil {
|
||||
t.Errorf("subject %s, SANs %v %v", cert.Subject.CommonName, cert.IPAddresses, cert.DNSNames)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntermediate(t *testing.T) {
|
||||
root, err := newCA("test root")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !root.canDelegate() {
|
||||
t.Fatal("new roots must be able to delegate one level")
|
||||
}
|
||||
dns, nets, err := parsePermits("int.example.com, 10.213.0.0/16")
|
||||
if err != nil || len(dns) != 1 || len(nets) != 1 {
|
||||
t.Fatalf("permits: %v %v %v", dns, nets, err)
|
||||
}
|
||||
ikey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
_, icert, err := root.intermediate(&ikey.PublicKey, "caddy", dns, nets, 30*24*time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !icert.IsCA || !icert.MaxPathLenZero || !icert.PermittedDNSDomainsCritical {
|
||||
t.Fatalf("intermediate shape: %+v", icert)
|
||||
}
|
||||
sub := &CA{Cert: icert, Key: ikey}
|
||||
roots, inters := x509.NewCertPool(), x509.NewCertPool()
|
||||
roots.AddCert(root.Cert)
|
||||
inters.AddCert(icert)
|
||||
verify := func(hosts ...string) error {
|
||||
_, leaf, err := sub.sign(&ikey.PublicKey, hosts[0], hosts, x509.ExtKeyUsageServerAuth)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = leaf.Verify(x509.VerifyOptions{Roots: roots, Intermediates: inters, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}})
|
||||
return err
|
||||
}
|
||||
if err := verify("node001.int.example.com", "10.213.77.50"); err != nil {
|
||||
t.Errorf("leaf inside the constraints: %v", err)
|
||||
}
|
||||
if err := verify("www.google.com"); err == nil {
|
||||
t.Error("leaf outside the DNS constraint verified")
|
||||
}
|
||||
if err := verify("node001.int.example.com", "192.168.1.1"); err == nil {
|
||||
t.Error("leaf outside the IP constraint verified")
|
||||
}
|
||||
// a second level below the intermediate is refused by path length 0
|
||||
k2, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
_, c2, err := sub.intermediate(&k2.PublicKey, "deeper", dns, nil, time.Hour)
|
||||
if err == nil {
|
||||
inters.AddCert(c2)
|
||||
_, leaf, _ := (&CA{Cert: c2, Key: k2}).sign(&k2.PublicKey, "x.int.example.com", []string{"x.int.example.com"}, x509.ExtKeyUsageServerAuth)
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, Intermediates: inters}); err == nil {
|
||||
t.Error("two levels of intermediates verified")
|
||||
}
|
||||
}
|
||||
// old roots (path length 0) refuse to delegate
|
||||
old := *root
|
||||
oc := *root.Cert
|
||||
oc.MaxPathLen, oc.MaxPathLenZero = 0, true
|
||||
old.Cert = &oc
|
||||
if _, _, err := old.intermediate(&ikey.PublicKey, "x", dns, nil, time.Hour); err == nil {
|
||||
t.Error("path-length-0 root signed an intermediate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUIRedirectsToHTTPS(t *testing.T) {
|
||||
ok := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) })
|
||||
try := func(admin, listen, remote, host string) (int, string) {
|
||||
a := &App{cfg: &Config{AdminHTTP: admin, TLSListen: listen}}
|
||||
req := httptest.NewRequest("GET", "/index.html?x=1", nil)
|
||||
req.RemoteAddr, req.Host = remote, host
|
||||
rec := httptest.NewRecorder()
|
||||
a.uiOverHTTPS(ok).ServeHTTP(rec, req)
|
||||
return rec.Code, rec.Header().Get("Location")
|
||||
}
|
||||
if c, l := try("loopback", ":443", "10.0.0.5:5555", "10.0.0.234"); c != 308 || l != "https://10.0.0.234/index.html?x=1" {
|
||||
t.Errorf("remote over http: %d %s", c, l)
|
||||
}
|
||||
if c, l := try("deny", ":8443", "10.0.0.5:5555", "pxe.example:80"); c != 308 || l != "https://pxe.example:8443/index.html?x=1" {
|
||||
t.Errorf("other port: %d %s", c, l)
|
||||
}
|
||||
if c, l := try("loopback", ":443", "[fd00::5]:5555", "[fd00::1]"); c != 308 || l != "https://[fd00::1]/index.html?x=1" {
|
||||
t.Errorf("ipv6: %d %s", c, l)
|
||||
}
|
||||
if c, _ := try("loopback", ":443", "127.0.0.1:5555", "localhost"); c != 200 {
|
||||
t.Errorf("loopback client redirected: %d", c)
|
||||
}
|
||||
if c, _ := try("allow", ":443", "10.0.0.5:5555", "10.0.0.234"); c != 200 {
|
||||
t.Errorf("allow redirected: %d", c)
|
||||
}
|
||||
if c, _ := try("loopback", "", "10.0.0.5:5555", "10.0.0.234"); c != 200 {
|
||||
t.Errorf("no TLS listener but redirected: %d", c)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAdminClientCertificate(t *testing.T) {
|
||||
ca, _ := newCA("test")
|
||||
key, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
_, admin, _ := ca.sign(&key.PublicKey, "admin:t440s", nil, x509.ExtKeyUsageClientAuth)
|
||||
_, server, _ := ca.sign(&key.PublicKey, "admin:t440s", []string{"x.example"}, x509.ExtKeyUsageServerAuth)
|
||||
_, rack, _ := ca.sign(&key.PublicKey, "rack:r2", nil, x509.ExtKeyUsageClientAuth)
|
||||
a := &App{cfg: &Config{AdminToken: "secret"}}
|
||||
a.adminCC.serial, a.adminCC.at = map[string]string{admin.SerialNumber.Text(16): "t440s", server.SerialNumber.Text(16): "t440s"}, time.Now()
|
||||
req := func(c *x509.Certificate, verified bool) *http.Request {
|
||||
r := httptest.NewRequest("GET", "/api/v1/machines", nil)
|
||||
r.TLS = &tls.ConnectionState{PeerCertificates: []*x509.Certificate{c}}
|
||||
if verified {
|
||||
r.TLS.VerifiedChains = [][]*x509.Certificate{{c, ca.Cert}}
|
||||
}
|
||||
return r
|
||||
}
|
||||
if !a.isAdmin(req(admin, true)) {
|
||||
t.Error("registered admin certificate refused")
|
||||
}
|
||||
if a.isAdmin(req(admin, false)) {
|
||||
t.Error("unverified certificate accepted")
|
||||
}
|
||||
if a.isAdmin(req(server, true)) {
|
||||
t.Error("server certificate accepted as admin")
|
||||
}
|
||||
if a.isAdmin(req(rack, true)) {
|
||||
t.Error("rack certificate accepted as admin")
|
||||
}
|
||||
a.adminCC.serial = map[string]string{} // revoked
|
||||
if a.isAdmin(req(admin, true)) {
|
||||
t.Error("revoked certificate accepted")
|
||||
}
|
||||
tok := httptest.NewRequest("GET", "/", nil)
|
||||
tok.Header.Set("Authorization", "Bearer secret")
|
||||
if !a.isAdmin(tok) {
|
||||
t.Error("token no longer works")
|
||||
}
|
||||
}
|
||||
+401
-23
@@ -3,13 +3,17 @@ package main
|
||||
import (
|
||||
"archive/tar"
|
||||
"bytes"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
@@ -17,14 +21,51 @@ import (
|
||||
type client struct {
|
||||
base, token string
|
||||
http *http.Client
|
||||
ca string
|
||||
}
|
||||
|
||||
// clientTLS: the region's CA from the server config (root on the region), the
|
||||
// client config (trusted on first connect), or ROLLCALL_CA; else the system roots.
|
||||
func clientTLS(caPEM string) *tls.Config {
|
||||
pem := []byte(caPEM)
|
||||
if f := os.Getenv("ROLLCALL_CA"); f != "" {
|
||||
if b, err := os.ReadFile(f); err == nil {
|
||||
pem = b
|
||||
}
|
||||
}
|
||||
if len(pem) == 0 {
|
||||
if cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json")); err == nil {
|
||||
pem, _ = os.ReadFile(filepath.Join(cfg.CADir, "ca.pem"))
|
||||
}
|
||||
}
|
||||
tc := &tls.Config{MinVersion: tls.VersionTLS12}
|
||||
if cert := clientCertificate(); cert != nil {
|
||||
tc.Certificates = []tls.Certificate{*cert}
|
||||
}
|
||||
if len(pem) > 0 {
|
||||
pool, err := x509.SystemCertPool()
|
||||
if err != nil || pool == nil {
|
||||
pool = x509.NewCertPool()
|
||||
}
|
||||
pool.AppendCertsFromPEM(pem)
|
||||
tc.RootCAs = pool
|
||||
}
|
||||
return tc
|
||||
}
|
||||
|
||||
type httpTransport = http.Transport
|
||||
|
||||
func newHTTPClient(caPEM string) *http.Client {
|
||||
return &http.Client{Timeout: 90 * time.Second, Transport: &http.Transport{TLSClientConfig: clientTLS(caPEM), Proxy: http.ProxyFromEnvironment}}
|
||||
}
|
||||
|
||||
func newClient() (*client, error) {
|
||||
base, token := os.Getenv("ROLLCALL_URL"), os.Getenv("ROLLCALL_TOKEN")
|
||||
caPEM := ""
|
||||
if base == "" || token == "" {
|
||||
if cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json")); err == nil {
|
||||
if base == "" {
|
||||
base = "http://" + strings.Replace(cfg.Listen, "0.0.0.0", "127.0.0.1", 1)
|
||||
base = "http://" + localListen(cfg.Listen)
|
||||
}
|
||||
if token == "" {
|
||||
token = cfg.AdminToken
|
||||
@@ -32,9 +73,25 @@ func newClient() (*client, error) {
|
||||
}
|
||||
}
|
||||
if base == "" || token == "" {
|
||||
return nil, fmt.Errorf("set ROLLCALL_URL and ROLLCALL_TOKEN (or make the config file readable)")
|
||||
if cc, ok := loadClientConf(); ok {
|
||||
if base == "" {
|
||||
base = cc.URL
|
||||
}
|
||||
if token == "" {
|
||||
token = cc.Token
|
||||
}
|
||||
caPEM = cc.CA
|
||||
}
|
||||
}
|
||||
return &client{base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 90 * time.Second}}, nil
|
||||
if base == "" || (token == "" && clientCertificate() == nil) {
|
||||
return nil, fmt.Errorf("set ROLLCALL_URL and ROLLCALL_TOKEN (or an admin certificate: rollcall admin-cert request), make the server config readable, or run rollcall in a terminal to connect")
|
||||
}
|
||||
if caPEM == "" {
|
||||
if cc, ok := loadClientConf(); ok && strings.TrimRight(cc.URL, "/") == strings.TrimRight(base, "/") {
|
||||
caPEM = cc.CA
|
||||
}
|
||||
}
|
||||
return &client{base: strings.TrimRight(base, "/"), token: token, http: newHTTPClient(caPEM), ca: caPEM}, nil
|
||||
}
|
||||
|
||||
func (c *client) do(method, path string, body any, out any) error {
|
||||
@@ -47,7 +104,9 @@ func (c *client) do(method, path string, body any, out any) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
@@ -95,13 +154,22 @@ func runClient(cmd string, args []string) error {
|
||||
}
|
||||
switch cmd {
|
||||
case "list":
|
||||
var ms []Summary
|
||||
if err := c.do("GET", "/api/v1/machines", nil, &ms); err != nil {
|
||||
fs := newFlags("list")
|
||||
tag := fs.String("tag", "", "only machines with this tag")
|
||||
rack := fs.String("rack", "", "only machines in this rack")
|
||||
room := fs.String("room", "", "only machines in this room")
|
||||
bld := fs.String("building", "", "only machines in this building")
|
||||
_ = fs.Parse(args)
|
||||
ms, err := c.machines(*tag)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f := "%-16s %-14s %-10s %-16s %-14s %-24s %4s %7s %3s %7s %3s %2s %3s %s\n"
|
||||
fmt.Printf(f, "ID", "STATE", "NODE", "PROFILE/OS", "MGMT-IP", "MODEL", "CPU", "RAM", "DSK", "STORAGE", "NIC", "IB", "GPU", "BMC")
|
||||
f := "%-16s %-14s %-10s %-16s %-14s %-22s %-18s %4s %7s %3s %7s %3s %2s %3s %-18s %s\n"
|
||||
fmt.Printf(f, "ID", "STATE", "NODE", "PROFILE/OS", "MGMT-IP", "LOCATION", "MODEL", "CPU", "RAM", "DSK", "STORAGE", "NIC", "IB", "GPU", "BMC", "TAGS")
|
||||
for _, m := range ms {
|
||||
if (*rack != "" && !strings.EqualFold(m.Location.Rack, *rack)) || (*room != "" && !strings.EqualFold(m.Location.Room, *room)) || (*bld != "" && !strings.EqualFold(m.Location.Building, *bld)) {
|
||||
continue
|
||||
}
|
||||
state := m.State
|
||||
if m.Job != nil {
|
||||
var p obj
|
||||
@@ -122,8 +190,8 @@ func runClient(cmd string, args []string) error {
|
||||
bmc = fmt.Sprintf("%s (%s,%s)", str(b["address"]), str(b["source"]), ok)
|
||||
}
|
||||
model := strings.TrimSpace(deref(m.Manufacturer) + " " + deref(m.Product))
|
||||
fmt.Printf(f, m.ID, state, deref(m.NodeName), trunc(what, 16), deref(m.MgmtIP), trunc(model, 24), m.CPUs, human(m.MemoryBytes),
|
||||
fmt.Sprint(int(m.Disks)), human(m.StorageBytes), fmt.Sprint(m.NICs), fmt.Sprint(m.IBHCAs), fmt.Sprint(m.GPUs), bmc)
|
||||
fmt.Printf(f, m.ID, state, deref(m.NodeName), trunc(what, 16), deref(m.MgmtIP), trunc(m.Location.String(), 22), trunc(model, 18), m.CPUs, human(m.MemoryBytes),
|
||||
fmt.Sprint(int(m.Disks)), human(m.StorageBytes), fmt.Sprint(m.NICs), fmt.Sprint(m.IBHCAs), fmt.Sprint(m.GPUs), trunc(bmc, 18), strings.Join(m.Tags, ","))
|
||||
}
|
||||
case "show":
|
||||
_ = need(1)
|
||||
@@ -151,13 +219,26 @@ func runClient(cmd string, args []string) error {
|
||||
}
|
||||
case "power":
|
||||
_ = need(2)
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/machines/"+args[0]+"/power", obj{"action": args[1]}, &d); err != nil {
|
||||
ids, err := c.targets(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println(str(d["output"]))
|
||||
if ok, _ := d["ok"].(bool); !ok {
|
||||
return fmt.Errorf("ipmitool failed")
|
||||
failed := 0
|
||||
for _, id := range ids {
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/machines/"+id+"/power", obj{"action": args[1]}, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
if len(ids) > 1 {
|
||||
fmt.Printf("%s: ", id)
|
||||
}
|
||||
fmt.Println(str(d["output"]))
|
||||
if ok, _ := d["ok"].(bool); !ok {
|
||||
failed++
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("ipmitool failed on %d machine(s)", failed)
|
||||
}
|
||||
case "bootdev":
|
||||
_ = need(2)
|
||||
@@ -189,12 +270,156 @@ func runClient(cmd string, args []string) error {
|
||||
fmt.Println("deleted", str(d["deleted"]))
|
||||
case "deploy", "release", "jobs", "job":
|
||||
return c.deployCmd(cmd, args)
|
||||
case "tag", "tags":
|
||||
return c.tagCmd(cmd, args)
|
||||
case "location":
|
||||
return c.locationCmd(args)
|
||||
case "bmc":
|
||||
return c.bmcCmd(args)
|
||||
case "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp", "ipam":
|
||||
return c.netCmd(cmd, args)
|
||||
case "admin-cert":
|
||||
return c.adminCertCmd(args)
|
||||
default:
|
||||
return c.statelessCmd(cmd, args)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *client) machines(tag string) ([]Summary, error) {
|
||||
q := ""
|
||||
if tag != "" {
|
||||
q = "?tag=" + tag
|
||||
}
|
||||
var ms []Summary
|
||||
err := c.do("GET", "/api/v1/machines"+q, nil, &ms)
|
||||
return ms, err
|
||||
}
|
||||
|
||||
// targets expands "-tag NAME" (passed as a single "tag:NAME" or "-tag=NAME" token) or returns the id.
|
||||
func (c *client) targets(arg string) ([]string, error) {
|
||||
tag := ""
|
||||
switch {
|
||||
case strings.HasPrefix(arg, "tag:"):
|
||||
tag = strings.TrimPrefix(arg, "tag:")
|
||||
case strings.HasPrefix(arg, "-tag="):
|
||||
tag = strings.TrimPrefix(arg, "-tag=")
|
||||
default:
|
||||
return []string{arg}, nil
|
||||
}
|
||||
ms, err := c.machines(tag)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(ms) == 0 {
|
||||
return nil, fmt.Errorf("no machine has tag %s", tag)
|
||||
}
|
||||
var ids []string
|
||||
for _, m := range ms {
|
||||
ids = append(ids, m.ID)
|
||||
}
|
||||
return ids, nil
|
||||
}
|
||||
|
||||
func (c *client) tagCmd(cmd string, args []string) error {
|
||||
if cmd == "tag" {
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
req := obj{"add": []string{}, "remove": []string{}}
|
||||
var add, rm []string
|
||||
for _, t := range args[1:] {
|
||||
switch {
|
||||
case strings.HasPrefix(t, "+"):
|
||||
add = append(add, t[1:])
|
||||
case strings.HasPrefix(t, "-"):
|
||||
rm = append(rm, t[1:])
|
||||
default:
|
||||
add = append(add, t)
|
||||
}
|
||||
}
|
||||
req["add"], req["remove"] = add, rm
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/machines/"+args[0]+"/tags", req, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s: manual=%v auto=%v\n", str(d["id"]), d["tags"], d["auto_tags"])
|
||||
return nil
|
||||
}
|
||||
if len(args) < 1 {
|
||||
usage()
|
||||
}
|
||||
switch args[0] {
|
||||
case "list":
|
||||
var defs []TagDef
|
||||
if err := c.do("GET", "/api/v1/tags", nil, &defs); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-16s %5s %-32s %-16s %-8s %s\n", "TAG", "MACH", "MATCH", "OVERLAYS", "CLOUDCFG", "COMMENT")
|
||||
for _, t := range defs {
|
||||
cc := "-"
|
||||
if t.CloudConfig != "" {
|
||||
cc = fmt.Sprintf("%d B", len(t.CloudConfig))
|
||||
}
|
||||
fmt.Printf("%-16s %5d %-32s %-16s %-8s %s\n", t.Name, t.Machines, trunc(strings.ReplaceAll(t.Match, "\n", " "), 32), strings.Join(t.Overlays, ","), cc, t.Comment)
|
||||
}
|
||||
case "show":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
var defs []TagDef
|
||||
if err := c.do("GET", "/api/v1/tags", nil, &defs); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, t := range defs {
|
||||
if t.Name == args[1] {
|
||||
fmt.Printf("tag %s (%d machines)\ncomment: %s\nmatch: %s\noverlays: %v\nlayout: %s\n--- cloud_config ---\n%s\n", t.Name, t.Machines, t.Comment, t.Match, t.Overlays, orDash(t.NetLayout), t.CloudConfig)
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errNotFound
|
||||
case "add":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
fs := newFlags("tags add")
|
||||
match := fs.String("match", "", "Go template over .Node that renders true, e.g. '{{ gt .Node.GPUs 0 }}'")
|
||||
ov := fs.String("overlays", "", "comma-separated overlays for stateless nodes with this tag")
|
||||
cc := fs.String("cloud-config", "", "file with a #cloud-config or #!/bin/sh fragment applied at deployment")
|
||||
layout := fs.String("layout", "", "network layout for machines with this tag")
|
||||
comment := fs.String("comment", "", "comment")
|
||||
_ = fs.Parse(args[2:])
|
||||
t := TagDef{Name: args[1], Match: *match, Comment: *comment, NetLayout: *layout}
|
||||
if *ov != "" {
|
||||
t.Overlays = strings.Split(*ov, ",")
|
||||
}
|
||||
if *cc != "" {
|
||||
b, err := os.ReadFile(*cc)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
t.CloudConfig = string(b)
|
||||
}
|
||||
var out TagDef
|
||||
if err := c.do("POST", "/api/v1/tags", t, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("tag %s saved\n", out.Name)
|
||||
case "rm":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
var d obj
|
||||
if err := c.do("DELETE", "/api/v1/tags/"+args[1], nil, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Println("deleted", str(d["deleted"]))
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *client) deployCmd(cmd string, args []string) error {
|
||||
switch cmd {
|
||||
case "deploy":
|
||||
@@ -222,16 +447,29 @@ func (c *client) deployCmd(cmd string, args []string) error {
|
||||
}
|
||||
req["user_data"] = string(b)
|
||||
}
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/machines/"+args[0]+"/deploy", req, &d); err != nil {
|
||||
ids, err := c.targets(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s: job %s %s/%s -> %s hostname=%s mgmt=%s\n", str(d["id"]), str(d["job"]), str(d["os"]), str(d["version"]), str(d["disk"]), str(d["hostname"]), str(d["mgmt_ip"]))
|
||||
if d["ipmi"] != nil {
|
||||
fmt.Println(str(d["ipmi"]))
|
||||
if len(ids) > 1 && (*host != "" || *disk != "") {
|
||||
return fmt.Errorf("-hostname/-disk cannot be used with a tag target")
|
||||
}
|
||||
if *watch {
|
||||
return c.watchJob(args[0])
|
||||
for _, id := range ids {
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/machines/"+id+"/deploy", req, &d); err != nil {
|
||||
if len(ids) == 1 {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s: error: %v\n", id, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s: job %s %s/%s -> %s hostname=%s mgmt=%s\n", str(d["id"]), str(d["job"]), str(d["os"]), str(d["version"]), str(d["disk"]), str(d["hostname"]), str(d["mgmt_ip"]))
|
||||
if d["ipmi"] != nil {
|
||||
fmt.Println(str(d["ipmi"]))
|
||||
}
|
||||
}
|
||||
if *watch && len(ids) == 1 {
|
||||
return c.watchJob(ids[0])
|
||||
}
|
||||
case "release":
|
||||
if len(args) < 1 {
|
||||
@@ -465,12 +703,42 @@ func (c *client) statelessCmd(cmd string, args []string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func readFileArg(p string) ([]byte, error) {
|
||||
if p == "-" {
|
||||
return io.ReadAll(os.Stdin)
|
||||
}
|
||||
return os.ReadFile(p)
|
||||
}
|
||||
|
||||
func (c *client) rawBody(method, path, ctype string, body []byte) ([]byte, error) {
|
||||
req, err := http.NewRequest(method, c.base+path, bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
req.Header.Set("Content-Type", ctype)
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode >= 400 {
|
||||
return nil, fmt.Errorf("%s: %s", resp.Status, strings.TrimSpace(string(data)))
|
||||
}
|
||||
return data, nil
|
||||
}
|
||||
|
||||
func (c *client) raw(method, path string) ([]byte, error) {
|
||||
req, err := http.NewRequest(method, c.base+path, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
if c.token != "" {
|
||||
req.Header.Set("Authorization", "Bearer "+c.token)
|
||||
}
|
||||
resp, err := c.http.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
@@ -602,3 +870,113 @@ func asList(v any) []any {
|
||||
}
|
||||
return []any{v}
|
||||
}
|
||||
|
||||
func (c *client) bmcCmd(args []string) error {
|
||||
if len(args) < 1 {
|
||||
usage()
|
||||
}
|
||||
if args[0] == "drivers" {
|
||||
var ds []powerDriver
|
||||
if err := c.do("GET", "/api/v1/power-drivers", nil, &ds); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, d := range ds {
|
||||
fmt.Printf("%-11s %-8s %s\n", d.Name, d.Family, d.Description)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
fs := newFlags("bmc")
|
||||
driver := fs.String("driver", "", "power driver (rollcall bmc drivers)")
|
||||
address := fs.String("address", "", "BMC address (host, IP or https://host:port)")
|
||||
port := fs.Int("port", 0, "port (0 = the driver's default)")
|
||||
user := fs.String("user", "", "account (proxmox: token id user@realm!name)")
|
||||
password := fs.String("password", "", "password (proxmox: token secret); - reads it from stdin")
|
||||
verify := fs.Bool("verify", false, "check with a status call before saving")
|
||||
probe := fs.Bool("probe", false, "re-run Redfish detection with the stored account")
|
||||
var sets multiFlag
|
||||
fs.Var(&sets, "set", "driver setting KEY=VALUE, repeatable (system, verify_tls, cipher, privilege, interface, node, vmid); KEY= removes it")
|
||||
_ = fs.Parse(args[1:])
|
||||
id := args[0]
|
||||
req := obj{}
|
||||
set := func(k, v string) {
|
||||
if v != "" {
|
||||
req[k] = v
|
||||
}
|
||||
}
|
||||
set("driver", *driver)
|
||||
set("address", *address)
|
||||
set("user", *user)
|
||||
if *port != 0 {
|
||||
req["port"] = *port
|
||||
}
|
||||
if *password == "-" {
|
||||
b, err := io.ReadAll(os.Stdin)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req["password"] = strings.TrimRight(string(b), "\r\n")
|
||||
} else if *password != "" {
|
||||
req["password"] = *password
|
||||
}
|
||||
if len(sets) > 0 {
|
||||
ex := map[string]string{}
|
||||
for _, s := range sets {
|
||||
k, v, _ := strings.Cut(s, "=")
|
||||
ex[k] = v
|
||||
}
|
||||
req["extra"] = ex
|
||||
}
|
||||
var d obj
|
||||
switch {
|
||||
case *probe:
|
||||
if err := c.do("POST", "/api/v1/machines/"+id+"/bmc", obj{"probe": true}, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
case len(req) == 0:
|
||||
if err := c.do("GET", "/api/v1/machines/"+id+"/bmc", nil, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
default:
|
||||
req["verify"] = *verify
|
||||
if err := c.do("POST", "/api/v1/machines/"+id+"/bmc", req, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
b, _ := d["bmc"].(obj)
|
||||
drv := str(b["driver"])
|
||||
if drv == "" {
|
||||
drv = "ipmi"
|
||||
}
|
||||
fmt.Printf("%s driver %s address %s port %s user %s source %s%s\n", id, drv, str(b["address"]), orDash(str(b["port"])), str(b["user"]), str(b["source"]),
|
||||
map[bool]string{true: " (set by operator)", false: ""}[b["driver_set"] == true])
|
||||
if ex, ok := b["extra"].(obj); ok && len(ex) > 0 {
|
||||
var kv []string
|
||||
for k, v := range ex {
|
||||
kv = append(kv, k+"="+str(v))
|
||||
}
|
||||
sort.Strings(kv)
|
||||
fmt.Printf(" settings: %s\n", strings.Join(kv, " "))
|
||||
}
|
||||
if rf, ok := b["redfish"].(obj); ok {
|
||||
fmt.Printf(" redfish %s · %s %s · %s\n", str(rf["redfish"]), str(rf["manufacturer"]), str(rf["model"]), str(rf["system"]))
|
||||
}
|
||||
if out := str(d["output"]); out != "" {
|
||||
fmt.Println(" " + out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// localListen turns a listen address into one this host can dial.
|
||||
func localListen(listen string) string {
|
||||
host, port, err := net.SplitHostPort(listen)
|
||||
if err != nil {
|
||||
return listen
|
||||
}
|
||||
switch host {
|
||||
case "", "0.0.0.0":
|
||||
host = "127.0.0.1"
|
||||
case "::":
|
||||
host = "::1"
|
||||
}
|
||||
return net.JoinHostPort(host, port)
|
||||
}
|
||||
+44
-4
@@ -3,6 +3,7 @@ package main
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
@@ -17,7 +18,7 @@ type Config struct {
|
||||
MasterKeyFile string `json:"master_key_file"` // 32-byte hex key for AES-GCM at-rest encryption
|
||||
AdminToken string `json:"admin_token"` // bearer token for the management API / web UI
|
||||
AgentToken string `json:"agent_token"` // X-Rollcall-Token for the agent endpoints ("" = open)
|
||||
CommissionAppend string `json:"commission_append"` // kernel cmdline used in generated pxelinux files
|
||||
CommissionAppend string `json:"commission_append"` // kernel cmdline used in generated pxelinux files ("" = derived from server_url)
|
||||
IPMITool string `json:"ipmitool"` // path, default "ipmitool"
|
||||
BMCHints []BMCHint `json:"bmc_hints"` // out-of-band BMC mapping for machines without in-band IPMI
|
||||
|
||||
@@ -32,6 +33,26 @@ type Config struct {
|
||||
// disk-based deployment
|
||||
OSDir string `json:"os_dir"` // <os_dir>/<name>/<version>/{disk.raw.zst,meta.json}, served under /os/
|
||||
OSSources string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync`
|
||||
|
||||
// region/rack: this server runs an embedded rack controller named RackName;
|
||||
// remote rack controllers (rollcall rack) authenticate with RackSecret
|
||||
RackName string `json:"rack_name"`
|
||||
RackSecret string `json:"rack_secret"` // legacy shared secret; rack controllers join with a code and mutual TLS now
|
||||
// rollcall-linux-<arch> for rack controllers on another architecture than the region
|
||||
RackBinaries string `json:"rack_binaries"`
|
||||
|
||||
// certificate authority and HTTPS
|
||||
CADir string `json:"ca_dir"` // ca.pem, ca-key.pem, region.pem, region-key.pem (default /etc/rollcall/ca)
|
||||
TLSListen string `json:"tls_listen"` // HTTPS listener, default ":443"; "off" disables
|
||||
TLSNames []string `json:"tls_names"` // extra names/addresses for the region certificate
|
||||
AdminHTTP string `json:"admin_http"` // admin API over plain HTTP: allow (default) | loopback | deny
|
||||
|
||||
// a deploy/release job the machine has not started by then fails (default 30m;
|
||||
// servers can take 4-5 minutes to POST, longer when many boot together)
|
||||
JobStartTimeout string `json:"job_start_timeout"`
|
||||
|
||||
// external IPAM authority (NetBox; Infoblox at sites that run it)
|
||||
IPAM IPAMConfig `json:"ipam"`
|
||||
}
|
||||
|
||||
// BMCHint maps a machine MAC to a known BMC (virtual lab nodes).
|
||||
@@ -59,13 +80,17 @@ func loadConfig(path string) (*Config, error) {
|
||||
}
|
||||
}
|
||||
if c.ServerURL == "" {
|
||||
host, _, _ := strings.Cut(c.Listen, ":")
|
||||
if host == "" || host == "0.0.0.0" {
|
||||
host, _, err := net.SplitHostPort(c.Listen)
|
||||
if err != nil || host == "" || host == "0.0.0.0" || host == "::" {
|
||||
return nil, fmt.Errorf("%s: server_url is required when listen is not a specific address", path)
|
||||
}
|
||||
c.ServerURL = "http://" + c.Listen
|
||||
c.ServerURL = "http://" + c.Listen // host:port, IPv6 already bracketed ([::1]:80)
|
||||
}
|
||||
c.ServerURL = strings.TrimRight(c.ServerURL, "/")
|
||||
if c.CommissionAppend == "" { // follows server_url (it often changes once the provisioning network is known)
|
||||
c.CommissionAppend = fmt.Sprintf("boot=live components fetch=%s/rollcall/filesystem.squashfs rollcall.server=%s noautologin rollcall.token=%s console=tty0 console=ttyS0,115200n8",
|
||||
c.ServerURL, c.ServerURL, c.AgentToken)
|
||||
}
|
||||
if c.ImagesDir == "" {
|
||||
c.ImagesDir = filepath.Join(filepath.Dir(c.ImageDir), "images")
|
||||
}
|
||||
@@ -78,5 +103,20 @@ func loadConfig(path string) (*Config, error) {
|
||||
if c.OSSources == "" {
|
||||
c.OSSources = "/etc/rollcall/os-sources.json"
|
||||
}
|
||||
if c.RackName == "" {
|
||||
c.RackName, _ = os.Hostname()
|
||||
}
|
||||
if c.CADir == "" {
|
||||
c.CADir = "/etc/rollcall/ca"
|
||||
}
|
||||
switch c.TLSListen {
|
||||
case "":
|
||||
c.TLSListen = ":443"
|
||||
case "off", "none":
|
||||
c.TLSListen = ""
|
||||
}
|
||||
if c.AdminHTTP == "" {
|
||||
c.AdminHTTP = "allow"
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
+147
-17
@@ -49,6 +49,8 @@ type DeploySpec struct {
|
||||
Packages []string `json:"packages,omitempty"`
|
||||
UserData string `json:"user_data,omitempty"` // full #cloud-config override
|
||||
Wipe string `json:"wipe,omitempty"` // release: quick | full
|
||||
NetworkConfig string `json:"network_config,omitempty"`
|
||||
LinkRules string `json:"link_rules,omitempty"`
|
||||
}
|
||||
|
||||
type seedFile struct {
|
||||
@@ -131,6 +133,24 @@ func (s *Store) Jobs(ctx context.Context, machineID string, limit int) ([]*Job,
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// RecentJobs lists jobs across all machines, newest first (active ones first when activeFirst).
|
||||
func (s *Store) RecentJobs(ctx context.Context, limit int) ([]*Job, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT `+jobCols+` FROM jobs ORDER BY (state IN ('pending','running')) DESC, created DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []*Job{}
|
||||
for rows.Next() {
|
||||
j, err := scanJob(rows)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, j)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) JobProgress(ctx context.Context, id string, progress []byte) error {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE jobs SET state='running', started=coalesce(started, now()), progress=$2 WHERE id=$1 AND state IN ('pending','running')`, id, progress)
|
||||
return err
|
||||
@@ -147,7 +167,11 @@ func (s *Store) SetDeployed(ctx context.Context, id, os, version string) error {
|
||||
}
|
||||
|
||||
func (s *Store) SetReleased(ctx context.Context, id string) error {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE machines SET state='ready', os=NULL, os_version=NULL, deployed_at=NULL, booted_at=NULL, last_seen=now() WHERE id=$1`, id)
|
||||
if _, err := s.pool.Exec(ctx, `DELETE FROM ip_addresses WHERE machine_id=$1 AND kind='auto'`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `UPDATE machines SET state='ready', os=NULL, os_version=NULL, deployed_at=NULL, booted_at=NULL, last_seen=now(),
|
||||
mgmt_ip=(SELECT ip FROM ip_addresses a WHERE a.machine_id=machines.id ORDER BY ip LIMIT 1), ib_ip=NULL WHERE id=$1`, id)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -235,6 +259,11 @@ func yamlList(l []string) string {
|
||||
}
|
||||
|
||||
func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile {
|
||||
base := a.cfg.ServerURL
|
||||
m, merr := a.store.Get(context.Background(), j.MachineID)
|
||||
if merr == nil {
|
||||
base = a.urlFor(m)
|
||||
}
|
||||
meta := fmt.Sprintf("instance-id: %s\nlocal-hostname: %s\n", j.ID, sp.Hostname)
|
||||
ud := sp.UserData
|
||||
if ud == "" {
|
||||
@@ -256,15 +285,44 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile {
|
||||
if len(sp.Packages) > 0 {
|
||||
fmt.Fprintf(&b, "packages: %s\n", yamlList(sp.Packages))
|
||||
}
|
||||
fmt.Fprintf(&b, "phone_home:\n url: %s/api/v1/phone-home/%s\n post: [instance_id, hostname, fqdn, pub_key_ed25519]\n tries: 10\n", a.cfg.ServerURL, token)
|
||||
b.WriteString("final_message: \"rollcall: deployed $hostname up after $UPTIME s\"\n")
|
||||
fmt.Fprintf(&b, "phone_home:\n url: %s/api/v1/phone-home/%s\n post: [instance_id, hostname, fqdn, pub_key_ed25519]\n tries: 10\n", base, token)
|
||||
b.WriteString("final_message: \"rollcall: deployed, up after $UPTIME s\"\n") // only $UPTIME/$TIMESTAMP/$DATASOURCE/$VERSION are expanded
|
||||
ud = b.String()
|
||||
}
|
||||
return []seedFile{
|
||||
var frags []TagDef
|
||||
if a.ca != nil {
|
||||
// The CA file itself is written into the image (seed files below); this rebuilds
|
||||
// the trust store on every boot, early, with whatever the distribution has.
|
||||
// cloud-init's ca_certs runs update-ca-certificates even on RHEL-family systems
|
||||
// and fails there, so it is not used.
|
||||
frags = append(frags, TagDef{Name: "rollcall-ca", CloudConfig: "#cloud-config\n# rollcall: trust the installation's certificate authority\nbootcmd:\n" +
|
||||
" - [sh, -c, 'if command -v update-ca-trust >/dev/null 2>&1; then update-ca-trust extract; elif command -v update-ca-certificates >/dev/null 2>&1; then update-ca-certificates; fi']\n"})
|
||||
}
|
||||
if merr == nil {
|
||||
if tf, err := a.tagFragments(context.Background(), m); err == nil {
|
||||
frags = append(frags, tf...)
|
||||
}
|
||||
}
|
||||
if len(frags) > 0 {
|
||||
ud = multipartUserData(ud, frags)
|
||||
}
|
||||
files := []seedFile{
|
||||
{Path: "var/lib/cloud/seed/nocloud/meta-data", Mode: "0600", Content: meta},
|
||||
{Path: "var/lib/cloud/seed/nocloud/user-data", Mode: "0600", Content: ud},
|
||||
{Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\n"},
|
||||
}
|
||||
if a.ca != nil { // both layouts; the bootcmd above picks the one the distribution uses
|
||||
files = append(files,
|
||||
seedFile{Path: "usr/local/share/ca-certificates/rollcall-ca.crt", Mode: "0644", Content: string(a.ca.PEM)},
|
||||
seedFile{Path: "etc/pki/ca-trust/source/anchors/rollcall-ca.crt", Mode: "0644", Content: string(a.ca.PEM)})
|
||||
}
|
||||
if sp.NetworkConfig != "" {
|
||||
files = append(files, seedFile{Path: "var/lib/cloud/seed/nocloud/network-config", Mode: "0600", Content: sp.NetworkConfig})
|
||||
}
|
||||
if sp.LinkRules != "" {
|
||||
files = append(files, seedFile{Path: "etc/udev/rules.d/70-rollcall-net.rules", Mode: "0644", Content: sp.LinkRules})
|
||||
}
|
||||
return files
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- PXE
|
||||
@@ -314,9 +372,9 @@ LABEL recommission
|
||||
`
|
||||
|
||||
// deployAppend derives the deploy-mode cmdline from commission_append (same image, no agent token).
|
||||
func (a *App) deployAppend(token string) string {
|
||||
func (a *App) deployAppend(base, token string) string {
|
||||
var parts []string
|
||||
for _, p := range strings.Fields(a.cfg.CommissionAppend) {
|
||||
for _, p := range strings.Fields(a.appendFor(base)) {
|
||||
if strings.HasPrefix(p, "rollcall.token=") || strings.HasPrefix(p, "rollcall.nopoweroff") {
|
||||
continue
|
||||
}
|
||||
@@ -328,13 +386,13 @@ func (a *App) deployAppend(token string) string {
|
||||
|
||||
func (a *App) writePXEDeploy(m *Machine, j *Job, sp DeploySpec, token string) error {
|
||||
body := fmt.Sprintf(deployTmpl, time.Now().Format(time.RFC3339), m.ID, j.ID, j.Kind, sp.OS, sp.Version, sp.Disk,
|
||||
m.ID, j.Kind, sp.OS, j.ID, a.deployAppend(token), a.cfg.CommissionAppend)
|
||||
m.ID, j.Kind, sp.OS, j.ID, a.deployAppend(a.urlFor(m), token), a.appendFor(a.urlFor(m)))
|
||||
return a.writePXEBody(m, body)
|
||||
}
|
||||
|
||||
func (a *App) writePXEDeployed(m *Machine) error {
|
||||
body := fmt.Sprintf(deployedTmpl, time.Now().Format(time.RFC3339), m.ID, deref(m.OS), deref(m.OSVersion),
|
||||
deref(m.NodeName), deref(m.OS), m.ID, deref(m.OS), a.cfg.CommissionAppend)
|
||||
deref(m.NodeName), deref(m.OS), m.ID, deref(m.OS), a.appendFor(a.urlFor(m)))
|
||||
return a.writePXEBody(m, body)
|
||||
}
|
||||
|
||||
@@ -420,40 +478,64 @@ func (a *App) handleDeploy(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
// node name + management IP: keep an existing one, otherwise allocate like a stateless node
|
||||
// node name: keep the existing one unless another is asked for
|
||||
name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP)
|
||||
if req.Hostname != "" && req.Hostname != name {
|
||||
name = ""
|
||||
}
|
||||
if name == "" {
|
||||
if name, mgmt, ib, err = a.allocate(r.Context(), str(a.cfg.ClusterVars["name_format"]), req.Hostname); err != nil {
|
||||
if name, _, _, err = a.allocatePools(r.Context(), str(a.cfg.ClusterVars["name_format"]), req.Hostname, false); err != nil {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
if err := a.store.SetNodeAddr(r.Context(), m.ID, name, mgmt, ib); err != nil {
|
||||
fail(w, 500, err)
|
||||
m.NodeName = &name
|
||||
}
|
||||
// addresses: the network plan (layout, IPAM); without a network model, the legacy pools
|
||||
netcfg, links := "", ""
|
||||
var warnings []string
|
||||
if len(a.subnetsSnapshot()) > 0 {
|
||||
plan, err := a.netPlan(r.Context(), m, true)
|
||||
if err != nil {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
if err := a.exportDnsmasq(r.Context()); err != nil {
|
||||
log.Printf("dnsmasq export: %v", err)
|
||||
warnings = plan.Warnings
|
||||
if pm, pib := plan.primaryIPs(); pm != "" {
|
||||
mgmt, ib = pm, pib
|
||||
}
|
||||
nb, _ := json.MarshalIndent(networkConfig(plan), "", " ")
|
||||
netcfg = string(nb) + "\n"
|
||||
links = linkRules(plan)
|
||||
} else if mgmt == "" {
|
||||
if _, mgmt, ib, err = a.allocatePools(r.Context(), "", name, true); err != nil {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := a.store.SetNodeAddr(r.Context(), m.ID, name, mgmt, ib); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
keys := req.SSHKeys
|
||||
if keys == nil {
|
||||
keys = a.configSSHKeys()
|
||||
}
|
||||
sp := DeploySpec{OS: meta.Name, Version: meta.Version,
|
||||
ImageURL: a.cfg.ServerURL + "/os/" + meta.Name + "/" + meta.Version + "/disk.raw.zst",
|
||||
ImageURL: a.urlFor(m) + "/os/" + meta.Name + "/" + meta.Version + "/disk.raw.zst",
|
||||
SHA256: meta.SHA256, Size: meta.Size, RawSize: meta.RawSize, Disk: disk.Path, DiskSerial: disk.Serial,
|
||||
Hostname: name, Domain: str(a.cfg.ClusterVars["domain"]), SSHKeys: keys, PackageUpgrade: req.PackageUpgrade,
|
||||
Packages: req.Packages, UserData: req.UserData}
|
||||
Packages: req.Packages, UserData: req.UserData, NetworkConfig: netcfg, LinkRules: links}
|
||||
out, err := a.startJob(r.Context(), m, "deploy", sp, req.Boot)
|
||||
if err != nil {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
out["mgmt_ip"] = mgmt
|
||||
out["rack"] = a.rackOf(m)
|
||||
if len(warnings) > 0 {
|
||||
out["warnings"] = warnings
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
|
||||
@@ -525,6 +607,7 @@ func (a *App) afterJob(ctx context.Context, m *Machine, j *Job, state, errMsg st
|
||||
_ = a.store.AddEvent(ctx, m.ID, "deployed", fmt.Sprintf("job=%s os=%s/%s disk=%s", j.ID, sp.OS, sp.Version, sp.Disk))
|
||||
case state == "done" && j.Kind == "release":
|
||||
_ = a.store.SetReleased(ctx, m.ID)
|
||||
a.kickIPAM()
|
||||
m, _ = a.store.Get(ctx, m.ID)
|
||||
_ = a.writePXE(m)
|
||||
_ = a.store.AddEvent(ctx, m.ID, "released", fmt.Sprintf("job=%s disk=%s wipe=%s", j.ID, sp.Disk, sp.Wipe))
|
||||
@@ -652,3 +735,50 @@ func (a *App) handlePhoneHome(w http.ResponseWriter, r *http.Request) {
|
||||
log.Printf("phone-home %s: %s", j.MachineID, detail)
|
||||
writeJSON(w, 200, obj{"ok": true})
|
||||
}
|
||||
|
||||
// jobStartTimeout: a deploy or release job the machine has not picked up by then
|
||||
// is failed, so a lost power command or a machine that did not PXE-boot is
|
||||
// visible instead of an endless "pending". Real servers spend 4-5 minutes in
|
||||
// POST on a cold start (more after a firmware or memory change), and many
|
||||
// machines booting at once share the PXE and image servers: the default leaves
|
||||
// room for that; job_start_timeout in config.json changes it.
|
||||
const jobStartTimeout = 30 * time.Minute
|
||||
|
||||
func (a *App) jobStartTimeout() time.Duration {
|
||||
if d, err := time.ParseDuration(a.cfg.JobStartTimeout); err == nil && d >= time.Minute {
|
||||
return d
|
||||
}
|
||||
return jobStartTimeout
|
||||
}
|
||||
|
||||
// staleJobs picks the pending jobs created before cutoff.
|
||||
func staleJobs(active map[string]*Job, cutoff time.Time) []*Job {
|
||||
var out []*Job
|
||||
for _, j := range active {
|
||||
if j.State == "pending" && j.Created.Before(cutoff) {
|
||||
out = append(out, j)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, k int) bool { return out[i].Created.Before(out[k].Created) })
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *App) jobWatchdog() {
|
||||
for range time.Tick(time.Minute) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
active, err := a.store.ActiveJobs(ctx)
|
||||
if err == nil {
|
||||
limit := a.jobStartTimeout()
|
||||
for _, j := range staleJobs(active, time.Now().Add(-limit)) {
|
||||
m, err := a.store.Get(ctx, j.MachineID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
msg := fmt.Sprintf("the machine did not start the job within %s: check its power, that it PXE-boots on the provisioning network, and its console for a hung POST (events show the power command's result; job_start_timeout raises the limit)", limit)
|
||||
_ = a.store.FinishJob(ctx, j.ID, "failed", msg)
|
||||
a.afterJob(ctx, m, j, "failed", msg)
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,324 @@
|
||||
package main
|
||||
|
||||
// rollcall net discover: what this host sees on each interface, for the day
|
||||
// when the provisioning network is not decided yet. Addresses (and the subnet
|
||||
// they imply), the switch port and VLAN from LLDP (lldpd), DHCP servers that
|
||||
// answer, neighbours; then a suggestion of what to do with each interface.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type discoIface struct {
|
||||
Name string `json:"name"`
|
||||
Kind string `json:"kind"` // physical | vlan | bond | bridge | virtual
|
||||
MAC string `json:"mac"`
|
||||
Up bool `json:"up"`
|
||||
SpeedMbps int `json:"speed_mbps,omitempty"`
|
||||
Addrs []string `json:"addrs"`
|
||||
Subnets []string `json:"subnets"`
|
||||
Default bool `json:"default_route"`
|
||||
Switch string `json:"switch,omitempty"`
|
||||
Port string `json:"port,omitempty"`
|
||||
VLAN string `json:"vlan,omitempty"`
|
||||
DHCP []string `json:"dhcp_servers"`
|
||||
DHCPProbe string `json:"dhcp_probe,omitempty"` // why it was not probed
|
||||
Neighbours int `json:"neighbours"`
|
||||
Suggestion string `json:"suggestion"`
|
||||
}
|
||||
|
||||
func discoverCmd(args []string) error {
|
||||
fs := flag.NewFlagSet("net discover", flag.ExitOnError)
|
||||
asJSON := fs.Bool("json", false, "JSON output")
|
||||
noProbe := fs.Bool("no-dhcp-probe", false, "do not send DHCPDISCOVERs")
|
||||
_ = fs.Parse(args)
|
||||
ifs := discoverInterfaces(!*noProbe)
|
||||
if *asJSON {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(ifs)
|
||||
}
|
||||
fmt.Printf("%-12s %-8s %-5s %6s %-22s %-28s %-14s %s\n", "INTERFACE", "KIND", "LINK", "SPEED", "ADDRESSES", "SWITCH / PORT / VLAN", "DHCP SERVERS", "NEIGH")
|
||||
for _, i := range ifs {
|
||||
link := "down"
|
||||
if i.Up {
|
||||
link = "up"
|
||||
}
|
||||
speed := "-"
|
||||
if i.SpeedMbps > 0 {
|
||||
speed = fmt.Sprintf("%dG", i.SpeedMbps/1000)
|
||||
if i.SpeedMbps < 1000 {
|
||||
speed = fmt.Sprintf("%dM", i.SpeedMbps)
|
||||
}
|
||||
}
|
||||
sw := "-"
|
||||
if i.Switch != "" || i.Port != "" {
|
||||
sw = strings.Trim(i.Switch+" / "+i.Port+" / "+orDash(i.VLAN), " /")
|
||||
}
|
||||
dh := strings.Join(i.DHCP, ",")
|
||||
if dh == "" {
|
||||
dh = orDash(i.DHCPProbe)
|
||||
}
|
||||
name := i.Name
|
||||
if i.Default {
|
||||
name += "*"
|
||||
}
|
||||
fmt.Printf("%-12s %-8s %-5s %6s %-22s %-28s %-14s %d\n", name, i.Kind, link, speed, trunc(strings.Join(i.Addrs, " "), 22), trunc(sw, 28), trunc(dh, 14), i.Neighbours)
|
||||
}
|
||||
fmt.Println("(* carries the default route)")
|
||||
fmt.Println()
|
||||
for _, i := range ifs {
|
||||
if i.Suggestion != "" {
|
||||
fmt.Printf("%-12s %s\n", i.Name, i.Suggestion)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func discoverInterfaces(probe bool) []discoIface {
|
||||
def := defaultRouteIface()
|
||||
lldp := lldpNeighbours()
|
||||
nets, _ := net.Interfaces()
|
||||
var out []discoIface
|
||||
type probeRes struct {
|
||||
servers []string
|
||||
err error
|
||||
}
|
||||
probes := map[string]chan probeRes{}
|
||||
if probe { // in parallel: a DHCP server may take seconds to offer (it pings the address first)
|
||||
for _, n := range nets {
|
||||
if n.Flags&net.FlagLoopback != 0 || n.Flags&net.FlagUp == 0 || skipIface(n.Name) || ifaceKind(n.Name) == "bridge" || ifaceKind(n.Name) == "virtual" {
|
||||
continue
|
||||
}
|
||||
ch := make(chan probeRes, 1)
|
||||
probes[n.Name] = ch
|
||||
go func(name string) { s, err := dhcpProbe(name, 6*time.Second); ch <- probeRes{s, err} }(n.Name)
|
||||
}
|
||||
}
|
||||
for _, n := range nets {
|
||||
if n.Flags&net.FlagLoopback != 0 || skipIface(n.Name) {
|
||||
continue
|
||||
}
|
||||
d := discoIface{Name: n.Name, MAC: n.HardwareAddr.String(), Kind: ifaceKind(n.Name), Default: n.Name == def, Addrs: []string{}, Subnets: []string{}, DHCP: []string{}}
|
||||
if b, err := os.ReadFile("/sys/class/net/" + n.Name + "/operstate"); err == nil {
|
||||
d.Up = strings.TrimSpace(string(b)) == "up"
|
||||
}
|
||||
if b, err := os.ReadFile("/sys/class/net/" + n.Name + "/speed"); err == nil {
|
||||
if v, err := strconv.Atoi(strings.TrimSpace(string(b))); err == nil && v > 0 {
|
||||
d.SpeedMbps = v
|
||||
}
|
||||
}
|
||||
addrs, _ := n.Addrs()
|
||||
for _, a := range addrs {
|
||||
ipn, ok := a.(*net.IPNet)
|
||||
if !ok || ipn.IP.IsLinkLocalUnicast() {
|
||||
continue
|
||||
}
|
||||
ones, _ := ipn.Mask.Size()
|
||||
d.Addrs = append(d.Addrs, fmt.Sprintf("%s/%d", ipn.IP, ones))
|
||||
d.Subnets = append(d.Subnets, (&net.IPNet{IP: ipn.IP.Mask(ipn.Mask), Mask: ipn.Mask}).String())
|
||||
}
|
||||
if l, ok := lldp[n.Name]; ok {
|
||||
d.Switch, d.Port, d.VLAN = l[0], l[1], l[2]
|
||||
}
|
||||
d.Neighbours = neighbourCount(n.Name)
|
||||
switch {
|
||||
case !probe:
|
||||
d.DHCPProbe = "not probed"
|
||||
case !d.Up:
|
||||
d.DHCPProbe = "link down"
|
||||
case d.Kind == "bridge" || d.Kind == "virtual":
|
||||
d.DHCPProbe = "skipped"
|
||||
case probes[n.Name] == nil:
|
||||
d.DHCPProbe = "skipped"
|
||||
default:
|
||||
r := <-probes[n.Name]
|
||||
if s, err := r.servers, r.err; err != nil {
|
||||
d.DHCPProbe = "needs root"
|
||||
} else {
|
||||
for _, x := range s {
|
||||
if ifaceWithAddr(x) == "" {
|
||||
d.DHCP = append(d.DHCP, x)
|
||||
} else {
|
||||
d.DHCP = append(d.DHCP, x+"(this host)")
|
||||
}
|
||||
}
|
||||
if len(s) == 0 {
|
||||
d.DHCPProbe = "none"
|
||||
}
|
||||
}
|
||||
}
|
||||
d.Suggestion = discoSuggest(d)
|
||||
out = append(out, d)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool { return out[i].Name < out[j].Name })
|
||||
return out
|
||||
}
|
||||
|
||||
func discoSuggest(d discoIface) string {
|
||||
foreign := 0
|
||||
for _, s := range d.DHCP {
|
||||
if !strings.HasSuffix(s, "(this host)") {
|
||||
foreign++
|
||||
}
|
||||
}
|
||||
v4 := ""
|
||||
for _, s := range d.Subnets {
|
||||
if !strings.Contains(s, ":") {
|
||||
v4 = s
|
||||
break
|
||||
}
|
||||
}
|
||||
vlan := ""
|
||||
if d.VLAN != "" {
|
||||
vlan = " (switch says VLAN " + d.VLAN + ")"
|
||||
}
|
||||
switch {
|
||||
case d.Kind == "bridge" || d.Kind == "virtual":
|
||||
return ""
|
||||
case !d.Up:
|
||||
return "no link: cable it to the provisioning network" + vlan
|
||||
case d.Default && foreign > 0:
|
||||
return "uplink: management network with its own DHCP (" + strings.Join(d.DHCP, ",") + "); keep it for the region's access" + vlan
|
||||
case d.Default:
|
||||
return "uplink: the region is reached here" + vlan
|
||||
case v4 == "":
|
||||
return "link up, no IPv4 address" + vlan + ": give it its address on the provisioning network, then bootstrap.sh --iface " + d.Name
|
||||
case foreign > 0:
|
||||
return v4 + " already has DHCP (" + strings.Join(d.DHCP, ",") + "): either the network team turns it off for this VLAN, or keep it and point it at this host (next-server " + strings.Split(d.Addrs[0], "/")[0] + ", or a DHCP relay); then bootstrap.sh --iface " + d.Name + " --no-dhcp"
|
||||
default:
|
||||
return "candidate provisioning network " + v4 + vlan + ": bootstrap.sh --iface " + d.Name + " (or Network -> New subnet in the web UI)"
|
||||
}
|
||||
}
|
||||
|
||||
func skipIface(name string) bool {
|
||||
for _, p := range []string{"docker", "veth", "virbr", "br-", "cni", "flannel", "tap", "vnet", "wg", "tun"} {
|
||||
if strings.HasPrefix(name, p) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func ifaceKind(name string) string {
|
||||
base := "/sys/class/net/" + name
|
||||
switch {
|
||||
case exists(base + "/bridge"):
|
||||
return "bridge"
|
||||
case exists(base + "/bonding"):
|
||||
return "bond"
|
||||
case exists("/proc/net/vlan/" + name):
|
||||
return "vlan"
|
||||
case exists(base + "/device"):
|
||||
return "physical"
|
||||
}
|
||||
return "virtual"
|
||||
}
|
||||
|
||||
func exists(p string) bool { _, err := os.Stat(p); return err == nil }
|
||||
|
||||
func defaultRouteIface() string {
|
||||
b, err := os.ReadFile("/proc/net/route")
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
for _, l := range strings.Split(string(b), "\n")[1:] {
|
||||
f := strings.Fields(l)
|
||||
if len(f) > 2 && f[1] == "00000000" && f[7] == "00000000" {
|
||||
return f[0]
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func neighbourCount(iface string) int {
|
||||
out, err := exec.Command("ip", "-j", "neigh", "show", "dev", iface).Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
var ns []map[string]any
|
||||
_ = json.Unmarshal(out, &ns)
|
||||
n := 0
|
||||
for _, x := range ns {
|
||||
if st, _ := x["state"].([]any); len(st) > 0 && st[0] != "FAILED" && st[0] != "INCOMPLETE" {
|
||||
n++
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// lldpNeighbours: interface -> [switch name, port, vlan] from lldpd.
|
||||
func lldpNeighbours() map[string][3]string {
|
||||
out := map[string][3]string{}
|
||||
bin, err := exec.LookPath("lldpcli")
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
b, err := exec.Command(bin, "-f", "json0", "show", "neighbors", "details").Output()
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
var doc struct {
|
||||
LLDP []struct {
|
||||
Interface []struct {
|
||||
Name string `json:"name"`
|
||||
Chassis []struct {
|
||||
Name []struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"name"`
|
||||
} `json:"chassis"`
|
||||
Port []struct {
|
||||
ID []struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"id"`
|
||||
Descr []struct {
|
||||
Value string `json:"value"`
|
||||
} `json:"descr"`
|
||||
} `json:"port"`
|
||||
VLAN []struct {
|
||||
VlanID string `json:"vlan-id"`
|
||||
PVID bool `json:"pvid"`
|
||||
} `json:"vlan"`
|
||||
} `json:"interface"`
|
||||
} `json:"lldp"`
|
||||
}
|
||||
if json.Unmarshal(b, &doc) != nil {
|
||||
return out
|
||||
}
|
||||
for _, l := range doc.LLDP {
|
||||
for _, i := range l.Interface {
|
||||
var sw, port, vlan string
|
||||
if len(i.Chassis) > 0 && len(i.Chassis[0].Name) > 0 {
|
||||
sw = i.Chassis[0].Name[0].Value
|
||||
}
|
||||
if len(i.Port) > 0 {
|
||||
if len(i.Port[0].Descr) > 0 {
|
||||
port = i.Port[0].Descr[0].Value
|
||||
} else if len(i.Port[0].ID) > 0 {
|
||||
port = i.Port[0].ID[0].Value
|
||||
}
|
||||
}
|
||||
var vs []string
|
||||
for _, v := range i.VLAN {
|
||||
s := v.VlanID
|
||||
if v.PVID {
|
||||
s += "(untagged)"
|
||||
}
|
||||
vs = append(vs, s)
|
||||
}
|
||||
vlan = strings.Join(vs, ",")
|
||||
out[filepath.Base(i.Name)] = [3]string{sw, port, vlan}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,488 @@
|
||||
package main
|
||||
|
||||
// rollcall doctor: checks an installation from the region host and says what
|
||||
// to do about what is missing. Local checks read the configuration; checks of
|
||||
// the running state ask the region's API with its own admin token.
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/binary"
|
||||
"encoding/hex"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"time"
|
||||
)
|
||||
|
||||
type doctor struct {
|
||||
fails, warns int
|
||||
}
|
||||
|
||||
func (d *doctor) ok(name, detail string) {
|
||||
fmt.Printf(" \033[32mOK \033[0m %-24s %s\n", name, detail)
|
||||
}
|
||||
func (d *doctor) warn(name, detail, fix string) {
|
||||
d.warns++
|
||||
fmt.Printf(" \033[33mWARN\033[0m %-24s %s\n", name, detail)
|
||||
if fix != "" {
|
||||
fmt.Printf(" %-24s -> %s\n", "", fix)
|
||||
}
|
||||
}
|
||||
func (d *doctor) fail(name, detail, fix string) {
|
||||
d.fails++
|
||||
fmt.Printf(" \033[31mFAIL\033[0m %-24s %s\n", name, detail)
|
||||
if fix != "" {
|
||||
fmt.Printf(" %-24s -> %s\n", "", fix)
|
||||
}
|
||||
}
|
||||
|
||||
func doctorCmd(args []string) error {
|
||||
fs := flag.NewFlagSet("doctor", flag.ExitOnError)
|
||||
cfgPath := fs.String("config", envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"), "config file")
|
||||
deep := fs.Bool("deep", false, "also verify the commissioning image checksums")
|
||||
noProbe := fs.Bool("no-dhcp-probe", false, "do not send a DHCPDISCOVER on the provisioning interface")
|
||||
_ = fs.Parse(args)
|
||||
d := &doctor{}
|
||||
fmt.Println("rollcall doctor", version)
|
||||
|
||||
cfg, err := loadConfig(*cfgPath)
|
||||
if err != nil {
|
||||
d.fail("config", err.Error(), "deploy/bootstrap.sh writes one; or see deploy/config.example.json")
|
||||
return d.done()
|
||||
}
|
||||
d.ok("config", *cfgPath)
|
||||
srvHost := ""
|
||||
if u, err := urlHost(cfg.ServerURL); err == nil {
|
||||
srvHost = u
|
||||
}
|
||||
iface := ifaceWithAddr(srvHost)
|
||||
if iface == "" {
|
||||
d.warn("server_url", cfg.ServerURL+" is not an address of this host", "machines reach the region there; set server_url to this host's address on the provisioning network")
|
||||
} else {
|
||||
d.ok("server_url", fmt.Sprintf("%s on %s", cfg.ServerURL, iface))
|
||||
}
|
||||
|
||||
// database
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
st, err := openStore(ctx, cfg.DatabaseURL)
|
||||
cancel()
|
||||
if err != nil {
|
||||
d.fail("database", err.Error(), "systemctl status postgresql; check database_url (role/password, sslmode=disable)")
|
||||
} else {
|
||||
d.ok("database", "connected, schema current")
|
||||
st.pool.Close()
|
||||
}
|
||||
|
||||
// listeners
|
||||
ports := listeningPorts()
|
||||
for _, p := range []struct {
|
||||
proto string
|
||||
port int
|
||||
what string
|
||||
}{{"udp", 67, "DHCP (dnsmasq)"}, {"udp", 69, "TFTP (dnsmasq)"}, {"tcp", portOf(cfg.Listen, 80), "HTTP (rollcall)"}, {"tcp", portOf(cfg.TLSListen, 443), "HTTPS (rollcall)"}} {
|
||||
if p.port == 0 {
|
||||
continue
|
||||
}
|
||||
if ports[fmt.Sprintf("%s/%d", p.proto, p.port)] {
|
||||
d.ok(fmt.Sprintf("%s/%d", p.proto, p.port), p.what)
|
||||
} else if p.port == 69 && !ports["udp/67"] {
|
||||
d.ok("udp/69", "TFTP starts with the first VLAN served")
|
||||
} else if p.port == 67 {
|
||||
d.warn("udp/67", "this host serves no DHCP yet", "no provisioning network decided: rollcall net discover, then deploy/bootstrap.sh --iface IFACE (or a VLAN with dhcp in the web UI)")
|
||||
} else {
|
||||
d.fail(fmt.Sprintf("%s/%d", p.proto, p.port), "nothing listens for "+p.what, "systemctl status rollcall dnsmasq")
|
||||
}
|
||||
}
|
||||
hc := &http.Client{Timeout: 5 * time.Second}
|
||||
if resp, err := hc.Get("http://127.0.0.1" + localPort(cfg.Listen) + "/ca.pem"); err != nil || resp.StatusCode != 200 {
|
||||
d.fail("http", fmt.Sprintf("GET /ca.pem: %v", errOrStatus(err, resp)), "journalctl -u rollcall")
|
||||
} else {
|
||||
resp.Body.Close()
|
||||
d.ok("http", "region answers")
|
||||
}
|
||||
|
||||
// certificates
|
||||
if cb, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")); err == nil {
|
||||
if c, err := parseCertPEM(cb); err == nil {
|
||||
left := time.Until(c.NotAfter)
|
||||
switch {
|
||||
case left < 90*24*time.Hour:
|
||||
d.warn("CA", "expires "+c.NotAfter.Format("2006-01-02"), "plan rollcall ca rotate")
|
||||
default:
|
||||
d.ok("CA", fmt.Sprintf("until %s, delegation %v", c.NotAfter.Format("2006-01-02"), (&CA{Cert: c}).canDelegate()))
|
||||
}
|
||||
if cfg.TLSListen != "" {
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(c)
|
||||
conn, err := tls.DialWithDialer(&net.Dialer{Timeout: 5 * time.Second}, "tcp", "127.0.0.1"+localPort(cfg.TLSListen),
|
||||
&tls.Config{RootCAs: pool, ServerName: srvHost})
|
||||
if err != nil {
|
||||
d.fail("https", err.Error(), "the region certificate should cover "+srvHost+" (tls_names); journalctl -u rollcall")
|
||||
} else {
|
||||
leaf := conn.ConnectionState().PeerCertificates[0]
|
||||
conn.Close()
|
||||
d.ok("https", fmt.Sprintf("certificate for %s until %s", srvHost, leaf.NotAfter.Format("2006-01-02")))
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
d.fail("CA", "no CA yet", "it is created when rollcall serve starts")
|
||||
}
|
||||
|
||||
// TFTP + image
|
||||
missing := []string{}
|
||||
for _, f := range []string{"pxelinux.0", "ldlinux.c32", "menu.c32", "libcom32.c32", "libutil.c32", "efi64/syslinux.efi", "efi64/ldlinux.e64", "pxelinux.cfg/default"} {
|
||||
if _, err := os.Stat(filepath.Join(cfg.TFTPRoot, f)); err != nil {
|
||||
missing = append(missing, f)
|
||||
}
|
||||
}
|
||||
if len(missing) > 0 {
|
||||
d.fail("tftp", "missing in "+cfg.TFTPRoot+": "+strings.Join(missing, " "), "deploy/bootstrap.sh (step 3) copies them from pxelinux/syslinux-efi")
|
||||
} else {
|
||||
d.ok("tftp", cfg.TFTPRoot+" (BIOS + UEFI)")
|
||||
}
|
||||
img := []string{}
|
||||
for _, f := range []string{"vmlinuz", "initrd.img", "filesystem.squashfs"} {
|
||||
if st, err := os.Stat(filepath.Join(cfg.ImageDir, f)); err != nil || st.Size() == 0 {
|
||||
img = append(img, f)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(img) > 0:
|
||||
d.fail("image", "commissioning image incomplete, missing "+strings.Join(img, " "), "deploy/bootstrap.sh --image-url URL or --build-image")
|
||||
case *deep:
|
||||
if bad := verifySums(cfg.ImageDir); bad == "no SHA256SUMS next to the image" {
|
||||
d.warn("image", bad, "cd "+cfg.ImageDir+" && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS")
|
||||
} else if bad != "" {
|
||||
d.fail("image", bad, "download the image again")
|
||||
} else {
|
||||
d.ok("image", "checksums match SHA256SUMS")
|
||||
}
|
||||
default:
|
||||
d.ok("image", cfg.ImageDir)
|
||||
}
|
||||
|
||||
// dnsmasq
|
||||
if out, _ := exec.Command("systemctl", "is-active", "dnsmasq").Output(); strings.TrimSpace(string(out)) != "active" {
|
||||
d.fail("dnsmasq", "not running", "journalctl -u dnsmasq -n 20")
|
||||
} else if _, err := os.Stat(filepath.Join(cfg.DnsmasqDir, "rollcall.conf.rejected")); err == nil {
|
||||
d.fail("dnsmasq", "the generated configuration was rejected; the last working one is in use",
|
||||
"journalctl -u rollcall-dnsmasq; a hand-written DHCP/TFTP file in /etc/dnsmasq.d usually conflicts")
|
||||
} else {
|
||||
d.ok("dnsmasq", "running")
|
||||
}
|
||||
|
||||
// DNS: outside, and through dnsmasq as the machines see it
|
||||
if _, err := net.DefaultResolver.LookupHost(context.Background(), "deb.debian.org"); err != nil {
|
||||
d.fail("dns (host)", err.Error(), "this host cannot resolve; check /etc/resolv.conf / resolvectl")
|
||||
} else {
|
||||
d.ok("dns (host)", "resolves outside names")
|
||||
}
|
||||
if srvHost != "" && ports["udp/53"] && ports["udp/67"] {
|
||||
r := &net.Resolver{PreferGo: true, Dial: func(ctx context.Context, _, _ string) (net.Conn, error) {
|
||||
return (&net.Dialer{Timeout: 3 * time.Second}).DialContext(ctx, "udp", net.JoinHostPort(srvHost, "53"))
|
||||
}}
|
||||
if _, err := r.LookupHost(context.Background(), "deb.debian.org"); err != nil {
|
||||
d.fail("dns (machines)", "dnsmasq on "+srvHost+" does not resolve outside names: "+err.Error(),
|
||||
"the generated dnsmasq configuration needs an upstream (rollcall >= 0.2 writes one); check its # upstream section")
|
||||
} else {
|
||||
d.ok("dns (machines)", "dnsmasq on "+srvHost+" forwards outside names")
|
||||
}
|
||||
}
|
||||
|
||||
// time
|
||||
if out, err := exec.Command("timedatectl", "show", "-p", "NTPSynchronized", "--value").Output(); err == nil {
|
||||
if strings.TrimSpace(string(out)) == "yes" {
|
||||
d.ok("time", "synchronised")
|
||||
} else {
|
||||
d.warn("time", "clock not synchronised", "certificates and tokens depend on it: systemctl enable --now systemd-timesyncd")
|
||||
}
|
||||
}
|
||||
|
||||
// other DHCP servers on the provisioning network
|
||||
if iface != "" && !*noProbe && !ports["udp/67"] {
|
||||
d.ok("dhcp probe", "skipped: this host serves no DHCP yet")
|
||||
} else if iface != "" && !*noProbe {
|
||||
servers, err := dhcpProbe(iface, 6*time.Second)
|
||||
switch {
|
||||
case err != nil:
|
||||
d.warn("dhcp probe", err.Error(), "run as root; or --no-dhcp-probe")
|
||||
default:
|
||||
var foreign []string
|
||||
for _, s := range servers {
|
||||
if ifaceWithAddr(s) == "" {
|
||||
foreign = append(foreign, s)
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case len(foreign) > 0:
|
||||
d.fail("dhcp probe", "another DHCP server answers on "+iface+": "+strings.Join(foreign, ", "),
|
||||
"machines may PXE-boot from it; switch it off for this VLAN, or use --no-dhcp and let it hand out next-server "+srvHost)
|
||||
case len(servers) == 0:
|
||||
d.warn("dhcp probe", "no DHCP server answered on "+iface, "is a VLAN with dhcp assigned to this rack? (rollcall vlan add ... -dhcp)")
|
||||
default:
|
||||
d.ok("dhcp probe", "only this host answers on "+iface)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// running state through the API
|
||||
if c, err := newClient(); err == nil {
|
||||
var racks []RackController
|
||||
if err := c.do("GET", "/api/v1/controllers", nil, &racks); err == nil {
|
||||
regionVer := version
|
||||
for _, r := range racks {
|
||||
if r.Kind == "embedded" && r.Version != "" {
|
||||
regionVer = r.Version // what the region runs (this binary may differ)
|
||||
}
|
||||
}
|
||||
for _, r := range racks {
|
||||
switch {
|
||||
case !r.Online:
|
||||
d.fail("rack "+r.Name, "offline since "+timeStr(r.LastSeen), "on that host: systemctl status rollcall-rack")
|
||||
case r.Version != regionVer && r.Version != "":
|
||||
d.warn("rack "+r.Name, "runs "+r.Version+", region "+regionVer, "it updates itself within minutes (rollcall-rack-update.path); see /var/lib/rollcall/update/last-result there")
|
||||
default:
|
||||
d.ok("rack "+r.Name, fmt.Sprintf("online, %d machines", r.Machines))
|
||||
}
|
||||
}
|
||||
}
|
||||
var oss []map[string]any
|
||||
if err := c.do("GET", "/api/v1/os", nil, &oss); err == nil {
|
||||
if len(oss) == 0 {
|
||||
d.warn("os images", "none", "runuser -u rollcall -- rollcall os sync debian-13")
|
||||
} else {
|
||||
d.ok("os images", fmt.Sprintf("%d in the catalog", len(oss)))
|
||||
}
|
||||
}
|
||||
var ipam map[string]any
|
||||
if err := c.do("GET", "/api/v1/ipam", nil, &ipam); err == nil && ipam["provider"] != "" && ipam["provider"] != nil {
|
||||
last, _ := ipam["last_sync"].(map[string]any)
|
||||
if e, _ := last["error"].(string); e != "" {
|
||||
d.fail("ipam", e, "check ipam.url/token_file/ca_file; rollcall ipam sync")
|
||||
} else {
|
||||
d.ok("ipam", fmt.Sprintf("%v in sync", ipam["provider"]))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
d.warn("api", err.Error(), "")
|
||||
}
|
||||
return d.done()
|
||||
}
|
||||
|
||||
func (d *doctor) done() error {
|
||||
fmt.Printf("\n%d failed, %d warnings\n", d.fails, d.warns)
|
||||
if d.fails > 0 {
|
||||
os.Exit(1)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func errOrStatus(err error, resp *http.Response) string {
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
return resp.Status
|
||||
}
|
||||
|
||||
func urlHost(u string) (string, error) {
|
||||
u = strings.TrimPrefix(strings.TrimPrefix(u, "http://"), "https://")
|
||||
u = strings.SplitN(u, "/", 2)[0]
|
||||
if h, _, err := net.SplitHostPort(u); err == nil {
|
||||
return strings.Trim(h, "[]"), nil
|
||||
}
|
||||
return strings.Trim(u, "[]"), nil
|
||||
}
|
||||
|
||||
// ifaceWithAddr: the interface holding ip on this host, "" if none.
|
||||
func ifaceWithAddr(ip string) string {
|
||||
want := net.ParseIP(ip)
|
||||
if want == nil {
|
||||
return ""
|
||||
}
|
||||
ifs, _ := net.Interfaces()
|
||||
for _, i := range ifs {
|
||||
addrs, _ := i.Addrs()
|
||||
for _, a := range addrs {
|
||||
if n, ok := a.(*net.IPNet); ok && n.IP.Equal(want) {
|
||||
return i.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func portOf(listen string, def int) int {
|
||||
if listen == "" {
|
||||
return 0
|
||||
}
|
||||
_, p, err := net.SplitHostPort(listen)
|
||||
if err != nil {
|
||||
return def
|
||||
}
|
||||
n, _ := strconv.Atoi(p)
|
||||
return n
|
||||
}
|
||||
|
||||
func localPort(listen string) string {
|
||||
if p := portOf(listen, 0); p != 0 && p != 80 && p != 443 {
|
||||
return ":" + strconv.Itoa(p)
|
||||
}
|
||||
if p := portOf(listen, 0); p == 443 {
|
||||
return ":443"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// listeningPorts reads /proc/net: "tcp/80", "udp/67" ...
|
||||
func listeningPorts() map[string]bool {
|
||||
out := map[string]bool{}
|
||||
for _, f := range []string{"tcp", "tcp6", "udp", "udp6"} {
|
||||
fh, err := os.Open("/proc/net/" + f)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
sc := bufio.NewScanner(fh)
|
||||
sc.Scan()
|
||||
for sc.Scan() {
|
||||
fl := strings.Fields(sc.Text())
|
||||
if len(fl) < 4 {
|
||||
continue
|
||||
}
|
||||
if strings.HasPrefix(f, "tcp") && fl[3] != "0A" { // LISTEN
|
||||
continue
|
||||
}
|
||||
i := strings.LastIndex(fl[1], ":")
|
||||
p, err := strconv.ParseUint(fl[1][i+1:], 16, 16)
|
||||
if err == nil {
|
||||
out[strings.TrimSuffix(f, "6")+"/"+strconv.Itoa(int(p))] = true
|
||||
}
|
||||
}
|
||||
fh.Close()
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func verifySums(dir string) string {
|
||||
b, err := os.ReadFile(filepath.Join(dir, "SHA256SUMS"))
|
||||
if err != nil {
|
||||
return "no SHA256SUMS next to the image"
|
||||
}
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) != 2 {
|
||||
continue
|
||||
}
|
||||
name := strings.TrimPrefix(f[1], "*")
|
||||
fh, err := os.Open(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
return name + ": " + err.Error()
|
||||
}
|
||||
h := sha256.New()
|
||||
_, _ = io.Copy(h, fh)
|
||||
fh.Close()
|
||||
if hex.EncodeToString(h.Sum(nil)) != f[0] {
|
||||
return name + ": checksum mismatch"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// dhcpProbe broadcasts a DHCPDISCOVER on iface and returns the server
|
||||
// identifiers that offer an address. Needs root (port 68, SO_BINDTODEVICE).
|
||||
func dhcpProbe(iface string, wait time.Duration) ([]string, error) {
|
||||
lc := net.ListenConfig{Control: func(_, _ string, c syscall.RawConn) error {
|
||||
var serr error
|
||||
err := c.Control(func(fd uintptr) {
|
||||
serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)
|
||||
if serr == nil {
|
||||
serr = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_REUSEADDR, 1)
|
||||
}
|
||||
if serr == nil {
|
||||
serr = bindToDevice(int(fd), iface)
|
||||
}
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return serr
|
||||
}}
|
||||
pc, err := lc.ListenPacket(context.Background(), "udp4", "0.0.0.0:68")
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("cannot listen on udp/68: %v", err)
|
||||
}
|
||||
defer pc.Close()
|
||||
if _, err := net.InterfaceByName(iface); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// a locally administered MAC so the probe never takes a real host's lease
|
||||
mac := make([]byte, 6)
|
||||
_, _ = rand.Read(mac)
|
||||
mac[0] = (mac[0] | 0x02) & 0xfe
|
||||
xid := make([]byte, 4)
|
||||
_, _ = rand.Read(xid)
|
||||
p := make([]byte, 240)
|
||||
p[0], p[1], p[2] = 1, 1, 6 // BOOTREQUEST, ethernet, hlen
|
||||
copy(p[4:8], xid)
|
||||
binary.BigEndian.PutUint16(p[10:12], 0x8000) // broadcast reply
|
||||
copy(p[28:34], mac)
|
||||
copy(p[236:240], []byte{99, 130, 83, 99})
|
||||
p = append(p, 53, 1, 1, 55, 3, 1, 3, 6, 255) // DISCOVER; ask router, DNS
|
||||
if _, err := pc.WriteTo(p, &net.UDPAddr{IP: net.IPv4bcast, Port: 67}); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
var out []string
|
||||
_ = pc.SetReadDeadline(time.Now().Add(wait))
|
||||
buf := make([]byte, 1500)
|
||||
for {
|
||||
n, _, err := pc.ReadFrom(buf)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
r := buf[:n]
|
||||
if n < 240 || r[0] != 2 || string(r[4:8]) != string(xid) {
|
||||
continue
|
||||
}
|
||||
srv := ""
|
||||
for i := 240; i < n; {
|
||||
o := r[i]
|
||||
if o == 255 {
|
||||
break
|
||||
}
|
||||
if o == 0 {
|
||||
i++
|
||||
continue
|
||||
}
|
||||
if i+1 >= n {
|
||||
break
|
||||
}
|
||||
l := int(r[i+1])
|
||||
if o == 54 && l == 4 && i+6 <= n {
|
||||
srv = net.IP(r[i+2 : i+6]).String()
|
||||
}
|
||||
i += 2 + l
|
||||
}
|
||||
if srv == "" {
|
||||
srv = net.IP(r[20:24]).String() // siaddr
|
||||
}
|
||||
if !seen[srv] {
|
||||
seen[srv] = true
|
||||
out = append(out, srv)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
//go:build linux
|
||||
|
||||
package main
|
||||
|
||||
import "syscall"
|
||||
|
||||
func bindToDevice(fd int, iface string) error {
|
||||
return syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, iface)
|
||||
}
|
||||
@@ -0,0 +1,7 @@
|
||||
//go:build !linux
|
||||
|
||||
package main
|
||||
|
||||
import "errors"
|
||||
|
||||
func bindToDevice(int, string) error { return errors.New("the DHCP probe needs Linux") }
|
||||
+24
-3
@@ -1,18 +1,39 @@
|
||||
module rollcall
|
||||
|
||||
go 1.24
|
||||
go 1.24.2
|
||||
|
||||
require (
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
github.com/charmbracelet/lipgloss v1.1.0
|
||||
github.com/charmbracelet/x/ansi v0.11.6
|
||||
github.com/jackc/pgx/v5 v5.7.5
|
||||
github.com/klauspost/compress v1.18.0
|
||||
github.com/mattn/go-isatty v0.0.24
|
||||
golang.org/x/crypto v0.37.0
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.4
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/charmbracelet/colorprofile v0.4.1 // indirect
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 // indirect
|
||||
github.com/charmbracelet/x/term v0.2.2 // indirect
|
||||
github.com/clipperhouse/displaywidth v0.9.0 // indirect
|
||||
github.com/clipperhouse/stringish v0.1.1 // indirect
|
||||
github.com/clipperhouse/uax29/v2 v2.5.0 // indirect
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect
|
||||
github.com/jackc/pgpassfile v1.0.0 // indirect
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 // indirect
|
||||
github.com/jackc/puddle/v2 v2.2.2 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 // indirect
|
||||
github.com/mattn/go-localereader v0.0.1 // indirect
|
||||
github.com/mattn/go-runewidth v0.0.19 // indirect
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect
|
||||
github.com/muesli/cancelreader v0.2.2 // indirect
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
golang.org/x/sys v0.32.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
)
|
||||
+47
-2
@@ -1,6 +1,28 @@
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/charmbracelet/bubbletea v1.3.10 h1:otUDHWMMzQSB0Pkc87rm691KZ3SWa4KUlvF9nRvCICw=
|
||||
github.com/charmbracelet/bubbletea v1.3.10/go.mod h1:ORQfo0fk8U+po9VaNvnV95UPWA1BitP1E0N6xJPlHr4=
|
||||
github.com/charmbracelet/colorprofile v0.4.1 h1:a1lO03qTrSIRaK8c3JRxJDZOvhvIeSco3ej+ngLk1kk=
|
||||
github.com/charmbracelet/colorprofile v0.4.1/go.mod h1:U1d9Dljmdf9DLegaJ0nGZNJvoXAhayhmidOdcBwAvKk=
|
||||
github.com/charmbracelet/lipgloss v1.1.0 h1:vYXsiLHVkK7fp74RkV7b2kq9+zDLoEU4MZoFqR/noCY=
|
||||
github.com/charmbracelet/lipgloss v1.1.0/go.mod h1:/6Q8FR2o+kj8rz4Dq0zQc3vYf7X+B0binUUBwA0aL30=
|
||||
github.com/charmbracelet/x/ansi v0.11.6 h1:GhV21SiDz/45W9AnV2R61xZMRri5NlLnl6CVF7ihZW8=
|
||||
github.com/charmbracelet/x/ansi v0.11.6/go.mod h1:2JNYLgQUsyqaiLovhU2Rv/pb8r6ydXKS3NIttu3VGZQ=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15 h1:ur3pZy0o6z/R7EylET877CBxaiE1Sp1GMxoFPAIztPI=
|
||||
github.com/charmbracelet/x/cellbuf v0.0.15/go.mod h1:J1YVbR7MUuEGIFPCaaZ96KDl5NoS0DAWkskup+mOY+Q=
|
||||
github.com/charmbracelet/x/term v0.2.2 h1:xVRT/S2ZcKdhhOuSP4t5cLi5o+JxklsoEObBSgfgZRk=
|
||||
github.com/charmbracelet/x/term v0.2.2/go.mod h1:kF8CY5RddLWrsgVwpw4kAa6TESp6EB5y3uxGLeCqzAI=
|
||||
github.com/clipperhouse/displaywidth v0.9.0 h1:Qb4KOhYwRiN3viMv1v/3cTBlz3AcAZX3+y9OLhMtAtA=
|
||||
github.com/clipperhouse/displaywidth v0.9.0/go.mod h1:aCAAqTlh4GIVkhQnJpbL0T/WfcrJXHcj8C0yjYcjOZA=
|
||||
github.com/clipperhouse/stringish v0.1.1 h1:+NSqMOr3GR6k1FdRhhnXrLfztGzuG+VuFDfatpWHKCs=
|
||||
github.com/clipperhouse/stringish v0.1.1/go.mod h1:v/WhFtE1q0ovMta2+m+UbpZ+2/HEXNWYXQgCt4hdOzA=
|
||||
github.com/clipperhouse/uax29/v2 v2.5.0 h1:x7T0T4eTHDONxFJsL94uKNKPHrclyFI0lm7+w94cO8U=
|
||||
github.com/clipperhouse/uax29/v2 v2.5.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsVRgg6W7ihQeh4g=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4=
|
||||
github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM=
|
||||
github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM=
|
||||
github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg=
|
||||
github.com/jackc/pgservicefile v0.0.0-20240606120523-5a60cdf6a761 h1:iCEnooe7UlwOQYpKFhBabPMi4aNAfoODPEFNiAnClxo=
|
||||
@@ -11,19 +33,40 @@ github.com/jackc/puddle/v2 v2.2.2 h1:PR8nw+E/1w0GLuRFSmiioY6UooMp6KJv0/61nB7icHo
|
||||
github.com/jackc/puddle/v2 v2.2.2/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
|
||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag=
|
||||
github.com/lucasb-eyer/go-colorful v1.3.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
|
||||
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
|
||||
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
|
||||
github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4=
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo=
|
||||
github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc=
|
||||
github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
|
||||
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20=
|
||||
golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
|
||||
golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
@@ -32,3 +75,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.4 h1:OvJ7g/OB9RuVAOzahlFLns6HeIcP+SJotsDqmLfhd8w=
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.4/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=
|
||||
@@ -0,0 +1,643 @@
|
||||
package main
|
||||
|
||||
// External IPAM: an authority outside Rollcall (NetBox, or Infoblox where a
|
||||
// site runs it) in which every address Rollcall hands out is registered.
|
||||
//
|
||||
// - Allocation asks the authority first: an address someone else already
|
||||
// has there is skipped, so Rollcall never collides with the site's plan.
|
||||
// - Releases are local; the reconcile loop then makes the authority match
|
||||
// Rollcall's table: it creates what is missing, fixes names, and deletes
|
||||
// what Rollcall owns there but no longer uses.
|
||||
// - Ownership is explicit: a tag plus the machine ID in a custom field
|
||||
// (an extensible attribute on Infoblox). Records without both are never
|
||||
// changed or deleted, only reported as conflicts.
|
||||
// - DNS follows from the record's dns_name (NetBox + PowerDNS sync, or an
|
||||
// Infoblox host record), so Rollcall writes one object per address.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type IPAMConfig struct {
|
||||
Provider string `json:"provider"` // "" = built-in only | netbox
|
||||
URL string `json:"url"` // https://netbox.example
|
||||
TokenFile string `json:"token_file"` // API token (NetBox v2 "nbt_..." or v1)
|
||||
CAFile string `json:"ca_file"` // CA that signed the server certificate ("" = system roots)
|
||||
DNSZone string `json:"dns_zone"` // dns_name = <host>.<zone> unless the subnet has a domain
|
||||
Tag string `json:"tag"` // ownership tag, default "rollcall"
|
||||
MachineField string `json:"machine_field"` // custom field holding the machine ID, default "rollcall_machine"
|
||||
SyncInterval string `json:"sync_interval"` // reconcile period, default 5m
|
||||
}
|
||||
|
||||
// extAddr is one address as the authority holds it.
|
||||
type extAddr struct {
|
||||
ID string `json:"id,omitempty"`
|
||||
IP string `json:"ip"`
|
||||
Bits int `json:"bits"`
|
||||
DNSName string `json:"dns_name,omitempty"`
|
||||
Machine string `json:"machine,omitempty"` // owner mark; "-" = Rollcall reservation without a machine
|
||||
Status string `json:"status"` // active | reserved
|
||||
Desc string `json:"description,omitempty"`
|
||||
}
|
||||
|
||||
type extPrefix struct {
|
||||
Prefix string `json:"prefix"`
|
||||
Desc string `json:"description,omitempty"`
|
||||
VID int `json:"vid,omitempty"`
|
||||
Site string `json:"site,omitempty"`
|
||||
Local bool `json:"local"` // a Rollcall subnet exists for it
|
||||
Comment string `json:"comment,omitempty"`
|
||||
}
|
||||
|
||||
type ipamProvider interface {
|
||||
Name() string
|
||||
// Reserve registers a; taken is true when the address exists there and
|
||||
// is not Rollcall's. An existing Rollcall-owned record is updated.
|
||||
Reserve(ctx context.Context, a extAddr) (taken bool, err error)
|
||||
Owned(ctx context.Context) ([]extAddr, error)
|
||||
Update(ctx context.Context, a extAddr) error
|
||||
Delete(ctx context.Context, a extAddr) error
|
||||
Prefixes(ctx context.Context) ([]extPrefix, error)
|
||||
}
|
||||
|
||||
type ipamReport struct {
|
||||
Provider string `json:"provider"`
|
||||
At time.Time `json:"at"`
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
Created []string `json:"created"`
|
||||
Updated []string `json:"updated"`
|
||||
Deleted []string `json:"deleted"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
type ipamState struct {
|
||||
mu sync.Mutex // one reconcile at a time
|
||||
kick chan struct{}
|
||||
last ipamReport
|
||||
}
|
||||
|
||||
func newIPAMProvider(c IPAMConfig) (ipamProvider, error) {
|
||||
switch c.Provider {
|
||||
case "":
|
||||
return nil, nil
|
||||
case "netbox":
|
||||
return newNetBox(c)
|
||||
default:
|
||||
return nil, fmt.Errorf("ipam provider %q: unknown (netbox)", c.Provider)
|
||||
}
|
||||
}
|
||||
|
||||
func (c IPAMConfig) tag() string { return orDefault(c.Tag, "rollcall") }
|
||||
func (c IPAMConfig) field() string { return orDefault(c.MachineField, "rollcall_machine") }
|
||||
func orDefault(s, d string) string {
|
||||
if s == "" {
|
||||
return d
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- App side
|
||||
|
||||
var dnsLabel = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`)
|
||||
|
||||
// extFor maps a Rollcall address to the record the authority should hold.
|
||||
func (a *App) extFor(x IPAddr, sn *Subnet) extAddr {
|
||||
bits := 32
|
||||
if p, err := netip.ParsePrefix(x.Subnet); err == nil {
|
||||
bits = p.Bits()
|
||||
} else if ad, ok := parseAddr(x.IP); ok && ad.Is6() {
|
||||
bits = 128
|
||||
}
|
||||
e := extAddr{IP: x.IP, Bits: bits, Status: "active", Machine: x.MachineID}
|
||||
if x.Kind == "reserved" {
|
||||
e.Status = "reserved"
|
||||
}
|
||||
if e.Machine == "" {
|
||||
e.Machine = "-"
|
||||
}
|
||||
desc := "rollcall"
|
||||
if x.MachineID != "" {
|
||||
desc += " " + x.MachineID
|
||||
}
|
||||
if x.Iface != "" {
|
||||
desc += " " + x.Iface
|
||||
}
|
||||
if x.Kind == "reserved" && x.Hostname != "" {
|
||||
desc += ": " + x.Hostname
|
||||
}
|
||||
e.Desc = desc
|
||||
e.DNSName = a.extDNSName(x, sn)
|
||||
return e
|
||||
}
|
||||
|
||||
// extDNSName: the machine's main address gets <host>.<zone>; addresses on
|
||||
// other subnets (storage, IB, a tagged VLAN) get <host>-<iface>.<zone>. A
|
||||
// subnet's domain replaces the zone when it lies inside it.
|
||||
func (a *App) extDNSName(x IPAddr, sn *Subnet) string {
|
||||
zone := strings.TrimSuffix(strings.ToLower(a.cfg.IPAM.DNSZone), ".")
|
||||
if sn != nil && sn.Domain != "" { // a subnet domain counts only inside the authority's zone; "pxe.local" stays local
|
||||
d := strings.TrimSuffix(strings.ToLower(sn.Domain), ".")
|
||||
if zone == "" || d == zone || strings.HasSuffix(d, "."+zone) {
|
||||
zone = d
|
||||
}
|
||||
}
|
||||
host := strings.ToLower(strings.TrimSuffix(x.Hostname, "."))
|
||||
if zone == "" || host == "" {
|
||||
return ""
|
||||
}
|
||||
if strings.HasSuffix(host, "."+zone) {
|
||||
return host
|
||||
}
|
||||
if !dnsLabel.MatchString(host) {
|
||||
return "" // a free-text reservation name ("core switch") stays in the description
|
||||
}
|
||||
main := sn == nil || x.Iface == "" || sn.Purpose == "" || sn.Purpose == "provisioning" || sn.Purpose == "mgmt"
|
||||
if !main {
|
||||
suffix := strings.Trim(regexp.MustCompile(`[^a-z0-9]+`).ReplaceAllString(strings.ToLower(x.Iface), "-"), "-")
|
||||
if suffix != "" {
|
||||
host += "-" + suffix
|
||||
}
|
||||
}
|
||||
return host + "." + zone
|
||||
}
|
||||
|
||||
// claimIP records an address in the authority (when one is configured) and
|
||||
// then locally; false when it is taken in either.
|
||||
func (a *App) claimIP(ctx context.Context, x IPAddr) (bool, error) {
|
||||
if a.ipam != nil {
|
||||
sn := subnetByRef(a.subnetsSnapshot(), x.Subnet)
|
||||
taken, err := a.ipam.Reserve(ctx, a.extFor(x, sn))
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("%s: %w", a.ipam.Name(), err)
|
||||
}
|
||||
if taken {
|
||||
return false, nil
|
||||
}
|
||||
}
|
||||
ok, err := a.store.ClaimIP(ctx, x)
|
||||
if a.ipam != nil && (err != nil || !ok) {
|
||||
a.kickIPAM() // the record we just made is reconciled away or re-owned
|
||||
}
|
||||
return ok, err
|
||||
}
|
||||
|
||||
func (a *App) kickIPAM() {
|
||||
if a.ipamSt == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case a.ipamSt.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) startIPAM() error {
|
||||
p, err := newIPAMProvider(a.cfg.IPAM)
|
||||
if err != nil || p == nil {
|
||||
return err
|
||||
}
|
||||
a.ipam = p
|
||||
a.ipamSt = &ipamState{kick: make(chan struct{}, 1)}
|
||||
every := 5 * time.Minute
|
||||
if d, err := time.ParseDuration(a.cfg.IPAM.SyncInterval); err == nil && d >= 10*time.Second {
|
||||
every = d
|
||||
}
|
||||
go func() {
|
||||
a.kickIPAM() // first pass right after start
|
||||
for {
|
||||
select {
|
||||
case <-a.ipamSt.kick:
|
||||
case <-time.After(every):
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
rep := a.reconcileIPAM(ctx, false)
|
||||
cancel()
|
||||
if rep.Error != "" {
|
||||
log.Printf("ipam %s: %s", p.Name(), rep.Error)
|
||||
} else if n := len(rep.Created) + len(rep.Updated) + len(rep.Deleted) + len(rep.Conflicts); n > 0 {
|
||||
log.Printf("ipam %s: created %d updated %d deleted %d conflicts %d %v", p.Name(),
|
||||
len(rep.Created), len(rep.Updated), len(rep.Deleted), len(rep.Conflicts), rep.Conflicts)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("ipam: %s at %s (zone %q, owner tag %q)", p.Name(), a.cfg.IPAM.URL, a.cfg.IPAM.DNSZone, a.cfg.IPAM.tag())
|
||||
return nil
|
||||
}
|
||||
|
||||
// reconcileIPAM makes the authority match Rollcall's address table.
|
||||
func (a *App) reconcileIPAM(ctx context.Context, dry bool) ipamReport {
|
||||
rep := ipamReport{At: time.Now().UTC(), DryRun: dry, Created: []string{}, Updated: []string{}, Deleted: []string{}, Conflicts: []string{}}
|
||||
if a.ipam == nil {
|
||||
rep.Error = "no external IPAM configured (ipam.provider in config.json)"
|
||||
return rep
|
||||
}
|
||||
rep.Provider = a.ipam.Name()
|
||||
a.ipamSt.mu.Lock()
|
||||
defer a.ipamSt.mu.Unlock()
|
||||
defer func() {
|
||||
if !dry {
|
||||
a.ipamSt.last = rep
|
||||
}
|
||||
}()
|
||||
local, err := a.store.IPs(ctx, "", "")
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
a.reconcileAddrs(ctx, local, &rep)
|
||||
return rep
|
||||
}
|
||||
|
||||
// reconcileAddrs is the reconcile proper, over a given address table.
|
||||
func (a *App) reconcileAddrs(ctx context.Context, local []IPAddr, rep *ipamReport) {
|
||||
dry := rep.DryRun
|
||||
owned, err := a.ipam.Owned(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return
|
||||
}
|
||||
have := map[string]extAddr{}
|
||||
for _, o := range owned {
|
||||
have[o.IP] = o
|
||||
}
|
||||
subnets := a.subnetsSnapshot()
|
||||
for _, x := range local {
|
||||
want := a.extFor(x, subnetByRef(subnets, x.Subnet))
|
||||
if o, ok := have[x.IP]; ok {
|
||||
delete(have, x.IP)
|
||||
if o.DNSName != want.DNSName || o.Machine != want.Machine || o.Status != want.Status || o.Bits != want.Bits || o.Desc != want.Desc {
|
||||
want.ID = o.ID
|
||||
rep.Updated = append(rep.Updated, x.IP)
|
||||
if !dry {
|
||||
if err := a.ipam.Update(ctx, want); err != nil {
|
||||
rep.Error = fmt.Sprintf("update %s: %v", x.IP, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if dry {
|
||||
rep.Created = append(rep.Created, x.IP)
|
||||
continue
|
||||
}
|
||||
taken, err := a.ipam.Reserve(ctx, want)
|
||||
if err != nil {
|
||||
rep.Error = fmt.Sprintf("create %s: %v", x.IP, err)
|
||||
return
|
||||
}
|
||||
if taken {
|
||||
rep.Conflicts = append(rep.Conflicts, x.IP+" exists there and is not Rollcall's")
|
||||
} else {
|
||||
rep.Created = append(rep.Created, x.IP)
|
||||
}
|
||||
}
|
||||
rest := make([]extAddr, 0, len(have))
|
||||
for _, o := range have {
|
||||
rest = append(rest, o)
|
||||
}
|
||||
sort.Slice(rest, func(i, j int) bool { return ipLess(rest[i].IP, rest[j].IP) })
|
||||
for _, o := range rest {
|
||||
rep.Deleted = append(rep.Deleted, o.IP)
|
||||
if !dry {
|
||||
if err := a.ipam.Delete(ctx, o); err != nil {
|
||||
rep.Error = fmt.Sprintf("delete %s: %v", o.IP, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleIPAM(w http.ResponseWriter, r *http.Request) {
|
||||
if a.ipam == nil {
|
||||
writeJSON(w, 200, obj{"provider": "", "comment": "built-in IPAM only"})
|
||||
return
|
||||
}
|
||||
a.ipamSt.mu.Lock()
|
||||
last := a.ipamSt.last
|
||||
a.ipamSt.mu.Unlock()
|
||||
writeJSON(w, 200, obj{"provider": a.ipam.Name(), "url": a.cfg.IPAM.URL, "dns_zone": a.cfg.IPAM.DNSZone,
|
||||
"tag": a.cfg.IPAM.tag(), "machine_field": a.cfg.IPAM.field(), "last_sync": last})
|
||||
}
|
||||
|
||||
func (a *App) handleIPAMSync(w http.ResponseWriter, r *http.Request) {
|
||||
dry := r.URL.Query().Get("dry") != ""
|
||||
rep := a.reconcileIPAM(r.Context(), dry)
|
||||
code := 200
|
||||
if rep.Error != "" {
|
||||
code = 502
|
||||
if a.ipam == nil {
|
||||
code = 400
|
||||
}
|
||||
}
|
||||
writeJSON(w, code, rep)
|
||||
}
|
||||
|
||||
// handleIPAMPrefixes lists the prefixes the authority delegates to Rollcall
|
||||
// (tagged); POST imports the missing ones as subnets.
|
||||
func (a *App) handleIPAMPrefixes(w http.ResponseWriter, r *http.Request) {
|
||||
if a.ipam == nil {
|
||||
fail(w, 400, fmt.Errorf("no external IPAM configured"))
|
||||
return
|
||||
}
|
||||
ps, err := a.ipam.Prefixes(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 502, err)
|
||||
return
|
||||
}
|
||||
subnets := a.subnetsSnapshot()
|
||||
imported := []string{}
|
||||
for i := range ps {
|
||||
p, err := netip.ParsePrefix(ps[i].Prefix)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, sn := range subnets {
|
||||
if q, err := netip.ParsePrefix(sn.CIDR); err == nil && q.Masked() == p.Masked() {
|
||||
ps[i].Local = true
|
||||
}
|
||||
}
|
||||
if r.Method == http.MethodPost && !ps[i].Local {
|
||||
sn := Subnet{CIDR: p.Masked().String(), Name: ps[i].Desc, Purpose: "provisioning",
|
||||
Comment: "imported from " + a.ipam.Name(), DNS: []string{}}
|
||||
if len(sn.Name) > 60 {
|
||||
sn.Name = sn.Name[:60]
|
||||
}
|
||||
if err := a.store.PutSubnet(r.Context(), sn); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
ps[i].Local = true
|
||||
ps[i].Comment = "imported"
|
||||
imported = append(imported, sn.CIDR)
|
||||
}
|
||||
}
|
||||
if len(imported) > 0 {
|
||||
a.netChanged(r)
|
||||
}
|
||||
writeJSON(w, 200, obj{"prefixes": ps, "imported": imported})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- NetBox
|
||||
|
||||
type netBox struct {
|
||||
base string
|
||||
token string
|
||||
tag string
|
||||
field string
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newNetBox(c IPAMConfig) (*netBox, error) {
|
||||
if c.URL == "" || c.TokenFile == "" {
|
||||
return nil, fmt.Errorf("netbox: url and token_file are required")
|
||||
}
|
||||
tok, err := os.ReadFile(c.TokenFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("netbox token: %w", err)
|
||||
}
|
||||
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||
if c.CAFile != "" {
|
||||
pem, err := os.ReadFile(c.CAFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("netbox ca_file: %w", err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
return nil, fmt.Errorf("netbox ca_file %s: no certificates", c.CAFile)
|
||||
}
|
||||
tr.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}
|
||||
}
|
||||
return &netBox{base: strings.TrimRight(c.URL, "/"), token: strings.TrimSpace(string(tok)),
|
||||
tag: c.tag(), field: c.field(), client: &http.Client{Transport: tr, Timeout: 30 * time.Second}}, nil
|
||||
}
|
||||
|
||||
func (n *netBox) Name() string { return "netbox" }
|
||||
|
||||
type nbError struct {
|
||||
Code int
|
||||
Body string
|
||||
}
|
||||
|
||||
func (e *nbError) Error() string { return fmt.Sprintf("HTTP %d: %s", e.Code, trunc(e.Body, 300)) }
|
||||
|
||||
func (n *netBox) do(ctx context.Context, method, path string, body, out any) error {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
u := n.base + "/api/" + strings.TrimPrefix(path, "/")
|
||||
if strings.HasPrefix(path, "http") { // a "next" link: keep our base, the server may name itself differently
|
||||
if p, err := url.Parse(path); err == nil {
|
||||
u = n.base + p.Path
|
||||
if p.RawQuery != "" {
|
||||
u += "?" + p.RawQuery
|
||||
}
|
||||
}
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, u, rd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if strings.HasPrefix(n.token, "nbt_") {
|
||||
req.Header.Set("Authorization", "Bearer "+n.token)
|
||||
} else {
|
||||
req.Header.Set("Authorization", "Token "+n.token)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := n.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
if resp.StatusCode >= 300 {
|
||||
return &nbError{resp.StatusCode, string(b)}
|
||||
}
|
||||
if out != nil && len(b) > 0 {
|
||||
return json.Unmarshal(b, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type nbIP struct {
|
||||
ID int64 `json:"id"`
|
||||
Address string `json:"address"`
|
||||
DNSName string `json:"dns_name"`
|
||||
Description string `json:"description"`
|
||||
Status struct{ Value string } `json:"status"`
|
||||
Tags []struct{ Name string } `json:"tags"`
|
||||
CustomFields map[string]any `json:"custom_fields"`
|
||||
}
|
||||
|
||||
type nbPage[T any] struct {
|
||||
Next *string `json:"next"`
|
||||
Results []T `json:"results"`
|
||||
}
|
||||
|
||||
func (n *netBox) list(ctx context.Context, path string, q url.Values) ([]nbIP, error) {
|
||||
q.Set("limit", "500")
|
||||
next := path + "?" + q.Encode()
|
||||
var out []nbIP
|
||||
for next != "" {
|
||||
var p nbPage[nbIP]
|
||||
if err := n.do(ctx, "GET", next, nil, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p.Results...)
|
||||
next = ""
|
||||
if p.Next != nil {
|
||||
next = *p.Next
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (n *netBox) owner(x nbIP) string {
|
||||
tagged := false
|
||||
for _, t := range x.Tags {
|
||||
tagged = tagged || t.Name == n.tag
|
||||
}
|
||||
if !tagged {
|
||||
return ""
|
||||
}
|
||||
if v, ok := x.CustomFields[n.field].(string); ok {
|
||||
return v
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (n *netBox) toExt(x nbIP) extAddr {
|
||||
ip, bits := x.Address, 0
|
||||
if p, err := netip.ParsePrefix(x.Address); err == nil {
|
||||
ip, bits = p.Addr().String(), p.Bits()
|
||||
}
|
||||
return extAddr{ID: strconv.FormatInt(x.ID, 10), IP: ip, Bits: bits, DNSName: x.DNSName, Machine: n.owner(x),
|
||||
Status: x.Status.Value, Desc: x.Description}
|
||||
}
|
||||
|
||||
func (n *netBox) body(a extAddr) obj {
|
||||
return obj{"address": fmt.Sprintf("%s/%d", a.IP, a.Bits), "status": a.Status, "dns_name": a.DNSName,
|
||||
"description": a.Desc, "tags": []obj{{"name": n.tag}}, "custom_fields": obj{n.field: a.Machine}}
|
||||
}
|
||||
|
||||
func (n *netBox) find(ctx context.Context, ip string) ([]nbIP, error) {
|
||||
return n.list(ctx, "ipam/ip-addresses/", url.Values{"address": {ip}})
|
||||
}
|
||||
|
||||
func (n *netBox) Reserve(ctx context.Context, a extAddr) (bool, error) {
|
||||
cur, err := n.find(ctx, a.IP)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, x := range cur {
|
||||
if n.owner(x) == "" {
|
||||
return true, nil // someone else's
|
||||
}
|
||||
}
|
||||
if len(cur) > 0 {
|
||||
a.ID = strconv.FormatInt(cur[0].ID, 10)
|
||||
return false, n.Update(ctx, a)
|
||||
}
|
||||
err = n.do(ctx, "POST", "ipam/ip-addresses/", n.body(a), nil)
|
||||
var ne *nbError
|
||||
if err != nil && errors.As(err, &ne) && ne.Code == 400 && strings.Contains(strings.ToLower(ne.Body), "duplicate") {
|
||||
return true, nil // created by someone else between our look and our write
|
||||
}
|
||||
return false, err
|
||||
}
|
||||
|
||||
func (n *netBox) Owned(ctx context.Context) ([]extAddr, error) {
|
||||
xs, err := n.list(ctx, "ipam/ip-addresses/", url.Values{"tag": {n.tag}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []extAddr{}
|
||||
for _, x := range xs {
|
||||
if n.owner(x) != "" {
|
||||
out = append(out, n.toExt(x))
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (n *netBox) Update(ctx context.Context, a extAddr) error {
|
||||
return n.do(ctx, "PATCH", "ipam/ip-addresses/"+a.ID+"/", n.body(a), nil)
|
||||
}
|
||||
|
||||
func (n *netBox) Delete(ctx context.Context, a extAddr) error {
|
||||
var cur nbIP // re-read: only delete what is still ours
|
||||
if err := n.do(ctx, "GET", "ipam/ip-addresses/"+a.ID+"/", nil, &cur); err != nil {
|
||||
var ne *nbError
|
||||
if errors.As(err, &ne) && ne.Code == 404 {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
if n.owner(cur) == "" {
|
||||
return nil
|
||||
}
|
||||
err := n.do(ctx, "DELETE", "ipam/ip-addresses/"+a.ID+"/", nil, nil)
|
||||
var ne *nbError
|
||||
if errors.As(err, &ne) && ne.Code == 404 {
|
||||
return nil
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (n *netBox) Prefixes(ctx context.Context) ([]extPrefix, error) {
|
||||
type nbPrefix struct {
|
||||
Prefix string `json:"prefix"`
|
||||
Description string `json:"description"`
|
||||
VLAN *struct {
|
||||
VID int `json:"vid"`
|
||||
} `json:"vlan"`
|
||||
Scope *struct {
|
||||
Name string `json:"name"`
|
||||
} `json:"scope"`
|
||||
}
|
||||
next := "ipam/prefixes/?" + url.Values{"tag": {n.tag}, "limit": {"500"}}.Encode()
|
||||
out := []extPrefix{}
|
||||
for next != "" {
|
||||
var p nbPage[nbPrefix]
|
||||
if err := n.do(ctx, "GET", next, nil, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, x := range p.Results {
|
||||
e := extPrefix{Prefix: x.Prefix, Desc: x.Description}
|
||||
if x.VLAN != nil {
|
||||
e.VID = x.VLAN.VID
|
||||
}
|
||||
if x.Scope != nil {
|
||||
e.Site = x.Scope.Name
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
next = ""
|
||||
if p.Next != nil {
|
||||
next = *p.Next
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,255 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeNetBox implements the slice of the NetBox REST API the driver uses.
|
||||
type fakeNetBox struct {
|
||||
mu sync.Mutex
|
||||
next int64
|
||||
ips map[int64]map[string]any
|
||||
reqs []string
|
||||
}
|
||||
|
||||
func newFakeNetBox() *fakeNetBox { return &fakeNetBox{next: 1, ips: map[int64]map[string]any{}} }
|
||||
|
||||
func (f *fakeNetBox) add(addr, dns string, tags []string, machine string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
var ts []any
|
||||
for _, t := range tags {
|
||||
ts = append(ts, map[string]any{"name": t})
|
||||
}
|
||||
f.ips[f.next] = map[string]any{"id": f.next, "address": addr, "dns_name": dns, "description": "",
|
||||
"status": map[string]any{"value": "active"}, "tags": ts, "custom_fields": map[string]any{"rollcall_machine": machine}}
|
||||
f.next++
|
||||
}
|
||||
|
||||
func (f *fakeNetBox) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.reqs = append(f.reqs, r.Method+" "+r.URL.Path)
|
||||
if r.Header.Get("Authorization") != "Bearer nbt_k.secret" {
|
||||
http.Error(w, "auth", 403)
|
||||
return
|
||||
}
|
||||
host := func(a string) string { return strings.SplitN(a, "/", 2)[0] }
|
||||
write := func(code int, v any) { w.WriteHeader(code); _ = json.NewEncoder(w).Encode(v) }
|
||||
in := func() map[string]any {
|
||||
var b map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&b)
|
||||
if s, ok := b["status"].(string); ok {
|
||||
b["status"] = map[string]any{"value": s}
|
||||
}
|
||||
return b
|
||||
}
|
||||
var id int64
|
||||
if _, err := fmt.Sscanf(r.URL.Path, "/api/ipam/ip-addresses/%d/", &id); err == nil {
|
||||
x, ok := f.ips[id]
|
||||
if !ok {
|
||||
write(404, map[string]any{"detail": "not found"})
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
write(200, x)
|
||||
case "PATCH":
|
||||
for k, v := range in() {
|
||||
x[k] = v
|
||||
}
|
||||
write(200, x)
|
||||
case "DELETE":
|
||||
delete(f.ips, id)
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/api/ipam/ip-addresses/" && r.Method == "GET":
|
||||
res := []any{}
|
||||
ids := []int64{}
|
||||
for id := range f.ips {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
|
||||
for _, id := range ids {
|
||||
x := f.ips[id]
|
||||
if a := r.URL.Query().Get("address"); a != "" && host(x["address"].(string)) != a {
|
||||
continue
|
||||
}
|
||||
if t := r.URL.Query().Get("tag"); t != "" {
|
||||
found := false
|
||||
for _, tt := range x["tags"].([]any) {
|
||||
found = found || tt.(map[string]any)["name"] == t
|
||||
}
|
||||
if !found {
|
||||
continue
|
||||
}
|
||||
}
|
||||
res = append(res, x)
|
||||
}
|
||||
write(200, map[string]any{"count": len(res), "next": nil, "results": res})
|
||||
case r.URL.Path == "/api/ipam/ip-addresses/" && r.Method == "POST":
|
||||
b := in()
|
||||
for _, x := range f.ips {
|
||||
if host(x["address"].(string)) == host(b["address"].(string)) {
|
||||
write(400, map[string]any{"address": []string{"Duplicate IP address found in global table"}})
|
||||
return
|
||||
}
|
||||
}
|
||||
b["id"] = f.next
|
||||
f.ips[f.next] = b
|
||||
f.next++
|
||||
write(201, b)
|
||||
case r.URL.Path == "/api/ipam/prefixes/":
|
||||
write(200, map[string]any{"next": nil, "results": []any{
|
||||
map[string]any{"prefix": "10.213.77.0/24", "description": "lab", "vlan": nil, "scope": map[string]any{"name": "Ev"}},
|
||||
map[string]any{"prefix": "10.213.80.0/24", "description": "new hall", "vlan": map[string]any{"vid": 300}, "scope": nil},
|
||||
}})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}
|
||||
|
||||
func testIPAM(t *testing.T) (*App, *fakeNetBox) {
|
||||
t.Helper()
|
||||
f := newFakeNetBox()
|
||||
srv := httptest.NewServer(f)
|
||||
t.Cleanup(srv.Close)
|
||||
tok := filepath.Join(t.TempDir(), "token")
|
||||
if err := os.WriteFile(tok, []byte("nbt_k.secret\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &Config{IPAM: IPAMConfig{Provider: "netbox", URL: srv.URL, TokenFile: tok, DNSZone: "int.example"}}
|
||||
p, err := newIPAMProvider(cfg.IPAM)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &App{cfg: cfg, ipam: p, ipamSt: &ipamState{kick: make(chan struct{}, 1)}}
|
||||
a.subnets = []Subnet{
|
||||
{CIDR: "10.213.77.0/24", Purpose: "provisioning"},
|
||||
{CIDR: "10.213.79.0/24", Purpose: "storage"},
|
||||
{CIDR: "10.213.78.0/24", Purpose: "provisioning", Domain: "hall-b.int.example"},
|
||||
{CIDR: "10.213.76.0/24", Purpose: "provisioning", Domain: "pxe.local"},
|
||||
}
|
||||
return a, f
|
||||
}
|
||||
|
||||
func TestIPAMDNSNames(t *testing.T) {
|
||||
a, _ := testIPAM(t)
|
||||
sn := func(c string) *Subnet { return subnetByRef(a.subnets, c) }
|
||||
cases := []struct {
|
||||
x IPAddr
|
||||
want string
|
||||
}{
|
||||
{IPAddr{IP: "10.213.77.50", Subnet: "10.213.77.0/24", Hostname: "node001", Iface: "bond0"}, "node001.int.example"},
|
||||
{IPAddr{IP: "10.213.79.50", Subnet: "10.213.79.0/24", Hostname: "node001", Iface: "bond0.200"}, "node001-bond0-200.int.example"},
|
||||
{IPAddr{IP: "10.213.78.50", Subnet: "10.213.78.0/24", Hostname: "Node002"}, "node002.hall-b.int.example"},
|
||||
{IPAddr{IP: "10.213.76.50", Subnet: "10.213.76.0/24", Hostname: "node003", Iface: "eth0"}, "node003.int.example"},
|
||||
{IPAddr{IP: "10.213.77.5", Subnet: "10.213.77.0/24", Hostname: "core switch", Kind: "reserved"}, ""},
|
||||
{IPAddr{IP: "10.213.77.6", Subnet: "10.213.77.0/24", Kind: "reserved"}, ""},
|
||||
}
|
||||
for _, c := range cases {
|
||||
if got := a.extDNSName(c.x, sn(c.x.Subnet)); got != c.want {
|
||||
t.Errorf("%s %q/%s: got %q want %q", c.x.IP, c.x.Hostname, c.x.Iface, got, c.want)
|
||||
}
|
||||
}
|
||||
e := a.extFor(IPAddr{IP: "10.213.77.5", Subnet: "10.213.77.0/24", Hostname: "core switch", Kind: "reserved"}, sn("10.213.77.0/24"))
|
||||
if e.Status != "reserved" || e.Machine != "-" || e.Bits != 24 || !strings.Contains(e.Desc, "core switch") {
|
||||
t.Errorf("reserved mapping: %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPAMReserve(t *testing.T) {
|
||||
a, f := testIPAM(t)
|
||||
ctx := context.Background()
|
||||
f.add("10.213.77.20/24", "printer.int.example", nil, "") // someone else's
|
||||
f.add("10.213.77.21/24", "", []string{"rollcall"}, "") // tagged but no owner mark: still not ours
|
||||
f.add("10.213.77.22/24", "old.int.example", []string{"rollcall"}, "m-1")
|
||||
|
||||
for ip, wantTaken := range map[string]bool{"10.213.77.20": true, "10.213.77.21": true, "10.213.77.22": false, "10.213.77.23": false} {
|
||||
taken, err := a.ipam.Reserve(ctx, a.extFor(IPAddr{IP: ip, Subnet: "10.213.77.0/24", MachineID: "m-2", Hostname: "node002", Iface: "eth0"}, &a.subnets[0]))
|
||||
if err != nil {
|
||||
t.Fatal(ip, err)
|
||||
}
|
||||
if taken != wantTaken {
|
||||
t.Errorf("%s: taken=%v want %v", ip, taken, wantTaken)
|
||||
}
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, x := range f.ips {
|
||||
got[x["address"].(string)] = x["dns_name"].(string) + " " + fmt.Sprint(x["custom_fields"].(map[string]any)["rollcall_machine"])
|
||||
}
|
||||
if got["10.213.77.20/24"] != "printer.int.example " {
|
||||
t.Errorf("foreign record touched: %q", got["10.213.77.20/24"])
|
||||
}
|
||||
if got["10.213.77.22/24"] != "node002.int.example m-2" || got["10.213.77.23/24"] != "node002.int.example m-2" {
|
||||
t.Errorf("owned/new records: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPAMReconcile(t *testing.T) {
|
||||
a, f := testIPAM(t)
|
||||
ctx := context.Background()
|
||||
f.add("10.213.77.20/24", "printer.int.example", nil, "") // foreign, collides with a local address
|
||||
f.add("10.213.77.30/24", "gone.int.example", []string{"rollcall"}, "m-9") // ours, released locally
|
||||
f.add("10.213.77.50/24", "wrong.int.example", []string{"rollcall"}, "m-1")
|
||||
f.add("10.213.77.99/24", "keep.int.example", []string{"other"}, "m-3") // not tagged: never deleted
|
||||
local := []IPAddr{
|
||||
{IP: "10.213.77.20", Subnet: "10.213.77.0/24", MachineID: "m-2", Hostname: "node002", Kind: "auto"},
|
||||
{IP: "10.213.77.50", Subnet: "10.213.77.0/24", MachineID: "m-1", Hostname: "node001", Kind: "auto", Iface: "bond0"},
|
||||
{IP: "10.213.77.51", Subnet: "10.213.77.0/24", MachineID: "m-4", Hostname: "node004", Kind: "static", Iface: "eth0"},
|
||||
}
|
||||
dry := ipamReport{DryRun: true}
|
||||
a.reconcileAddrs(ctx, local, &dry)
|
||||
if len(f.ips) != 4 {
|
||||
t.Fatalf("dry run changed records: %d", len(f.ips))
|
||||
}
|
||||
if fmt.Sprint(dry.Created, dry.Updated, dry.Deleted) != "[10.213.77.20 10.213.77.51] [10.213.77.50] [10.213.77.30]" {
|
||||
t.Errorf("dry plan: %+v", dry)
|
||||
}
|
||||
rep := ipamReport{}
|
||||
a.reconcileAddrs(ctx, local, &rep)
|
||||
if rep.Error != "" {
|
||||
t.Fatal(rep.Error)
|
||||
}
|
||||
if fmt.Sprint(rep.Created, rep.Updated, rep.Deleted, len(rep.Conflicts)) != "[10.213.77.51] [10.213.77.50] [10.213.77.30] 1" {
|
||||
t.Errorf("report: %+v", rep)
|
||||
}
|
||||
addrs := map[string]string{}
|
||||
for _, x := range f.ips {
|
||||
addrs[x["address"].(string)] = x["dns_name"].(string)
|
||||
}
|
||||
want := map[string]string{"10.213.77.20/24": "printer.int.example", "10.213.77.50/24": "node001.int.example",
|
||||
"10.213.77.51/24": "node004.int.example", "10.213.77.99/24": "keep.int.example"}
|
||||
if fmt.Sprint(addrs) != fmt.Sprint(want) {
|
||||
t.Errorf("records after reconcile:\n got %v\nwant %v", addrs, want)
|
||||
}
|
||||
again := ipamReport{}
|
||||
a.reconcileAddrs(ctx, local, &again)
|
||||
if n := len(again.Created) + len(again.Updated) + len(again.Deleted); n != 0 {
|
||||
t.Errorf("second pass not idempotent: %+v", again)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIPAMPrefixes(t *testing.T) {
|
||||
a, _ := testIPAM(t)
|
||||
ps, err := a.ipam.Prefixes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(ps) != 2 || ps[0].Site != "Ev" || ps[1].VID != 300 {
|
||||
t.Errorf("prefixes: %+v", ps)
|
||||
}
|
||||
}
|
||||
+214
-39
@@ -1,69 +1,244 @@
|
||||
package main
|
||||
|
||||
// A machine's BMC: the record commissioning (or an operator) leaves in
|
||||
// machines.bmc, the sealed password, and the API that sets and probes it.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// bmcCreds is what the server needs to talk to a machine's BMC.
|
||||
// bmcCreds is what a power driver needs.
|
||||
type bmcCreds struct {
|
||||
Address string
|
||||
Port int
|
||||
User string
|
||||
Password string
|
||||
}
|
||||
|
||||
var powerActions = map[string][]string{
|
||||
"status": {"chassis", "power", "status"},
|
||||
"on": {"chassis", "power", "on"},
|
||||
"off": {"chassis", "power", "off"},
|
||||
"soft": {"chassis", "power", "soft"},
|
||||
"cycle": {"chassis", "power", "cycle"},
|
||||
"reset": {"chassis", "power", "reset"},
|
||||
Driver string `json:"driver"`
|
||||
Address string `json:"address"`
|
||||
Port int `json:"port"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
Extra map[string]string `json:"extra"`
|
||||
}
|
||||
|
||||
var bootDevices = map[string]bool{"pxe": true, "disk": true, "cdrom": true, "bios": true, "none": true, "safe": true}
|
||||
|
||||
type bmcRecord struct {
|
||||
Driver string `json:"driver"`
|
||||
Address string `json:"address"`
|
||||
Port int `json:"port"`
|
||||
User string `json:"user"`
|
||||
Extra map[string]string `json:"extra"`
|
||||
}
|
||||
|
||||
func (a *App) credsFor(m *Machine) (*bmcCreds, error) {
|
||||
if m.BMC == nil || m.bmcSecret == nil {
|
||||
return nil, fmt.Errorf("no BMC credentials stored")
|
||||
}
|
||||
var b struct {
|
||||
Address string `json:"address"`
|
||||
Port int `json:"port"`
|
||||
User string `json:"user"`
|
||||
}
|
||||
var b bmcRecord
|
||||
if err := json.Unmarshal(m.BMC, &b); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if b.Address == "" || b.User == "" {
|
||||
return nil, fmt.Errorf("BMC record has no address/user")
|
||||
if b.Driver == "manual" {
|
||||
return &bmcCreds{Driver: "manual"}, nil
|
||||
}
|
||||
if b.Address == "" {
|
||||
return nil, fmt.Errorf("BMC record has no address")
|
||||
}
|
||||
pw, err := a.sealer.Open(m.bmcSecret, m.ID)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("decrypt BMC secret: %w", err)
|
||||
}
|
||||
if b.Port == 0 {
|
||||
b.Port = 623
|
||||
if b.Driver == "" {
|
||||
b.Driver = "ipmi"
|
||||
}
|
||||
return &bmcCreds{Address: b.Address, Port: b.Port, User: b.User, Password: pw}, nil
|
||||
if b.Extra == nil {
|
||||
b.Extra = map[string]string{}
|
||||
}
|
||||
return &bmcCreds{Driver: b.Driver, Address: b.Address, Port: b.Port, User: b.User, Password: pw, Extra: b.Extra}, nil
|
||||
}
|
||||
|
||||
// ipmi runs ipmitool server-side; the password never leaves this process.
|
||||
func (a *App) ipmi(ctx context.Context, c *bmcCreds, args ...string) (string, error) {
|
||||
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
full := append([]string{"-I", "lanplus", "-H", c.Address, "-p", strconv.Itoa(c.Port), "-U", c.User, "-E"}, args...)
|
||||
cmd := exec.CommandContext(ctx, a.cfg.IPMITool, full...)
|
||||
cmd.Env = []string{"IPMI_PASSWORD=" + c.Password, "PATH=/usr/sbin:/usr/bin:/sbin:/bin"}
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &out
|
||||
err := cmd.Run()
|
||||
return strings.TrimSpace(out.String()), err
|
||||
var errNoBMC = errors.New("no BMC credentials stored")
|
||||
|
||||
// power runs a power operation for a machine on the rack controller that can reach its BMC.
|
||||
func (a *App) power(ctx context.Context, m *Machine, op, arg string) (string, error) {
|
||||
c, err := a.credsFor(m)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: %v", errNoBMC, err)
|
||||
}
|
||||
return a.powerVia(ctx, a.rackOf(m), c, op, arg)
|
||||
}
|
||||
|
||||
func (a *App) powerVia(ctx context.Context, rack string, c *bmcCreds, op, arg string) (string, error) {
|
||||
if rack == a.cfg.RackName || rack == "" {
|
||||
return powerRun(ctx, a.cfg.IPMITool, c, op, arg)
|
||||
}
|
||||
r, ok := a.racksSnapshot()[rack]
|
||||
if !ok || !rackOnline(r, a.cfg.RackName) {
|
||||
return "", fmt.Errorf("rack controller %s is offline", rack)
|
||||
}
|
||||
return a.hub.dispatch(ctx, rack, &rackJob{Kind: "power", Driver: c.Driver, Address: c.Address, Port: c.Port, User: c.User,
|
||||
Password: c.Password, Extra: c.Extra, Op: op, Arg: arg})
|
||||
}
|
||||
|
||||
func (s *Store) SetBMC(ctx context.Context, id string, bmc []byte, secret []byte) error {
|
||||
if secret != nil {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2, bmc_secret=$3 WHERE id=$1`, id, bmc, secret)
|
||||
return err
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2 WHERE id=$1`, id, bmc)
|
||||
return err
|
||||
}
|
||||
|
||||
// probeBMC: after commissioning, a BMC still on plain IPMI is asked whether it
|
||||
// speaks Redfish with the same account; if it does, the vendor's driver takes over.
|
||||
// An operator's explicit choice (driver_set) is left alone.
|
||||
func (a *App) probeBMC(m *Machine) {
|
||||
if m.BMC == nil || m.bmcSecret == nil {
|
||||
return
|
||||
}
|
||||
var b obj
|
||||
if json.Unmarshal(m.BMC, &b) != nil || b["driver_set"] == true || str(b["address"]) == "" {
|
||||
return
|
||||
}
|
||||
if d := str(b["driver"]); d != "" && d != "ipmi" {
|
||||
return
|
||||
}
|
||||
c, err := a.credsFor(m)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
probe := *c
|
||||
probe.Driver, probe.Port = "redfish", 0
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90e9)
|
||||
defer cancel()
|
||||
out, err := a.powerVia(ctx, a.rackOf(m), &probe, "probe", "")
|
||||
if err != nil {
|
||||
_ = a.store.AddEvent(ctx, m.ID, "bmc probe", "staying on ipmi: "+oneLine(err.Error()))
|
||||
return
|
||||
}
|
||||
var res obj
|
||||
if json.Unmarshal([]byte(out), &res) != nil || str(res["driver"]) == "" {
|
||||
return
|
||||
}
|
||||
extra, _ := b["extra"].(map[string]any)
|
||||
if extra == nil {
|
||||
extra = map[string]any{}
|
||||
}
|
||||
extra["system"] = str(res["system"])
|
||||
b["driver"], b["extra"], b["redfish"] = str(res["driver"]), extra, res
|
||||
nb, _ := json.Marshal(b)
|
||||
if err := a.store.SetBMC(ctx, m.ID, nb, nil); err != nil {
|
||||
log.Printf("probe %s: %v", m.ID, err)
|
||||
return
|
||||
}
|
||||
_ = a.store.AddEvent(ctx, m.ID, "bmc driver", fmt.Sprintf("%s (Redfish %s, %s %s)", str(res["driver"]), str(res["redfish"]), str(res["manufacturer"]), str(res["model"])))
|
||||
log.Printf("%s: BMC speaks Redfish %s (%s) -> driver %s", m.ID, str(res["redfish"]), str(res["vendor"]), str(res["driver"]))
|
||||
}
|
||||
|
||||
// handleBMC: GET shows the record (no secret); POST sets driver, address,
|
||||
// account and extra settings, checked with a status call first when verify is set.
|
||||
func (a *App) handleBMC(w http.ResponseWriter, r *http.Request) {
|
||||
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
var b obj
|
||||
_ = json.Unmarshal(m.BMC, &b)
|
||||
if b == nil {
|
||||
b = obj{}
|
||||
}
|
||||
if r.Method == http.MethodGet {
|
||||
writeJSON(w, 200, obj{"id": m.ID, "bmc": b, "has_secret": m.bmcSecret != nil, "drivers": powerDrivers})
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Driver *string `json:"driver"`
|
||||
Address *string `json:"address"`
|
||||
Port *int `json:"port"`
|
||||
User *string `json:"user"`
|
||||
Password *string `json:"password"`
|
||||
Extra map[string]string `json:"extra"`
|
||||
Verify bool `json:"verify"`
|
||||
Probe bool `json:"probe"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
if req.Probe { // re-run Redfish detection
|
||||
b["driver_set"] = false
|
||||
b["driver"] = "ipmi"
|
||||
nb, _ := json.Marshal(b)
|
||||
_ = a.store.SetBMC(r.Context(), m.ID, nb, nil)
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
a.probeBMC(m)
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
_ = json.Unmarshal(m.BMC, &b)
|
||||
writeJSON(w, 200, obj{"id": m.ID, "bmc": b})
|
||||
return
|
||||
}
|
||||
if req.Driver != nil {
|
||||
if _, ok := driverByName(*req.Driver); !ok {
|
||||
fail(w, 400, fmt.Errorf("unknown driver %q", *req.Driver))
|
||||
return
|
||||
}
|
||||
b["driver"] = *req.Driver
|
||||
}
|
||||
if req.Address != nil {
|
||||
b["address"] = strings.TrimSpace(*req.Address)
|
||||
}
|
||||
if req.Port != nil {
|
||||
b["port"] = *req.Port
|
||||
}
|
||||
if req.User != nil {
|
||||
b["user"] = *req.User
|
||||
}
|
||||
if req.Extra != nil {
|
||||
ex, _ := b["extra"].(map[string]any)
|
||||
if ex == nil {
|
||||
ex = map[string]any{}
|
||||
}
|
||||
for k, v := range req.Extra {
|
||||
if v == "" {
|
||||
delete(ex, k)
|
||||
} else {
|
||||
ex[k] = v
|
||||
}
|
||||
}
|
||||
b["extra"] = ex
|
||||
}
|
||||
b["driver_set"] = true
|
||||
if str(b["source"]) == "" {
|
||||
b["source"] = "operator"
|
||||
}
|
||||
secret := m.bmcSecret
|
||||
if req.Password != nil {
|
||||
if secret, err = a.sealer.Seal(*req.Password, m.ID); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
nb, _ := json.Marshal(b)
|
||||
trial := *m
|
||||
trial.BMC, trial.bmcSecret = nb, secret
|
||||
out := ""
|
||||
if req.Verify {
|
||||
out, err = a.power(r.Context(), &trial, "status", "")
|
||||
if err != nil {
|
||||
fail(w, 409, fmt.Errorf("not saved, the BMC did not answer: %v %s", err, out))
|
||||
return
|
||||
}
|
||||
b["configured"], b["verified_oob"] = true, true
|
||||
nb, _ = json.Marshal(b)
|
||||
}
|
||||
if err := a.store.SetBMC(r.Context(), m.ID, nb, secret); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "bmc set", fmt.Sprintf("driver=%s address=%s user=%s", str(b["driver"]), str(b["address"]), str(b["user"])))
|
||||
writeJSON(w, 200, obj{"id": m.ID, "bmc": b, "output": out})
|
||||
}
|
||||
@@ -0,0 +1,456 @@
|
||||
package main
|
||||
|
||||
// Physical location: building / room / rack / unit (and an asset tag) per machine,
|
||||
// plus optional rack definitions (height) for the elevation view. Locations are
|
||||
// set by hand, in bulk, or imported from a CSV exported by a DCIM / asset list.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Location struct {
|
||||
Building string `json:"building"`
|
||||
Room string `json:"room"`
|
||||
Rack string `json:"rack"`
|
||||
Unit int `json:"unit"` // lowest U occupied, 0 = unknown
|
||||
AssetTag string `json:"asset_tag"`
|
||||
}
|
||||
|
||||
func (l Location) String() string {
|
||||
var p []string
|
||||
for _, s := range []string{l.Building, l.Room, l.Rack} {
|
||||
if s != "" {
|
||||
p = append(p, s)
|
||||
}
|
||||
}
|
||||
s := strings.Join(p, " / ")
|
||||
if l.Unit > 0 {
|
||||
s += fmt.Sprintf(" U%d", l.Unit)
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func (l Location) empty() bool {
|
||||
return l.Building == "" && l.Room == "" && l.Rack == "" && l.Unit == 0 && l.AssetTag == ""
|
||||
}
|
||||
|
||||
type Rack struct {
|
||||
Building string `json:"building"`
|
||||
Room string `json:"room"`
|
||||
Name string `json:"name"`
|
||||
Height int `json:"height"`
|
||||
Comment string `json:"comment"`
|
||||
Machines int `json:"machines,omitempty"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- store
|
||||
|
||||
func (s *Store) SetLocation(ctx context.Context, id string, l Location) error {
|
||||
tag, err := s.pool.Exec(ctx, `UPDATE machines SET building=nullif($2,''), room=nullif($3,''), rack=nullif($4,''), rack_unit=nullif($5,0), asset_tag=nullif($6,'') WHERE id=$1`,
|
||||
id, strings.TrimSpace(l.Building), strings.TrimSpace(l.Room), strings.TrimSpace(l.Rack), l.Unit, strings.TrimSpace(l.AssetTag))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return errNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Store) Racks(ctx context.Context) ([]Rack, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT building, room, name, height, coalesce(comment,'') FROM racks ORDER BY building, room, name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []Rack{}
|
||||
for rows.Next() {
|
||||
var r Rack
|
||||
if err := rows.Scan(&r.Building, &r.Room, &r.Name, &r.Height, &r.Comment); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) PutRack(ctx context.Context, r Rack) error {
|
||||
if r.Height <= 0 {
|
||||
r.Height = 42
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO racks(building, room, name, height, comment) VALUES ($1,$2,$3,$4,$5)
|
||||
ON CONFLICT (building, room, name) DO UPDATE SET height=$4, comment=$5`, r.Building, r.Room, r.Name, r.Height, r.Comment)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) DeleteRack(ctx context.Context, building, room, name string) error {
|
||||
_, err := s.pool.Exec(ctx, `DELETE FROM racks WHERE building=$1 AND room=$2 AND name=$3`, building, room, name)
|
||||
return err
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- handlers
|
||||
|
||||
// handleLocation sets one machine's location: {"building","room","rack","unit","asset_tag"}.
|
||||
// Fields that are absent keep their value; "" or 0 clears one.
|
||||
func (a *App) handleLocation(w http.ResponseWriter, r *http.Request) {
|
||||
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
var req map[string]any
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
l := m.Location
|
||||
if v, ok := req["building"]; ok {
|
||||
l.Building = str(v)
|
||||
}
|
||||
if v, ok := req["room"]; ok {
|
||||
l.Room = str(v)
|
||||
}
|
||||
if v, ok := req["rack"]; ok {
|
||||
l.Rack = str(v)
|
||||
}
|
||||
if v, ok := req["unit"]; ok {
|
||||
l.Unit = atoi(v)
|
||||
}
|
||||
if v, ok := req["asset_tag"]; ok {
|
||||
l.AssetTag = str(v)
|
||||
}
|
||||
if err := a.store.SetLocation(r.Context(), m.ID, l); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.ensureRack(r.Context(), l)
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "location", l.String())
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
a.retag(r.Context(), []*Machine{m}) // rules may look at the rack
|
||||
writeJSON(w, 200, obj{"id": m.ID, "location": m.Location})
|
||||
}
|
||||
|
||||
// ensureRack creates a rack definition (default height) the first time a machine lands in it.
|
||||
func (a *App) ensureRack(ctx context.Context, l Location) {
|
||||
if l.Rack == "" {
|
||||
return
|
||||
}
|
||||
racks, _ := a.store.Racks(ctx)
|
||||
for _, r := range racks {
|
||||
if r.Building == l.Building && r.Room == l.Room && r.Name == l.Rack {
|
||||
return
|
||||
}
|
||||
}
|
||||
_ = a.store.PutRack(ctx, Rack{Building: l.Building, Room: l.Room, Name: l.Rack})
|
||||
}
|
||||
|
||||
func (a *App) handleRacks(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
racks, err := a.store.Racks(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
ms, _ := a.store.List(r.Context())
|
||||
for i := range racks {
|
||||
for _, m := range ms {
|
||||
if m.Location.Building == racks[i].Building && m.Location.Room == racks[i].Room && m.Location.Rack == racks[i].Name {
|
||||
racks[i].Machines++
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, racks)
|
||||
case http.MethodPost:
|
||||
var rk Rack
|
||||
if err := json.NewDecoder(r.Body).Decode(&rk); err != nil || rk.Name == "" {
|
||||
fail(w, 400, errors.New("name is required"))
|
||||
return
|
||||
}
|
||||
if err := a.store.PutRack(r.Context(), rk); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, rk)
|
||||
case http.MethodDelete:
|
||||
q := r.URL.Query()
|
||||
if err := a.store.DeleteRack(r.Context(), q.Get("building"), q.Get("room"), q.Get("name")); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, obj{"deleted": q.Get("name")})
|
||||
}
|
||||
}
|
||||
|
||||
// handleLocationImport takes CSV (text/csv) or JSON rows. Columns (header names, any order):
|
||||
//
|
||||
// id | serial | mac | node_name one of these identifies the machine
|
||||
// building, room, rack, unit, asset_tag
|
||||
//
|
||||
// Response lists matched / unmatched rows. Dry run with ?dry=1.
|
||||
func (a *App) handleLocationImport(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 8<<20))
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
var rows []map[string]string
|
||||
if strings.HasPrefix(strings.TrimSpace(string(body)), "[") {
|
||||
var raw []map[string]any
|
||||
if err := json.Unmarshal(body, &raw); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
for _, x := range raw {
|
||||
row := map[string]string{}
|
||||
for k, v := range x {
|
||||
row[strings.ToLower(k)] = strings.TrimSpace(str(v))
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
} else {
|
||||
rd := csv.NewReader(strings.NewReader(string(body)))
|
||||
rd.FieldsPerRecord = -1
|
||||
rd.TrimLeadingSpace = true
|
||||
recs, err := rd.ReadAll()
|
||||
if err != nil {
|
||||
fail(w, 400, fmt.Errorf("csv: %w", err))
|
||||
return
|
||||
}
|
||||
if len(recs) < 2 {
|
||||
fail(w, 400, errors.New("csv needs a header and at least one row"))
|
||||
return
|
||||
}
|
||||
hdr := recs[0]
|
||||
for i := range hdr {
|
||||
hdr[i] = strings.ToLower(strings.TrimSpace(strings.TrimPrefix(hdr[i], "\xef\xbb\xbf"))) // strip UTF-8 BOM
|
||||
}
|
||||
for _, rec := range recs[1:] {
|
||||
row := map[string]string{}
|
||||
for i, v := range rec {
|
||||
if i < len(hdr) {
|
||||
row[hdr[i]] = strings.TrimSpace(v)
|
||||
}
|
||||
}
|
||||
rows = append(rows, row)
|
||||
}
|
||||
}
|
||||
ms, err := a.store.List(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
find := func(row map[string]string) *Machine {
|
||||
for _, m := range ms {
|
||||
if id := row["id"]; id != "" && m.ID == id {
|
||||
return m
|
||||
}
|
||||
if sn := row["serial"]; sn != "" && strings.EqualFold(deref(m.Serial), sn) {
|
||||
return m
|
||||
}
|
||||
if nn := row["node_name"]; nn != "" && deref(m.NodeName) == nn {
|
||||
return m
|
||||
}
|
||||
if mac := normMAC(row["mac"]); mac != "" && hasTag(m.MACs, mac) {
|
||||
return m
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
dry := r.URL.Query().Get("dry") == "1"
|
||||
var matched, unmatched []obj
|
||||
for i, row := range rows {
|
||||
m := find(row)
|
||||
if m == nil {
|
||||
unmatched = append(unmatched, obj{"row": i + 1, "keys": obj{"id": row["id"], "serial": row["serial"], "mac": row["mac"], "node_name": row["node_name"]}})
|
||||
continue
|
||||
}
|
||||
l := m.Location
|
||||
if v, ok := row["building"]; ok {
|
||||
l.Building = v
|
||||
}
|
||||
if v, ok := row["room"]; ok {
|
||||
l.Room = v
|
||||
}
|
||||
if v, ok := row["rack"]; ok {
|
||||
l.Rack = v
|
||||
}
|
||||
if v, ok := row["unit"]; ok {
|
||||
l.Unit, _ = strconv.Atoi(strings.TrimPrefix(strings.ToUpper(v), "U"))
|
||||
}
|
||||
if v, ok := row["asset_tag"]; ok {
|
||||
l.AssetTag = v
|
||||
}
|
||||
matched = append(matched, obj{"row": i + 1, "id": m.ID, "name": deref(m.NodeName), "location": l})
|
||||
if !dry {
|
||||
if err := a.store.SetLocation(r.Context(), m.ID, l); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.ensureRack(r.Context(), l)
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "location", l.String()+" (import)")
|
||||
}
|
||||
}
|
||||
if !dry {
|
||||
a.retag(r.Context(), nil)
|
||||
}
|
||||
if matched == nil {
|
||||
matched = []obj{}
|
||||
}
|
||||
if unmatched == nil {
|
||||
unmatched = []obj{}
|
||||
}
|
||||
writeJSON(w, 200, obj{"dry_run": dry, "matched": matched, "unmatched": unmatched})
|
||||
}
|
||||
|
||||
// locationFilter applies ?building= ?room= ?rack= to the list.
|
||||
func locationMatch(m *Machine, q map[string][]string) bool {
|
||||
for _, k := range []string{"building", "room", "rack"} {
|
||||
if v := q[k]; len(v) > 0 && v[0] != "" {
|
||||
var have string
|
||||
switch k {
|
||||
case "building":
|
||||
have = m.Location.Building
|
||||
case "room":
|
||||
have = m.Location.Room
|
||||
case "rack":
|
||||
have = m.Location.Rack
|
||||
}
|
||||
if !strings.EqualFold(have, v[0]) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- CLI
|
||||
|
||||
func (c *client) locationCmd(args []string) error {
|
||||
if len(args) < 1 {
|
||||
usage()
|
||||
}
|
||||
switch args[0] {
|
||||
case "import":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
fs := newFlags("location import")
|
||||
dry := fs.Bool("dry", false, "show what would change")
|
||||
_ = fs.Parse(args[2:])
|
||||
b, err := readFileArg(args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
q := ""
|
||||
if *dry {
|
||||
q = "?dry=1"
|
||||
}
|
||||
data, err := c.rawBody("POST", "/api/v1/locations/import"+q, "text/csv", b)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var res struct {
|
||||
DryRun bool `json:"dry_run"`
|
||||
Matched []obj `json:"matched"`
|
||||
Unmatched []obj `json:"unmatched"`
|
||||
}
|
||||
_ = json.Unmarshal(data, &res)
|
||||
for _, m := range res.Matched {
|
||||
lb, _ := json.Marshal(m["location"])
|
||||
fmt.Printf("row %v: %s %s -> %s\n", m["row"], str(m["id"]), str(m["name"]), lb)
|
||||
}
|
||||
for _, u := range res.Unmatched {
|
||||
kb, _ := json.Marshal(u["keys"])
|
||||
fmt.Printf("row %v: NO MATCH %s\n", u["row"], kb)
|
||||
}
|
||||
fmt.Printf("%d matched, %d unmatched%s\n", len(res.Matched), len(res.Unmatched), map[bool]string{true: " (dry run)", false: ""}[res.DryRun])
|
||||
case "racks":
|
||||
var racks []Rack
|
||||
if err := c.do("GET", "/api/v1/racks", nil, &racks); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-14s %-14s %-12s %6s %8s %s\n", "BUILDING", "ROOM", "RACK", "HEIGHT", "MACHINES", "COMMENT")
|
||||
for _, r := range racks {
|
||||
fmt.Printf("%-14s %-14s %-12s %6d %8d %s\n", r.Building, r.Room, r.Name, r.Height, r.Machines, r.Comment)
|
||||
}
|
||||
case "rack":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
fs := newFlags("location rack")
|
||||
b := fs.String("building", "", "building")
|
||||
rm := fs.String("room", "", "room")
|
||||
h := fs.Int("height", 42, "rack units")
|
||||
cm := fs.String("comment", "", "comment")
|
||||
_ = fs.Parse(args[2:])
|
||||
return c.do("POST", "/api/v1/racks", Rack{Building: *b, Room: *rm, Name: args[1], Height: *h, Comment: *cm}, nil)
|
||||
default: // location ID -building B -room R -rack K -unit N -asset T
|
||||
fs := newFlags("location")
|
||||
b := fs.String("building", "\x00", "building")
|
||||
rm := fs.String("room", "\x00", "room")
|
||||
rk := fs.String("rack", "\x00", "rack")
|
||||
u := fs.Int("unit", -1, "lowest rack unit")
|
||||
at := fs.String("asset", "\x00", "asset tag")
|
||||
_ = fs.Parse(args[1:])
|
||||
ids, err := c.targets(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req := obj{}
|
||||
if *b != "\x00" {
|
||||
req["building"] = *b
|
||||
}
|
||||
if *rm != "\x00" {
|
||||
req["room"] = *rm
|
||||
}
|
||||
if *rk != "\x00" {
|
||||
req["rack"] = *rk
|
||||
}
|
||||
if *u >= 0 {
|
||||
req["unit"] = *u
|
||||
}
|
||||
if *at != "\x00" {
|
||||
req["asset_tag"] = *at
|
||||
}
|
||||
if len(req) == 0 {
|
||||
var ms []Summary
|
||||
if err := c.do("GET", "/api/v1/machines", nil, &ms); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, m := range ms {
|
||||
if hasTag(ids, m.ID) {
|
||||
fmt.Printf("%s %s: %s\n", m.ID, deref(m.NodeName), m.Location.String())
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
for _, id := range ids {
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/machines/"+id+"/location", req, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
lb, _ := json.Marshal(d["location"])
|
||||
fmt.Printf("%s -> %s\n", id, lb)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedKeys(m map[string]bool) []string {
|
||||
out := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
out = append(out, k)
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
+74
-4
@@ -24,13 +24,22 @@ var version = "dev"
|
||||
|
||||
func usage() {
|
||||
_, _ = os.Stderr.WriteString(`usage:
|
||||
rollcall serve [-config PATH]
|
||||
rollcall in a terminal: the full-screen console
|
||||
rollcall tui [-dump WxH -page P -machine ID -plain]
|
||||
rollcall serve [-config PATH] region: API, database, web UI, embedded rack controller
|
||||
rollcall doctor [-deep] [-no-dhcp-probe] check this installation and say what to fix
|
||||
rollcall net discover [-json] what each interface of this host sees (LLDP, DHCP, neighbours)
|
||||
rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines
|
||||
rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate)
|
||||
rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host)
|
||||
rollcall ca intermediate NAME -permit DOMAIN,CIDR [-csr FILE] [-days 365] | rotate -yes delegate to an ACME CA / replace the root
|
||||
rollcall version
|
||||
rollcall keygen
|
||||
rollcall list
|
||||
rollcall show ID [-json]
|
||||
rollcall events ID
|
||||
rollcall power ID status|on|off|soft|cycle|reset
|
||||
rollcall bmc ID [-driver D] [-address A] [-user U] [-password P|-] [-set K=V] [-verify] [-probe] | bmc drivers
|
||||
rollcall bootdev ID pxe|disk|cdrom|bios|none
|
||||
rollcall recommission ID [-now]
|
||||
rollcall delete ID
|
||||
@@ -44,7 +53,25 @@ func usage() {
|
||||
rollcall deploy ID OS [-version V] [-disk /dev/sdX] [-hostname H] [-upgrade] [-user-data FILE] [-boot] [-watch]
|
||||
rollcall release ID [-boot] [-watch]
|
||||
rollcall jobs ID [-watch] | job cancel ID
|
||||
env: ROLLCALL_URL (default from config listen), ROLLCALL_TOKEN, ROLLCALL_CONFIG
|
||||
rollcall tag ID +NAME... -NAME... manual tags on a machine
|
||||
rollcall tags list | add NAME [-match TMPL] [-overlays a,b] [-cloud-config FILE] [-comment C] | show NAME | rm NAME
|
||||
rollcall location ID [-building B] [-room R] [-rack K] [-unit N] [-asset T] (no flags: show)
|
||||
rollcall location import FILE.csv [-dry] columns: id|serial|mac|node_name, building, room, rack, unit, asset_tag
|
||||
rollcall location racks | rack NAME -building B -room R [-height 42]
|
||||
rollcall list [-tag T] [-building B] [-room R] [-rack K]
|
||||
rollcall fabric list | add NAME [-kind ethernet|infiniband] | rm NAME
|
||||
rollcall vlan list | add FABRIC VID [-dhcp] [-rack R] [-secondary R] [-mtu N] [-name N] | rm FABRIC VID
|
||||
rollcall subnet list | show CIDR | add CIDR -fabric F [-vid N] [-purpose P] [-gateway G] [-dns a,b] [-domain D] | rm CIDR [-force]
|
||||
rollcall range add CIDR dynamic|reserved|static START END | rm ID
|
||||
rollcall ip list [-subnet CIDR] | reserve IP [-name N] | release IP
|
||||
rollcall admin-cert request NAME | list | revoke NAME|SERIAL admin client certificate (browser/CLI without the token)
|
||||
rollcall ipam [status] | sync [-dry] | prefixes | import external IPAM (NetBox): state, reconcile, delegated prefixes
|
||||
rollcall layout list | show NAME | add NAME FILE.json | rm NAME
|
||||
rollcall net show ID | set ID [-layout L] [-rack R] [-static IFACE=IP] [-unstatic IP]
|
||||
rollcall controllers | controller rm NAME | controller join-code [-name N] [-ttl 60] [-addr HOST:443] | dhcp CONTROLLER
|
||||
(deploy, power and location take tag:NAME instead of ID to act on every machine with that tag)
|
||||
env: ROLLCALL_URL (default from config listen), ROLLCALL_TOKEN, ROLLCALL_CONFIG;
|
||||
away from the server the console saves URL and token in ~/.config/rollcall/client.json
|
||||
`)
|
||||
os.Exit(2)
|
||||
}
|
||||
@@ -53,11 +80,53 @@ func newFlags(name string) *flag.FlagSet { return flag.NewFlagSet(name, flag.Exi
|
||||
|
||||
func main() {
|
||||
log.SetFlags(log.Ldate | log.Ltime | log.Lmicroseconds)
|
||||
if len(os.Args) > 2 && os.Args[1] == "net" && os.Args[2] == "discover" {
|
||||
os.Args = append([]string{os.Args[0], "net-discover"}, os.Args[3:]...)
|
||||
}
|
||||
if len(os.Args) < 2 {
|
||||
usage()
|
||||
// A bare `rollcall` in a terminal opens the console; in a pipe or a
|
||||
// script it prints its help, as it always did.
|
||||
if !interactive() {
|
||||
usage()
|
||||
}
|
||||
if err := runTUI(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
return
|
||||
}
|
||||
cmd, args := os.Args[1], os.Args[2:]
|
||||
switch cmd {
|
||||
case "rack":
|
||||
if len(args) > 0 && args[0] == "join" {
|
||||
if err := rackJoin(args[1:]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
if err := runRack(args); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
case "ca":
|
||||
if err := caCmd(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case "net-discover":
|
||||
if err := discoverCmd(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case "doctor":
|
||||
if err := doctorCmd(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case "tui", "console":
|
||||
if err := tuiDump(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case "serve":
|
||||
fs := flag.NewFlagSet("serve", flag.ExitOnError)
|
||||
cfg := fs.String("config", envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"), "config file")
|
||||
@@ -71,7 +140,8 @@ func main() {
|
||||
fmt.Println(randomToken(24))
|
||||
case "list", "show", "events", "power", "bootdev", "recommission", "delete",
|
||||
"images", "profile", "assign", "unassign", "overlay", "render",
|
||||
"deploy", "release", "jobs", "job":
|
||||
"deploy", "release", "jobs", "job", "tag", "tags", "location",
|
||||
"bmc", "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp", "ipam", "admin-cert":
|
||||
if err := runClient(cmd, args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
|
||||
@@ -0,0 +1,975 @@
|
||||
package main
|
||||
|
||||
// HTTP API and CLI for the network model and rack controllers.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// ---------------------------------------------------------------- HTTP
|
||||
|
||||
func (a *App) netChanged(r *http.Request) {
|
||||
a.reloadNet(r.Context())
|
||||
a.hub.bump()
|
||||
}
|
||||
|
||||
func (a *App) handleFabrics(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet {
|
||||
fs, err := a.store.Fabrics(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, fs)
|
||||
return
|
||||
}
|
||||
var f Fabric
|
||||
if err := json.NewDecoder(r.Body).Decode(&f); err != nil || !tagNameRe.MatchString(f.Name) {
|
||||
fail(w, 400, errors.New("name is required: lowercase letters, digits, . _ -"))
|
||||
return
|
||||
}
|
||||
if f.Kind != "" && f.Kind != "ethernet" && f.Kind != "infiniband" {
|
||||
fail(w, 400, errors.New("kind is ethernet or infiniband"))
|
||||
return
|
||||
}
|
||||
if err := a.store.PutFabric(r.Context(), f); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, f)
|
||||
}
|
||||
|
||||
func (a *App) handleFabricDelete(w http.ResponseWriter, r *http.Request) {
|
||||
if err := a.store.DeleteFabric(r.Context(), r.PathValue("name")); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, obj{"deleted": r.PathValue("name")})
|
||||
}
|
||||
|
||||
func (a *App) handleVLANs(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet {
|
||||
vs, err := a.store.VLANs(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, vs)
|
||||
return
|
||||
}
|
||||
var v VLAN
|
||||
if err := json.NewDecoder(r.Body).Decode(&v); err != nil || v.Fabric == "" || v.VID < 0 || v.VID > 4094 && v.VID < 0x8000 {
|
||||
fail(w, 400, errors.New("fabric and vid (0 untagged, 1-4094, or an IB pkey) are required"))
|
||||
return
|
||||
}
|
||||
if v.DHCP && v.PrimaryRack == "" {
|
||||
v.PrimaryRack = a.cfg.RackName
|
||||
}
|
||||
racks := a.racksSnapshot()
|
||||
for _, rk := range []string{v.PrimaryRack, v.SecondaryRack} {
|
||||
if _, ok := racks[rk]; rk != "" && !ok {
|
||||
fail(w, 400, fmt.Errorf("rack controller %s is not registered", rk))
|
||||
return
|
||||
}
|
||||
}
|
||||
id, err := a.store.PutVLAN(r.Context(), v)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
v.ID = id
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, v)
|
||||
}
|
||||
|
||||
func (a *App) handleVLANDelete(w http.ResponseWriter, r *http.Request) {
|
||||
vid, _ := strconv.Atoi(r.PathValue("vid"))
|
||||
if err := a.store.DeleteVLAN(r.Context(), r.PathValue("fabric"), vid); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, obj{"deleted": fmt.Sprintf("%s/%d", r.PathValue("fabric"), vid)})
|
||||
}
|
||||
|
||||
func (a *App) handleSubnets(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet {
|
||||
ss, err := a.store.Subnets(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, ss)
|
||||
return
|
||||
}
|
||||
var sn Subnet
|
||||
if err := json.NewDecoder(r.Body).Decode(&sn); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
_, n, err := net.ParseCIDR(sn.CIDR)
|
||||
if err != nil {
|
||||
fail(w, 400, errors.New("cidr must be a network, e.g. 10.0.0.0/24 or fd00:1::/64"))
|
||||
return
|
||||
}
|
||||
sn.CIDR = n.String()
|
||||
if familyOf(sn.CIDR) == 6 {
|
||||
if ones, _ := n.Mask.Size(); ones > 126 {
|
||||
fail(w, 400, errors.New("an IPv6 subnet needs room: /126 or shorter (/64 is usual)"))
|
||||
return
|
||||
}
|
||||
}
|
||||
if sn.Gateway != "" && !sn.contains(sn.Gateway) {
|
||||
fail(w, 400, fmt.Errorf("gateway %s is outside %s", sn.Gateway, sn.CIDR))
|
||||
return
|
||||
}
|
||||
switch sn.Purpose {
|
||||
case "", "provisioning", "mgmt", "bmc", "storage", "ib", "other":
|
||||
default:
|
||||
fail(w, 400, errors.New("purpose is provisioning, mgmt, bmc, storage, ib or other"))
|
||||
return
|
||||
}
|
||||
if sn.Fabric != "" && sn.VLANID == 0 {
|
||||
vs, _ := a.store.VLANs(r.Context())
|
||||
for _, v := range vs {
|
||||
if v.Fabric == sn.Fabric && v.VID == sn.VID {
|
||||
sn.VLANID = v.ID
|
||||
}
|
||||
}
|
||||
if sn.VLANID == 0 { // a subnet on a VLAN not declared yet: create it
|
||||
fs, _ := a.store.Fabrics(r.Context())
|
||||
known := false
|
||||
for _, f := range fs {
|
||||
known = known || f.Name == sn.Fabric
|
||||
}
|
||||
if !known {
|
||||
fail(w, 400, fmt.Errorf("fabric %s does not exist", sn.Fabric))
|
||||
return
|
||||
}
|
||||
if sn.VLANID, err = a.store.PutVLAN(r.Context(), VLAN{Fabric: sn.Fabric, VID: sn.VID}); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, other := range a.subnetsSnapshot() {
|
||||
if other.CIDR == sn.CIDR {
|
||||
continue
|
||||
}
|
||||
if _, on, err := net.ParseCIDR(other.CIDR); err == nil && (on.Contains(n.IP) || n.Contains(on.IP)) {
|
||||
fail(w, 409, fmt.Errorf("%s overlaps %s", sn.CIDR, other.CIDR))
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := a.store.PutSubnet(r.Context(), sn); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.adoptIPs(r.Context())
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, sn)
|
||||
}
|
||||
|
||||
func (a *App) handleSubnetDelete(w http.ResponseWriter, r *http.Request) {
|
||||
cidr := r.PathValue("cidr")
|
||||
if ips, _ := a.store.IPs(r.Context(), cidr, ""); len(ips) > 0 && r.URL.Query().Get("force") != "1" {
|
||||
fail(w, 409, fmt.Errorf("%s still has %d address(es) handed out (force=1 drops them)", cidr, len(ips)))
|
||||
return
|
||||
}
|
||||
if err := a.store.DeleteSubnet(r.Context(), cidr); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, obj{"deleted": cidr})
|
||||
}
|
||||
|
||||
func (a *App) handleRanges(w http.ResponseWriter, r *http.Request) {
|
||||
var rg IPRange
|
||||
if err := json.NewDecoder(r.Body).Decode(&rg); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
if rg.Kind != "dynamic" && rg.Kind != "reserved" && rg.Kind != "static" {
|
||||
fail(w, 400, errors.New("kind is dynamic, reserved or static"))
|
||||
return
|
||||
}
|
||||
sn := subnetByRef(a.subnetsSnapshot(), rg.Subnet)
|
||||
if sn == nil {
|
||||
fail(w, 404, fmt.Errorf("subnet %s: %w", rg.Subnet, errSubnetUnknown))
|
||||
return
|
||||
}
|
||||
lo, ok1 := parseAddr(rg.Start)
|
||||
hi, ok2 := parseAddr(rg.End)
|
||||
if !ok1 || !ok2 || hi.Less(lo) || !sn.contains(rg.Start) || !sn.contains(rg.End) {
|
||||
fail(w, 400, fmt.Errorf("range %s-%s is not inside %s", rg.Start, rg.End, sn.CIDR))
|
||||
return
|
||||
}
|
||||
rg.Start, rg.End = lo.String(), hi.String()
|
||||
for _, o := range sn.Ranges {
|
||||
ol, _ := parseAddr(o.Start)
|
||||
oh, _ := parseAddr(o.End)
|
||||
if !oh.Less(lo) && !hi.Less(ol) {
|
||||
fail(w, 409, fmt.Errorf("overlaps the %s range %s-%s", o.Kind, o.Start, o.End))
|
||||
return
|
||||
}
|
||||
}
|
||||
rg.Subnet = sn.CIDR
|
||||
id, err := a.store.AddRange(r.Context(), rg)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
rg.ID = id
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, rg)
|
||||
}
|
||||
|
||||
func (a *App) handleRangeDelete(w http.ResponseWriter, r *http.Request) {
|
||||
id, _ := strconv.ParseInt(r.PathValue("id"), 10, 64)
|
||||
if err := a.store.DeleteRange(r.Context(), id); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, obj{"deleted": id})
|
||||
}
|
||||
|
||||
func (a *App) handleIPs(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
ips, err := a.store.IPs(r.Context(), r.URL.Query().Get("subnet"), r.URL.Query().Get("machine"))
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, ips)
|
||||
case http.MethodPost: // reserve one address (a switch, a PDU, a VIP)
|
||||
var x IPAddr
|
||||
if err := json.NewDecoder(r.Body).Decode(&x); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
sn := subnetFor(a.subnetsSnapshot(), x.IP)
|
||||
if sn == nil {
|
||||
fail(w, 404, fmt.Errorf("%s is in no subnet", x.IP))
|
||||
return
|
||||
}
|
||||
ok, err := a.claimIP(r.Context(), IPAddr{IP: x.IP, Subnet: sn.CIDR, Kind: "reserved", Hostname: x.Hostname})
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
if !ok {
|
||||
fail(w, 409, fmt.Errorf("%s is taken", x.IP))
|
||||
return
|
||||
}
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, obj{"reserved": x.IP, "subnet": sn.CIDR})
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleIPDelete(w http.ResponseWriter, r *http.Request) {
|
||||
if err := a.store.ReleaseIP(r.Context(), r.PathValue("ip")); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.kickIPAM()
|
||||
a.netChanged(r)
|
||||
writeJSON(w, 200, obj{"released": r.PathValue("ip")})
|
||||
}
|
||||
|
||||
func validLayout(l NetLayout) error {
|
||||
if !tagNameRe.MatchString(l.Name) {
|
||||
return errors.New("name is required: lowercase letters, digits, . _ -")
|
||||
}
|
||||
if len(l.Interfaces) == 0 {
|
||||
return errors.New("a layout needs at least one interface")
|
||||
}
|
||||
names := map[string]bool{}
|
||||
for _, i := range l.Interfaces {
|
||||
switch i.Type {
|
||||
case "", "physical":
|
||||
case "bond":
|
||||
if len(i.Members) == 0 {
|
||||
return fmt.Errorf("bond %s has no members", i.Name)
|
||||
}
|
||||
case "vlan":
|
||||
if !names[i.Parent] || i.VID < 1 || i.VID > 4094 {
|
||||
return fmt.Errorf("vlan %q needs a parent defined above it and a vid 1-4094", i.Name)
|
||||
}
|
||||
default:
|
||||
return fmt.Errorf("interface type %q: physical, bond or vlan", i.Type)
|
||||
}
|
||||
for _, lk := range i.Links {
|
||||
switch lk.Mode {
|
||||
case "", "auto", "static", "dhcp", "link_up":
|
||||
default:
|
||||
return fmt.Errorf("link mode %q: auto, static, dhcp or link_up", lk.Mode)
|
||||
}
|
||||
}
|
||||
n := i.Name
|
||||
if n == "" && i.Type == "vlan" {
|
||||
n = fmt.Sprintf("%s.%d", i.Parent, i.VID)
|
||||
}
|
||||
names[n] = true
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) handleLayouts(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method == http.MethodGet {
|
||||
ls, err := a.store.Layouts(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
ms, _ := a.store.List(r.Context())
|
||||
for i := range ls {
|
||||
for _, m := range ms {
|
||||
if l, _ := a.layoutFor(r.Context(), m); l != nil && strings.HasPrefix(l.Name, ls[i].Name) {
|
||||
ls[i].Machines++
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, ls)
|
||||
return
|
||||
}
|
||||
var l NetLayout
|
||||
if err := json.NewDecoder(r.Body).Decode(&l); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := validLayout(l); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := a.store.PutLayout(r.Context(), l); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, l)
|
||||
}
|
||||
|
||||
func (a *App) handleLayoutDelete(w http.ResponseWriter, r *http.Request) {
|
||||
if err := a.store.DeleteLayout(r.Context(), r.PathValue("name")); err != nil {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, obj{"deleted": r.PathValue("name")})
|
||||
}
|
||||
|
||||
// handleMachineNetwork: GET shows the plan and the network-config it renders
|
||||
// (without handing out addresses); POST sets the layout and pinned addresses.
|
||||
func (a *App) handleMachineNetwork(w http.ResponseWriter, r *http.Request) {
|
||||
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost {
|
||||
var req struct {
|
||||
Layout *string `json:"layout"`
|
||||
Rack *string `json:"rack_controller"`
|
||||
Static []struct{ Iface, IP string }
|
||||
Unstatic []string `json:"unstatic"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
if req.Layout != nil {
|
||||
if *req.Layout != "" {
|
||||
ls, _ := a.store.Layouts(r.Context())
|
||||
found := false
|
||||
for _, l := range ls {
|
||||
found = found || l.Name == *req.Layout
|
||||
}
|
||||
if !found {
|
||||
fail(w, 404, fmt.Errorf("layout %s does not exist", *req.Layout))
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := a.store.SetMachineLayout(r.Context(), m.ID, *req.Layout); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if req.Rack != nil {
|
||||
if _, ok := a.racksSnapshot()[*req.Rack]; *req.Rack != "" && !ok {
|
||||
fail(w, 404, fmt.Errorf("rack controller %s is not registered", *req.Rack))
|
||||
return
|
||||
}
|
||||
if err := a.store.SetRackController(r.Context(), m.ID, *req.Rack); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
_ = a.writePXEAny(r.Context(), m) // its PXE file moves to the new rack
|
||||
}
|
||||
for _, ip := range req.Unstatic {
|
||||
have, _ := a.store.IPs(r.Context(), "", m.ID)
|
||||
for _, x := range have {
|
||||
if x.IP == ip && x.Kind == "static" {
|
||||
_ = a.store.ReleaseIP(r.Context(), ip)
|
||||
a.kickIPAM()
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, st := range req.Static {
|
||||
sn := subnetFor(a.subnetsSnapshot(), st.IP)
|
||||
if sn == nil {
|
||||
fail(w, 400, fmt.Errorf("%s is in no subnet", st.IP))
|
||||
return
|
||||
}
|
||||
have, _ := a.store.IPs(r.Context(), sn.CIDR, m.ID)
|
||||
for _, x := range have { // one pinned address per interface and subnet
|
||||
if x.Iface == st.Iface && x.IP != st.IP {
|
||||
_ = a.store.ReleaseIP(r.Context(), x.IP)
|
||||
}
|
||||
}
|
||||
if ok, err := a.claimIP(r.Context(), IPAddr{IP: st.IP, Subnet: sn.CIDR, MachineID: m.ID, Iface: st.Iface, Kind: "static", Hostname: deref(m.NodeName)}); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
} else if !ok {
|
||||
cur, _ := a.store.IPs(r.Context(), sn.CIDR, m.ID)
|
||||
mine := false
|
||||
for _, x := range cur {
|
||||
mine = mine || x.IP == st.IP
|
||||
}
|
||||
if !mine {
|
||||
fail(w, 409, fmt.Errorf("%s is taken", st.IP))
|
||||
return
|
||||
}
|
||||
_, _ = a.store.pool.Exec(r.Context(), `UPDATE ip_addresses SET kind='static', iface=$2 WHERE ip=$1`, st.IP, st.Iface)
|
||||
}
|
||||
}
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "network", fmt.Sprintf("layout=%v static=%v unstatic=%v", deref(req.Layout), req.Static, req.Unstatic))
|
||||
a.netChanged(r)
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
}
|
||||
plan, err := a.netPlan(r.Context(), m, false)
|
||||
if err != nil {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
ips, _ := a.store.IPs(r.Context(), "", m.ID)
|
||||
writeJSON(w, 200, obj{"id": m.ID, "layout": plan.Layout, "plan": plan, "network_config": networkConfig(plan), "addresses": ips,
|
||||
"rack_controller": a.rackOf(m), "modeled": len(a.subnetsSnapshot()) > 0})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- CLI
|
||||
|
||||
func (c *client) netCmd(cmd string, args []string) error {
|
||||
need := func(n int) {
|
||||
if len(args) < n {
|
||||
usage()
|
||||
}
|
||||
}
|
||||
sub := ""
|
||||
if len(args) > 0 {
|
||||
sub = args[0]
|
||||
}
|
||||
switch cmd {
|
||||
case "fabric":
|
||||
switch sub {
|
||||
case "", "list":
|
||||
var fs []Fabric
|
||||
if err := c.do("GET", "/api/v1/fabrics", nil, &fs); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-16s %-11s %s\n", "FABRIC", "KIND", "COMMENT")
|
||||
for _, f := range fs {
|
||||
fmt.Printf("%-16s %-11s %s\n", f.Name, f.Kind, f.Comment)
|
||||
}
|
||||
case "add":
|
||||
need(2)
|
||||
fs := newFlags("fabric add")
|
||||
kind := fs.String("kind", "ethernet", "ethernet | infiniband")
|
||||
comment := fs.String("comment", "", "comment")
|
||||
_ = fs.Parse(args[2:])
|
||||
return c.do("POST", "/api/v1/fabrics", Fabric{Name: args[1], Kind: *kind, Comment: *comment}, nil)
|
||||
case "rm":
|
||||
need(2)
|
||||
return c.do("DELETE", "/api/v1/fabrics/"+args[1], nil, nil)
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
case "vlan":
|
||||
switch sub {
|
||||
case "", "list":
|
||||
var vs []VLAN
|
||||
if err := c.do("GET", "/api/v1/vlans", nil, &vs); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-14s %5s %-14s %5s %-5s %-14s %s\n", "FABRIC", "VID", "NAME", "MTU", "DHCP", "RACK", "SECONDARY")
|
||||
for _, v := range vs {
|
||||
fmt.Printf("%-14s %5d %-14s %5d %-5v %-14s %s\n", v.Fabric, v.VID, v.Name, v.MTU, v.DHCP, v.PrimaryRack, v.SecondaryRack)
|
||||
}
|
||||
case "add", "set":
|
||||
need(3)
|
||||
fs := newFlags("vlan add")
|
||||
name := fs.String("name", "", "name")
|
||||
mtu := fs.Int("mtu", 1500, "MTU")
|
||||
dhcp := fs.Bool("dhcp", false, "serve DHCP/PXE on this VLAN")
|
||||
rack := fs.String("rack", "", "rack controller serving it (default: the region's embedded one)")
|
||||
sec := fs.String("secondary", "", "second rack controller (the dynamic range is split)")
|
||||
_ = fs.Parse(args[3:])
|
||||
vid, _ := strconv.Atoi(args[2])
|
||||
var v VLAN
|
||||
if err := c.do("POST", "/api/v1/vlans", VLAN{Fabric: args[1], VID: vid, Name: *name, MTU: *mtu, DHCP: *dhcp, PrimaryRack: *rack, SecondaryRack: *sec}, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("vlan %s/%d: dhcp=%v rack=%s\n", v.Fabric, v.VID, v.DHCP, v.PrimaryRack)
|
||||
case "rm":
|
||||
need(3)
|
||||
return c.do("DELETE", "/api/v1/vlans/"+args[1]+"/"+args[2], nil, nil)
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
case "subnet":
|
||||
switch sub {
|
||||
case "", "list":
|
||||
var ss []Subnet
|
||||
if err := c.do("GET", "/api/v1/subnets", nil, &ss); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-24s %-12s %-14s %-12s %-15s %-12s %9s %s\n", "SUBNET", "PURPOSE", "FABRIC/VLAN", "DHCP RACK", "GATEWAY", "DOMAIN", "USED", "RANGES")
|
||||
for _, s := range ss {
|
||||
dhcp := "-"
|
||||
if s.DHCP {
|
||||
dhcp = s.PrimaryRack
|
||||
if s.SecondaryRack != "" {
|
||||
dhcp += "+" + s.SecondaryRack
|
||||
}
|
||||
}
|
||||
var rs []string
|
||||
for _, r := range s.Ranges {
|
||||
rs = append(rs, fmt.Sprintf("%s:%s-%s", r.Kind, r.Start, lastOctets(r.End)))
|
||||
}
|
||||
size := strconv.Itoa(s.Size)
|
||||
if s.Size == 0 {
|
||||
size = "∞"
|
||||
}
|
||||
fmt.Printf("%-24s %-12s %-14s %-12s %-15s %-12s %4d/%-4s %s\n", s.CIDR, s.Purpose, fmt.Sprintf("%s/%d", orDash(s.Fabric), s.VID), dhcp, orDash(s.Gateway), orDash(s.Domain), s.Used, size, strings.Join(rs, " "))
|
||||
}
|
||||
case "show":
|
||||
need(2)
|
||||
var ss []Subnet
|
||||
if err := c.do("GET", "/api/v1/subnets", nil, &ss); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, s := range ss {
|
||||
if s.CIDR == args[1] || s.Name == args[1] {
|
||||
fmt.Printf("%s %s fabric %s vlan %d mtu %d dhcp=%v rack=%s %s\n gateway %s dns %v domain %s\n", s.CIDR, s.Purpose, s.Fabric, s.VID, s.MTU, s.DHCP, s.PrimaryRack, s.SecondaryRack, orDash(s.Gateway), s.DNS, orDash(s.Domain))
|
||||
for _, r := range s.Ranges {
|
||||
fmt.Printf(" range %-4d %-8s %s - %s %s\n", r.ID, r.Kind, r.Start, r.End, r.Comment)
|
||||
}
|
||||
var ips []IPAddr
|
||||
if err := c.do("GET", "/api/v1/ips?subnet="+s.CIDR, nil, &ips); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf(" %d of %d addresses in use\n", len(ips), s.Size)
|
||||
for _, x := range ips {
|
||||
fmt.Printf(" %-15s %-8s %-16s %-10s %s\n", x.IP, x.Kind, orDash(x.MachineID), x.Iface, x.Hostname)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errNotFound
|
||||
case "add", "set":
|
||||
need(2)
|
||||
fs := newFlags("subnet add")
|
||||
fabric := fs.String("fabric", "", "fabric (the VLAN is created when missing)")
|
||||
vid := fs.Int("vid", 0, "VLAN id (0 = untagged)")
|
||||
purpose := fs.String("purpose", "provisioning", "provisioning | mgmt | bmc | storage | ib | other")
|
||||
gw := fs.String("gateway", "", "default gateway")
|
||||
dns := fs.String("dns", "", "comma-separated DNS servers")
|
||||
domain := fs.String("domain", "", "DNS domain")
|
||||
name := fs.String("name", "", "name")
|
||||
comment := fs.String("comment", "", "comment")
|
||||
_ = fs.Parse(args[2:])
|
||||
sn := Subnet{CIDR: args[1], Fabric: *fabric, VID: *vid, Purpose: *purpose, Gateway: *gw, Domain: *domain, Name: *name, Comment: *comment, DNS: []string{}}
|
||||
if *dns != "" {
|
||||
sn.DNS = strings.Split(*dns, ",")
|
||||
}
|
||||
var out Subnet
|
||||
if err := c.do("POST", "/api/v1/subnets", sn, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("subnet %s saved\n", out.CIDR)
|
||||
case "rm":
|
||||
need(2)
|
||||
q := ""
|
||||
if len(args) > 2 && args[2] == "-force" {
|
||||
q = "?force=1"
|
||||
}
|
||||
return c.do("DELETE", "/api/v1/subnets/"+args[1]+q, nil, nil)
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
case "range":
|
||||
switch sub {
|
||||
case "add":
|
||||
need(5)
|
||||
fs := newFlags("range add")
|
||||
comment := fs.String("comment", "", "comment")
|
||||
_ = fs.Parse(args[5:])
|
||||
var rg IPRange
|
||||
if err := c.do("POST", "/api/v1/ranges", IPRange{Subnet: args[1], Kind: args[2], Start: args[3], End: args[4], Comment: *comment}, &rg); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("range %d: %s %s-%s on %s\n", rg.ID, rg.Kind, rg.Start, rg.End, rg.Subnet)
|
||||
case "rm":
|
||||
need(2)
|
||||
return c.do("DELETE", "/api/v1/ranges/"+args[1], nil, nil)
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
case "ip":
|
||||
switch sub {
|
||||
case "", "list":
|
||||
fs := newFlags("ip list")
|
||||
subnet := fs.String("subnet", "", "only this subnet")
|
||||
_ = fs.Parse(args[min(1, len(args)):])
|
||||
var ips []IPAddr
|
||||
if err := c.do("GET", "/api/v1/ips?subnet="+*subnet, nil, &ips); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-15s %-18s %-8s %-16s %-10s %s\n", "IP", "SUBNET", "KIND", "MACHINE", "IFACE", "HOSTNAME")
|
||||
for _, x := range ips {
|
||||
fmt.Printf("%-15s %-18s %-8s %-16s %-10s %s\n", x.IP, x.Subnet, x.Kind, orDash(x.MachineID), x.Iface, x.Hostname)
|
||||
}
|
||||
case "reserve":
|
||||
need(2)
|
||||
fs := newFlags("ip reserve")
|
||||
host := fs.String("name", "", "what it is (a switch, a VIP)")
|
||||
_ = fs.Parse(args[2:])
|
||||
return c.do("POST", "/api/v1/ips", IPAddr{IP: args[1], Hostname: *host}, nil)
|
||||
case "release":
|
||||
need(2)
|
||||
return c.do("DELETE", "/api/v1/ips/"+args[1], nil, nil)
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
case "layout":
|
||||
switch sub {
|
||||
case "", "list":
|
||||
var ls []NetLayout
|
||||
if err := c.do("GET", "/api/v1/layouts", nil, &ls); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-18s %8s %-50s %s\n", "LAYOUT", "MACHINES", "INTERFACES", "COMMENT")
|
||||
for _, l := range ls {
|
||||
fmt.Printf("%-18s %8d %-50s %s\n", l.Name, l.Machines, trunc(layoutSummary(l), 50), l.Comment)
|
||||
}
|
||||
case "show":
|
||||
need(2)
|
||||
var ls []NetLayout
|
||||
if err := c.do("GET", "/api/v1/layouts", nil, &ls); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, l := range ls {
|
||||
if l.Name == args[1] {
|
||||
b, _ := json.MarshalIndent(l, "", " ")
|
||||
fmt.Println(string(b))
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errNotFound
|
||||
case "add":
|
||||
need(3)
|
||||
fs := newFlags("layout add")
|
||||
comment := fs.String("comment", "", "comment")
|
||||
_ = fs.Parse(args[3:])
|
||||
b, err := readFileArg(args[2])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var l NetLayout
|
||||
if err := json.Unmarshal(b, &l); err != nil {
|
||||
return fmt.Errorf("%s: %w", args[2], err)
|
||||
}
|
||||
l.Name = args[1]
|
||||
if *comment != "" {
|
||||
l.Comment = *comment
|
||||
}
|
||||
return c.do("POST", "/api/v1/layouts", l, nil)
|
||||
case "rm":
|
||||
need(2)
|
||||
return c.do("DELETE", "/api/v1/layouts/"+args[1], nil, nil)
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
case "net":
|
||||
need(2)
|
||||
switch sub {
|
||||
case "show":
|
||||
var d obj
|
||||
if err := c.do("GET", "/api/v1/machines/"+args[1]+"/network", nil, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
printNet(d)
|
||||
case "set":
|
||||
fs := newFlags("net set")
|
||||
layout := fs.String("layout", "\x00", "layout name (\"\" = from tags / default)")
|
||||
rack := fs.String("rack", "\x00", "rack controller serving it (a machine moved to another segment)")
|
||||
var statics, unstatic multiFlag
|
||||
fs.Var(&statics, "static", "IFACE=IP pins an address (repeatable)")
|
||||
fs.Var(&unstatic, "unstatic", "IP drops a pinned address (repeatable)")
|
||||
_ = fs.Parse(args[2:])
|
||||
req := obj{"unstatic": []string(unstatic)}
|
||||
if *layout != "\x00" {
|
||||
req["layout"] = *layout
|
||||
}
|
||||
if *rack != "\x00" {
|
||||
req["rack_controller"] = *rack
|
||||
}
|
||||
var st []obj
|
||||
for _, s := range statics {
|
||||
i, ip, ok := strings.Cut(s, "=")
|
||||
if !ok {
|
||||
return fmt.Errorf("-static wants IFACE=IP")
|
||||
}
|
||||
st = append(st, obj{"iface": i, "ip": ip})
|
||||
}
|
||||
req["static"] = st
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/machines/"+args[1]+"/network", req, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
printNet(d)
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
case "controllers", "controller":
|
||||
switch {
|
||||
case cmd == "controller" && sub == "rm":
|
||||
need(2)
|
||||
return c.do("DELETE", "/api/v1/controllers/"+args[1], nil, nil)
|
||||
case cmd == "controller" && sub == "join-code":
|
||||
fs := newFlags("controller join-code")
|
||||
name := fs.String("name", "", "bind the code to this rack controller name")
|
||||
ttl := fs.Int("ttl", 60, "minutes the code is good for")
|
||||
addr := fs.String("addr", "", "host:port the rack reaches the region's HTTPS at (default: how you reached it)")
|
||||
_ = fs.Parse(args[1:])
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/controllers/join", obj{"name": *name, "ttl_minutes": *ttl, "addr": *addr}, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s\n\none use, until %s; region %s, CA %s\non the rack host: rollcall rack join <code> -url http://<its address on the segment>\n",
|
||||
str(d["code"]), str(d["expires"]), str(d["addr"]), str(d["ca_fingerprint"]))
|
||||
return nil
|
||||
default:
|
||||
var rs []RackController
|
||||
if err := c.do("GET", "/api/v1/controllers", nil, &rs); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-14s %-9s %-7s %-24s %8s %-22s %-20s %s\n", "CONTROLLER", "KIND", "ONLINE", "URL", "MACHINES", "VERSION", "LAST SEEN", "INTERFACES")
|
||||
for _, r := range rs {
|
||||
var ifs []string
|
||||
for _, i := range r.Interfaces {
|
||||
ifs = append(ifs, i.Name+"="+strings.Join(i.Addrs, ","))
|
||||
}
|
||||
fmt.Printf("%-14s %-9s %-7v %-24s %8d %-22s %-20s %s\n", r.Name, r.Kind, r.Online, r.URL, r.Machines, orDash(r.Version), timeStr(r.LastSeen), strings.Join(ifs, " "))
|
||||
}
|
||||
}
|
||||
case "dhcp":
|
||||
need(1)
|
||||
data, err := c.raw("GET", "/api/v1/controllers/"+args[0]+"/dhcp")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
os.Stdout.Write(data)
|
||||
case "ipam":
|
||||
return c.ipamCmd(sub, args)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *client) ipamCmd(sub string, args []string) error {
|
||||
printRep := func(rep ipamReport) {
|
||||
for _, x := range []struct {
|
||||
k string
|
||||
v []string
|
||||
}{{"created", rep.Created}, {"updated", rep.Updated}, {"deleted", rep.Deleted}, {"conflict", rep.Conflicts}} {
|
||||
for _, ip := range x.v {
|
||||
fmt.Printf("%-9s %s\n", x.k, ip)
|
||||
}
|
||||
}
|
||||
dry := ""
|
||||
if rep.DryRun {
|
||||
dry = " (dry run, nothing changed)"
|
||||
}
|
||||
fmt.Printf("%s: %d created, %d updated, %d deleted, %d conflicts%s\n", orDash(rep.Provider),
|
||||
len(rep.Created), len(rep.Updated), len(rep.Deleted), len(rep.Conflicts), dry)
|
||||
if rep.Error != "" {
|
||||
fmt.Println("error:", rep.Error)
|
||||
}
|
||||
}
|
||||
switch sub {
|
||||
case "", "status":
|
||||
var st struct {
|
||||
Provider string `json:"provider"`
|
||||
URL string `json:"url"`
|
||||
Zone string `json:"dns_zone"`
|
||||
Tag string `json:"tag"`
|
||||
Field string `json:"machine_field"`
|
||||
Last ipamReport `json:"last_sync"`
|
||||
}
|
||||
if err := c.do("GET", "/api/v1/ipam", nil, &st); err != nil {
|
||||
return err
|
||||
}
|
||||
if st.Provider == "" {
|
||||
fmt.Println("built-in IPAM only (set ipam.provider in config.json for NetBox)")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("provider %s %s\nzone %s\nowner tag %q, custom field %q\n", st.Provider, st.URL, orDash(st.Zone), st.Tag, st.Field)
|
||||
if st.Last.At.IsZero() {
|
||||
fmt.Println("last sync never")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("last sync %s\n", st.Last.At.Local().Format("2006-01-02 15:04:05"))
|
||||
printRep(st.Last)
|
||||
case "sync":
|
||||
fs := newFlags("ipam sync")
|
||||
dry := fs.Bool("dry", false, "show what would change")
|
||||
_ = fs.Parse(args[min(1, len(args)):])
|
||||
q := ""
|
||||
if *dry {
|
||||
q = "?dry=1"
|
||||
}
|
||||
var rep ipamReport
|
||||
err := c.do("POST", "/api/v1/ipam/sync"+q, nil, &rep)
|
||||
if rep.Provider != "" || rep.Error != "" {
|
||||
printRep(rep)
|
||||
}
|
||||
return err
|
||||
case "prefixes", "import":
|
||||
var res struct {
|
||||
Prefixes []extPrefix `json:"prefixes"`
|
||||
Imported []string `json:"imported"`
|
||||
}
|
||||
method := "GET"
|
||||
if sub == "import" {
|
||||
method = "POST"
|
||||
}
|
||||
if err := c.do(method, "/api/v1/ipam/prefixes", nil, &res); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-22s %-6s %-5s %-10s %s\n", "PREFIX", "LOCAL", "VLAN", "SITE", "DESCRIPTION")
|
||||
for _, p := range res.Prefixes {
|
||||
vid := "-"
|
||||
if p.VID > 0 {
|
||||
vid = strconv.Itoa(p.VID)
|
||||
}
|
||||
local := "no"
|
||||
if p.Local {
|
||||
local = "yes"
|
||||
if p.Comment != "" {
|
||||
local = p.Comment
|
||||
}
|
||||
}
|
||||
fmt.Printf("%-22s %-6s %-5s %-10s %s\n", p.Prefix, local, vid, orDash(p.Site), p.Desc)
|
||||
}
|
||||
if sub == "prefixes" {
|
||||
fmt.Println("prefixes tagged for Rollcall; `rollcall ipam import` creates the missing subnets")
|
||||
}
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
type multiFlag []string
|
||||
|
||||
func (m *multiFlag) String() string { return strings.Join(*m, ",") }
|
||||
func (m *multiFlag) Set(v string) error { *m = append(*m, v); return nil }
|
||||
|
||||
func lastOctets(ip string) string {
|
||||
if strings.Contains(ip, ":") {
|
||||
if i := strings.LastIndex(ip, ":"); i >= 0 {
|
||||
return ip[i:]
|
||||
}
|
||||
}
|
||||
if i := strings.LastIndex(ip, "."); i >= 0 {
|
||||
return ip[i+1:]
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
func layoutSummary(l NetLayout) string {
|
||||
var p []string
|
||||
for _, i := range l.Interfaces {
|
||||
var links []string
|
||||
for _, lk := range i.Links {
|
||||
links = append(links, lk.Subnet+":"+orDash(lk.Mode))
|
||||
}
|
||||
switch i.Type {
|
||||
case "bond":
|
||||
p = append(p, fmt.Sprintf("%s(%s)[%s]", orDash(i.Name), strings.Join(i.Members, "+"), strings.Join(links, ",")))
|
||||
case "vlan":
|
||||
p = append(p, fmt.Sprintf("%s.%d[%s]", i.Parent, i.VID, strings.Join(links, ",")))
|
||||
default:
|
||||
p = append(p, fmt.Sprintf("%s[%s]", orDash(i.Select), strings.Join(links, ",")))
|
||||
}
|
||||
}
|
||||
return strings.Join(p, " ")
|
||||
}
|
||||
|
||||
func printNet(d obj) {
|
||||
fmt.Printf("%s layout %s rack controller %s\n", str(d["id"]), str(d["layout"]), str(d["rack_controller"]))
|
||||
if d["modeled"] == false {
|
||||
fmt.Println(" no subnets are defined: deployed machines use DHCP (rollcall subnet add …)")
|
||||
}
|
||||
plan, _ := d["plan"].(obj)
|
||||
for _, iv := range toList(plan["interfaces"]) {
|
||||
i, _ := iv.(obj)
|
||||
desc := str(i["type"])
|
||||
switch desc {
|
||||
case "physical":
|
||||
desc = str(i["mac"])
|
||||
case "bond":
|
||||
var ms []string
|
||||
for _, x := range toList(i["members"]) {
|
||||
ms = append(ms, str(x))
|
||||
}
|
||||
desc = "bond " + str(i["bond_mode"]) + " of " + strings.Join(ms, ",")
|
||||
case "vlan":
|
||||
desc = fmt.Sprintf("vlan %s on %s", str(i["vid"]), str(i["parent"]))
|
||||
}
|
||||
mtu := ""
|
||||
if num(i["mtu"]) > 0 {
|
||||
mtu = fmt.Sprintf(" mtu %d", int(num(i["mtu"])))
|
||||
}
|
||||
fmt.Printf(" %-12s %s%s\n", str(i["name"]), desc, mtu)
|
||||
for _, lv := range toList(i["links"]) {
|
||||
l, _ := lv.(obj)
|
||||
ip := str(l["ip"])
|
||||
if ip == "" && str(l["mode"]) == "auto" {
|
||||
ip = "(allocated at deploy)"
|
||||
} else if ip != "" {
|
||||
ip = fmt.Sprintf("%s/%d", ip, int(num(l["prefix"])))
|
||||
}
|
||||
gw := ""
|
||||
if l["default_gateway"] == true {
|
||||
gw = " default via " + str(l["gateway"])
|
||||
}
|
||||
fmt.Printf(" %-8s %-18s %s%s\n", str(l["mode"]), orDash(str(l["subnet"])), ip, gw)
|
||||
}
|
||||
}
|
||||
for _, w := range toList(plan["warnings"]) {
|
||||
fmt.Printf(" ! %s\n", str(w))
|
||||
}
|
||||
b, _ := json.MarshalIndent(d["network_config"], " ", " ")
|
||||
fmt.Printf(" network-config:\n %s\n", b)
|
||||
}
|
||||
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,291 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func labSubnet() Subnet {
|
||||
return Subnet{CIDR: "10.0.0.0/24", Gateway: "10.0.0.1", DNS: []string{"10.0.0.1"}, Domain: "lab.local", Fabric: "f0", MTU: 1500,
|
||||
DHCP: true, PrimaryRack: "r1", Purpose: "provisioning",
|
||||
Ranges: []IPRange{{Kind: "reserved", Start: "10.0.0.2", End: "10.0.0.20"}, {Kind: "dynamic", Start: "10.0.0.100", End: "10.0.0.200"}}}
|
||||
}
|
||||
|
||||
func TestFreeCandidates(t *testing.T) {
|
||||
sn := labSubnet()
|
||||
got := freeCandidates(sn, map[string]bool{"10.0.0.21": true}, []string{"10.0.0.22/24"}, 3)
|
||||
if strings.Join(got, " ") != "10.0.0.23 10.0.0.24 10.0.0.25" {
|
||||
t.Errorf("candidates: %v", got)
|
||||
}
|
||||
for _, ip := range freeCandidates(sn, nil, nil, 0) {
|
||||
if ip == "10.0.0.1" || ip == "10.0.0.0" || ip == "10.0.0.255" || (ipLess("10.0.0.99", ip) && ipLess(ip, "10.0.0.201")) {
|
||||
t.Fatalf("handed out %s", ip)
|
||||
}
|
||||
}
|
||||
sn.Ranges = append(sn.Ranges, IPRange{Kind: "static", Start: "10.0.0.50", End: "10.0.0.52"})
|
||||
if got := freeCandidates(sn, nil, nil, 0); strings.Join(got, " ") != "10.0.0.50 10.0.0.51 10.0.0.52" {
|
||||
t.Errorf("static range: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func allocFrom(next ...string) func(Subnet, string) (string, error) {
|
||||
return func(Subnet, string) (string, error) {
|
||||
ip := next[0]
|
||||
next = next[1:]
|
||||
return ip, nil
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultPlanAndNetworkConfig(t *testing.T) {
|
||||
nics := []invNIC{{Name: "eno1", MAC: "aa:bb:cc:00:00:01", Speed: 1000, Link: true, IPv4: []string{"10.0.0.150/24"}}}
|
||||
p := resolvePlan(nil, nics, &nics[0], []Subnet{labSubnet()}, nil, allocFrom("10.0.0.21"))
|
||||
if len(p.Warnings) > 0 || len(p.Ifaces) != 1 {
|
||||
t.Fatalf("plan: %+v", p)
|
||||
}
|
||||
if mgmt, _ := p.primaryIPs(); mgmt != "10.0.0.21" {
|
||||
t.Errorf("mgmt %q", mgmt)
|
||||
}
|
||||
b, _ := json.Marshal(networkConfig(p))
|
||||
s := string(b)
|
||||
for _, want := range []string{`"macaddress":"aa:bb:cc:00:00:01"`, `"set-name":"eno1"`, `"addresses":["10.0.0.21/24"]`,
|
||||
`"routes":[{"to":"0.0.0.0/0","via":"10.0.0.1"}]`, `"search":["lab.local"]`, `"dhcp4":false`, `"accept-ra":false`} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Errorf("network-config lacks %s:\n%s", want, s)
|
||||
}
|
||||
}
|
||||
// a pinned address wins over allocation
|
||||
p = resolvePlan(nil, nics, &nics[0], []Subnet{labSubnet()}, []IPAddr{{IP: "10.0.0.60", Subnet: "10.0.0.0/24", Iface: "eno1", Kind: "static"}}, allocFrom("10.0.0.21"))
|
||||
if p.Ifaces[0].Links[0].IP != "10.0.0.60" || p.Ifaces[0].Links[0].Mode != "static" {
|
||||
t.Errorf("static pin ignored: %+v", p.Ifaces[0].Links[0])
|
||||
}
|
||||
// no subnet model for the boot network: DHCP, as before
|
||||
p = resolvePlan(nil, nics, &nics[0], nil, nil, nil)
|
||||
if p.Ifaces[0].Links[0].Mode != "dhcp" {
|
||||
t.Errorf("without subnets the default is dhcp, got %+v", p.Ifaces[0].Links)
|
||||
}
|
||||
}
|
||||
|
||||
func TestBondVLANLayout(t *testing.T) {
|
||||
nics := []invNIC{
|
||||
{Name: "eno1", MAC: "aa:00:00:00:00:01", Speed: 1000, Link: true, IPv4: []string{"10.0.0.150/24"}},
|
||||
{Name: "ens1f0", MAC: "aa:00:00:00:00:02", Speed: 100000, Link: true},
|
||||
{Name: "ens1f1", MAC: "aa:00:00:00:00:03", Speed: 100000, Link: true},
|
||||
{Name: "ibp1s0", MAC: "80:00:00:00", IB: true, Link: true},
|
||||
}
|
||||
storage := Subnet{CIDR: "10.20.0.0/24", Fabric: "f0", VID: 200, MTU: 9000}
|
||||
ib := Subnet{CIDR: "10.30.0.0/24", Fabric: "ib0"}
|
||||
l := &NetLayout{Name: "gpu", Interfaces: []LayoutIface{
|
||||
{Type: "physical", Select: "boot", Links: []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}},
|
||||
{Type: "bond", Name: "bond0", Members: []string{"fastest:2"}, Links: []LayoutLink{{Mode: "link_up"}}},
|
||||
{Type: "vlan", Parent: "bond0", VID: 200, Links: []LayoutLink{{Subnet: "10.20.0.0/24", Mode: "auto"}}},
|
||||
{Type: "physical", Select: "ib:0", Links: []LayoutLink{{Subnet: "10.30.0.0/24", Mode: "auto"}}},
|
||||
}}
|
||||
p := resolvePlan(l, nics, &nics[0], []Subnet{labSubnet(), storage, ib}, nil, allocFrom("10.0.0.21", "10.20.0.5", "10.30.0.5"))
|
||||
if len(p.Warnings) > 0 {
|
||||
t.Fatalf("warnings: %v", p.Warnings)
|
||||
}
|
||||
if got := p.Ifaces[1].Members; strings.Join(got, ",") != "ens1f0,ens1f1" {
|
||||
t.Errorf("bond members %v (the boot NIC must not be taken)", got)
|
||||
}
|
||||
if p.Ifaces[2].Name != "bond0.200" || p.Ifaces[2].MTU != 9000 {
|
||||
t.Errorf("vlan iface %+v", p.Ifaces[2])
|
||||
}
|
||||
if mgmt, ibip := p.primaryIPs(); mgmt != "10.0.0.21" || ibip != "10.30.0.5" {
|
||||
t.Errorf("primary %s ib %s", mgmt, ibip)
|
||||
}
|
||||
b, _ := json.Marshal(networkConfig(p))
|
||||
s := string(b)
|
||||
for _, want := range []string{`"bonds":{"bond0":`, `"interfaces":["ens1f0","ens1f1"]`, `"mode":"802.3ad"`, `"vlans":{"bond0.200":`,
|
||||
`"link":"bond0"`, `"id":200`, `"10.20.0.5/24"`, `"mtu":9000`, `"ibp1s0":{`, `"addresses":["10.30.0.5/24"]`} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Errorf("network-config lacks %s:\n%s", want, s)
|
||||
}
|
||||
}
|
||||
if strings.Count(s, "0.0.0.0/0") != 1 {
|
||||
t.Errorf("want exactly one default route:\n%s", s)
|
||||
}
|
||||
rules := linkRules(p)
|
||||
for _, want := range []string{`ATTR{address}=="aa:00:00:00:00:01", NAME="eno1"`, `ATTR{address}=="aa:00:00:00:00:02", NAME="ens1f0"`, `ATTR{address}=="aa:00:00:00:00:03", NAME="ens1f1"`} {
|
||||
if !strings.Contains(rules, want) {
|
||||
t.Errorf("udev rules lack %s:\n%s", want, rules)
|
||||
}
|
||||
}
|
||||
if strings.Contains(rules, "ibp1s0") {
|
||||
t.Errorf("InfiniBand ports keep their names:\n%s", rules)
|
||||
}
|
||||
}
|
||||
|
||||
// NICs of one speed (VMs, a box of identical 25G ports): the bond must not count the boot NIC.
|
||||
func TestBondSkipsUsedNICs(t *testing.T) {
|
||||
nics := []invNIC{
|
||||
{Name: "ens18", MAC: "bc:00:00:00:00:01", Link: true, IPv4: []string{"10.0.0.150/24"}},
|
||||
{Name: "ens19", MAC: "bc:00:00:00:00:02", Link: true},
|
||||
{Name: "ens20", MAC: "bc:00:00:00:00:03", Link: true},
|
||||
}
|
||||
l := &NetLayout{Interfaces: []LayoutIface{
|
||||
{Type: "physical", Select: "boot", Links: []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}},
|
||||
{Type: "bond", Name: "bond0", Members: []string{"fastest:2"}, BondMode: "active-backup", Links: []LayoutLink{{Mode: "link_up"}}},
|
||||
}}
|
||||
p := resolvePlan(l, nics, &nics[0], []Subnet{labSubnet()}, nil, allocFrom("10.0.0.21"))
|
||||
if got := strings.Join(p.Ifaces[1].Members, ","); got != "ens19,ens20" {
|
||||
t.Errorf("bond members %q, want ens19,ens20", got)
|
||||
}
|
||||
if lk := p.Ifaces[1].Links[0]; lk.Subnet != "" || lk.Mode != "link_up" {
|
||||
t.Errorf("a link_up link without a subnet stays without one: %+v", lk)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDnsmasqConfig(t *testing.T) {
|
||||
rc := RackController{Name: "r1", TFTPRoot: "/srv/tftp", Interfaces: []RackIface{{Name: "eth0", Addrs: []string{"192.168.1.5/24"}}, {Name: "eth1", Addrs: []string{"10.0.0.1/24"}}}}
|
||||
conf, ok := dnsmasqConfig(rc, []Subnet{labSubnet()}, "/var/lib/rollcall/dnsmasq")
|
||||
if !ok {
|
||||
t.Fatal("r1 serves 10.0.0.0/24")
|
||||
}
|
||||
for _, want := range []string{"interface=eth1\n", "dhcp-range=set:s0,10.0.0.100,10.0.0.200,255.255.255.0,12h", "dhcp-option=tag:s0,option:router,10.0.0.1",
|
||||
"dhcp-option=tag:s0,option:dns-server,10.0.0.1", "domain=lab.local,10.0.0.0/24", "dhcp-authoritative", "tftp-root=/srv/tftp"} {
|
||||
if !strings.Contains(conf, want) {
|
||||
t.Errorf("dnsmasq conf lacks %q:\n%s", want, conf)
|
||||
}
|
||||
}
|
||||
if strings.Contains(conf, "interface=eth0") {
|
||||
t.Errorf("must not serve the management interface:\n%s", conf)
|
||||
}
|
||||
// two racks: each takes half of the pool, nobody is authoritative
|
||||
sn := labSubnet()
|
||||
sn.SecondaryRack = "r2"
|
||||
c1, _ := dnsmasqConfig(rc, []Subnet{sn}, "/d")
|
||||
c2, _ := dnsmasqConfig(RackController{Name: "r2", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"10.0.0.2/24"}}}}, []Subnet{sn}, "/d")
|
||||
if !strings.Contains(c1, "10.0.0.100,10.0.0.150,") || !strings.Contains(c2, "10.0.0.151,10.0.0.200,") || strings.Contains(c1, "dhcp-authoritative") {
|
||||
t.Errorf("split pool:\n%s\n%s", c1, c2)
|
||||
}
|
||||
if _, ok := dnsmasqConfig(RackController{Name: "other"}, []Subnet{labSubnet()}, "/d"); ok {
|
||||
t.Errorf("a rack that serves nothing must not get a config")
|
||||
}
|
||||
}
|
||||
|
||||
func v6Subnet() Subnet {
|
||||
return Subnet{CIDR: "fd00:213:77::/64", Gateway: "fd00:213:77::1", DNS: []string{"fd00:213:77::1"}, Domain: "lab.local", Fabric: "f0", VLANID: 1,
|
||||
DHCP: true, PrimaryRack: "r1", Purpose: "provisioning",
|
||||
Ranges: []IPRange{{Kind: "static", Start: "fd00:213:77::50", End: "fd00:213:77::99"}, {Kind: "dynamic", Start: "fd00:213:77::100", End: "fd00:213:77::1ff"}}}
|
||||
}
|
||||
|
||||
func TestIPv6Candidates(t *testing.T) {
|
||||
got := freeCandidates(v6Subnet(), map[string]bool{"fd00:213:77::50": true}, []string{"fd00:213:77::51/64"}, 2)
|
||||
if strings.Join(got, " ") != "fd00:213:77::52 fd00:213:77::53" {
|
||||
t.Errorf("v6 candidates %v", got)
|
||||
}
|
||||
open := Subnet{CIDR: "fd00:1::/64", Gateway: "fd00:1::1", Ranges: []IPRange{{Kind: "reserved", Start: "fd00:1::2", End: "fd00:1::ff"}}}
|
||||
if got := freeCandidates(open, nil, nil, 1); len(got) != 1 || got[0] != "fd00:1::100" {
|
||||
t.Errorf("v6 without static range %v", got)
|
||||
}
|
||||
if familyOf("fd00:1::/64") != 6 || familyOf("10.0.0.0/24") != 4 {
|
||||
t.Error("familyOf")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDualStackDefaultPlan(t *testing.T) {
|
||||
v4 := labSubnet()
|
||||
v4.VLANID = 1
|
||||
nics := []invNIC{{Name: "eno1", MAC: "aa:bb:cc:00:00:01", Link: true, IPv4: []string{"10.0.0.150/24"}}}
|
||||
p := resolvePlan(nil, nics, &nics[0], []Subnet{v4, v6Subnet()}, nil, allocFrom("10.0.0.21", "fd00:213:77::50"))
|
||||
if len(p.Ifaces) != 1 || len(p.Ifaces[0].Links) != 2 {
|
||||
t.Fatalf("plan %+v", p)
|
||||
}
|
||||
b, _ := json.Marshal(networkConfig(p))
|
||||
s := string(b)
|
||||
for _, want := range []string{`"10.0.0.21/24"`, `"fd00:213:77::50/64"`, `{"to":"0.0.0.0/0","via":"10.0.0.1"}`, `{"to":"::/0","via":"fd00:213:77::1"}`,
|
||||
`"addresses":["10.0.0.1","fd00:213:77::1"]`} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Errorf("network-config lacks %s:\n%s", want, s)
|
||||
}
|
||||
}
|
||||
if mgmt, _ := p.primaryIPs(); mgmt != "10.0.0.21" {
|
||||
t.Errorf("mgmt prefers IPv4, got %s", mgmt)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDnsmasqIPv6(t *testing.T) {
|
||||
rc := RackController{Name: "r1", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"10.0.0.1/24", "fd00:213:77::2/64"}}}}
|
||||
conf, _ := dnsmasqConfig(rc, []Subnet{labSubnet(), v6Subnet()}, "/d")
|
||||
for _, want := range []string{"enable-ra", "ra-param=eth1,10,0", "dhcp-range=set:s1,fd00:213:77::100,fd00:213:77::1ff,64,12h",
|
||||
"dhcp-option=tag:s1,option6:dns-server,[fd00:213:77::1]", "dhcp-option=tag:s1,option6:domain-search,lab.local", "dhcp-option=tag:s0,option:dns-server,10.0.0.1\n"} {
|
||||
if !strings.Contains(conf, want) {
|
||||
t.Errorf("dnsmasq conf lacks %q:\n%s", want, conf)
|
||||
}
|
||||
}
|
||||
if strings.Contains(conf, "tag:s1,option:router") {
|
||||
t.Errorf("no IPv4 router option on an IPv6 subnet:\n%s", conf)
|
||||
}
|
||||
// the rack is the IPv6 gateway: it advertises itself as router (no lifetime 0)
|
||||
rc.Interfaces[0].Addrs[1] = "fd00:213:77::1/64"
|
||||
conf, _ = dnsmasqConfig(rc, []Subnet{labSubnet(), v6Subnet()}, "/d")
|
||||
if strings.Contains(conf, "ra-param=eth1,10,0") {
|
||||
t.Errorf("router lifetime 0 although the rack is the gateway:\n%s", conf)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDnsmasqUpstream(t *testing.T) {
|
||||
d := t.TempDir()
|
||||
stub, resolv := filepath.Join(d, "stub"), filepath.Join(d, "resolv")
|
||||
if got := dnsmasqUpstream(stub, resolv, nil); got != "" {
|
||||
t.Errorf("no files: %q", got)
|
||||
}
|
||||
_ = os.WriteFile(resolv, []byte("# x\nnameserver 127.0.0.1\nnameserver 10.213.77.1\nnameserver 10.0.0.1\nnameserver 1.1.1.1\nsearch lan\n"), 0o644)
|
||||
got := dnsmasqUpstream(stub, resolv, []string{"10.213.77.1/24"})
|
||||
if !strings.Contains(got, "no-resolv\nserver=10.0.0.1\nserver=1.1.1.1\n") || strings.Contains(got, "127.0.0.1") || strings.Contains(got, "10.213.77.1") {
|
||||
t.Errorf("resolv.conf upstream:\n%s", got)
|
||||
}
|
||||
_ = os.WriteFile(stub, []byte("nameserver 127.0.0.53\n"), 0o644)
|
||||
if got := dnsmasqUpstream(stub, resolv, nil); !strings.Contains(got, "server=127.0.0.53") {
|
||||
t.Errorf("resolved stub: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSecondarySubnetPolicyRouting(t *testing.T) {
|
||||
nics := []invNIC{
|
||||
{Name: "eno1", MAC: "aa:00:00:00:00:01", Link: true, IPv4: []string{"10.0.0.150/24"}},
|
||||
{Name: "eno2", MAC: "aa:00:00:00:00:02", Link: true},
|
||||
}
|
||||
storage := Subnet{CIDR: "10.20.0.0/24", Gateway: "10.20.0.1", Fabric: "f0", VID: 200, Purpose: "storage"}
|
||||
l := &NetLayout{Name: "two", Interfaces: []LayoutIface{
|
||||
{Type: "physical", Select: "boot", Links: []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}},
|
||||
{Type: "physical", Select: "name:eno2", Links: []LayoutLink{{Subnet: "10.20.0.0/24", Mode: "auto"}}},
|
||||
}}
|
||||
p := resolvePlan(l, nics, &nics[0], []Subnet{labSubnet(), storage}, nil, allocFrom("10.0.0.21", "10.20.0.5"))
|
||||
b, _ := json.Marshal(networkConfig(p))
|
||||
s := string(b)
|
||||
for _, want := range []string{
|
||||
`{"to":"0.0.0.0/0","via":"10.0.0.1"}`, // main table: the default link only
|
||||
`{"scope":"link","table":100,"to":"10.20.0.0/24"}`,
|
||||
`{"table":100,"to":"0.0.0.0/0","via":"10.20.0.1"}`,
|
||||
`"routing-policy":[{"from":"10.20.0.5/32","table":100}]`,
|
||||
} {
|
||||
if !strings.Contains(s, want) {
|
||||
t.Errorf("network-config lacks %s:\n%s", want, s)
|
||||
}
|
||||
}
|
||||
// a secondary subnet without a gateway stays on-link: no table, no rule
|
||||
storage.Gateway = ""
|
||||
p = resolvePlan(l, nics, &nics[0], []Subnet{labSubnet(), storage}, nil, allocFrom("10.0.0.21", "10.20.0.5"))
|
||||
b, _ = json.Marshal(networkConfig(p))
|
||||
if strings.Contains(string(b), "routing-policy") || strings.Contains(string(b), `"table"`) {
|
||||
t.Errorf("gateway-less subnet got policy routing:\n%s", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStaleJobs(t *testing.T) {
|
||||
now := time.Now()
|
||||
active := map[string]*Job{
|
||||
"m1": {ID: "old", State: "pending", Created: now.Add(-45 * time.Minute)},
|
||||
"m2": {ID: "new", State: "pending", Created: now.Add(-5 * time.Minute)},
|
||||
"m3": {ID: "running", State: "running", Created: now.Add(-60 * time.Minute)},
|
||||
}
|
||||
got := staleJobs(active, now.Add(-jobStartTimeout))
|
||||
if len(got) != 1 || got[0].ID != "old" {
|
||||
t.Errorf("stale jobs: %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -146,7 +146,7 @@ func osCatalog(dir, sourcesFile string) ([]OSImage, error) {
|
||||
}
|
||||
if ents, err := os.ReadDir(dir); err == nil {
|
||||
for _, e := range ents {
|
||||
if e.IsDir() {
|
||||
if e.IsDir() && !strings.HasPrefix(e.Name(), ".") {
|
||||
names[e.Name()] = true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,535 @@
|
||||
package main
|
||||
|
||||
// Power drivers. A machine's BMC record names a driver; the command runs on
|
||||
// the rack controller that reaches the BMC network (the embedded one runs it
|
||||
// in-process). Every driver answers the same operations:
|
||||
//
|
||||
// status on off soft cycle reset bootdev(pxe|disk|bios|cdrom|none) probe
|
||||
//
|
||||
// and reports power state the way ipmitool does ("Chassis Power is on"), so
|
||||
// callers do not care which one ran.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type powerDriver struct {
|
||||
Name string `json:"name"`
|
||||
Description string `json:"description"`
|
||||
Family string `json:"family"` // ipmi | redfish | api | none
|
||||
DefaultPort int `json:"default_port"`
|
||||
}
|
||||
|
||||
var powerDrivers = []powerDriver{
|
||||
{"ipmi", "IPMI 2.0 over LAN (ipmitool lanplus); any vendor", "ipmi", 623},
|
||||
{"redfish", "DMTF Redfish, any vendor", "redfish", 443},
|
||||
{"idrac", "Dell iDRAC 8/9 (Redfish)", "redfish", 443},
|
||||
{"ilo", "HPE iLO 4 (2.30+), 5, 6 (Redfish)", "redfish", 443},
|
||||
{"supermicro", "Supermicro X11 and later (Redfish)", "redfish", 443},
|
||||
{"xcc", "Lenovo XClarity Controller (Redfish)", "redfish", 443},
|
||||
{"openbmc", "OpenBMC (Redfish)", "redfish", 443},
|
||||
{"proxmox", "Proxmox VE virtual machine (API token: user = token id, password = secret; extra node, vmid)", "api", 8006},
|
||||
{"manual", "no power control: someone presses the button", "none", 0},
|
||||
}
|
||||
|
||||
func driverByName(n string) (powerDriver, bool) {
|
||||
if n == "" {
|
||||
n = "ipmi"
|
||||
}
|
||||
for _, d := range powerDrivers {
|
||||
if d.Name == n {
|
||||
return d, true
|
||||
}
|
||||
}
|
||||
return powerDriver{}, false
|
||||
}
|
||||
|
||||
var powerOps = map[string]bool{"status": true, "on": true, "off": true, "soft": true, "cycle": true, "reset": true, "bootdev": true, "probe": true}
|
||||
|
||||
var errManualPower = errors.New("this machine has manual power control: do it by hand")
|
||||
|
||||
// powerRun executes one operation with a driver.
|
||||
func powerRun(ctx context.Context, ipmitool string, c *bmcCreds, op, arg string) (string, error) {
|
||||
if !powerOps[op] {
|
||||
return "", fmt.Errorf("unknown power operation %q", op)
|
||||
}
|
||||
if op == "bootdev" && !bootDevices[arg] {
|
||||
return "", fmt.Errorf("boot device must be one of pxe, disk, cdrom, bios, none")
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(ctx, 90*time.Second)
|
||||
defer cancel()
|
||||
d, ok := driverByName(c.Driver)
|
||||
if !ok {
|
||||
return "", fmt.Errorf("unknown power driver %q", c.Driver)
|
||||
}
|
||||
switch d.Family {
|
||||
case "ipmi":
|
||||
return ipmiOp(ctx, ipmitool, c, op, arg)
|
||||
case "redfish":
|
||||
if op == "probe" {
|
||||
return redfishProbe(ctx, c)
|
||||
}
|
||||
return redfishOp(ctx, c, op, arg)
|
||||
case "api":
|
||||
return proxmoxOp(ctx, c, op, arg)
|
||||
default:
|
||||
if op == "status" {
|
||||
return "Chassis Power is unknown (manual power control)", nil
|
||||
}
|
||||
return "", errManualPower
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- IPMI
|
||||
|
||||
func ipmiOp(ctx context.Context, tool string, c *bmcCreds, op, arg string) (string, error) {
|
||||
var args []string
|
||||
switch op {
|
||||
case "bootdev":
|
||||
args = []string{"chassis", "bootdev", arg}
|
||||
case "probe":
|
||||
args = []string{"mc", "info"}
|
||||
default:
|
||||
args = []string{"chassis", "power", op}
|
||||
}
|
||||
return ipmiExec(ctx, tool, c, args...)
|
||||
}
|
||||
|
||||
func ipmiExec(ctx context.Context, tool string, c *bmcCreds, args ...string) (string, error) {
|
||||
if tool == "" {
|
||||
tool = "ipmitool"
|
||||
}
|
||||
iface := c.Extra["interface"]
|
||||
if iface == "" {
|
||||
iface = "lanplus"
|
||||
}
|
||||
port := c.Port
|
||||
if port == 0 || port == 443 {
|
||||
port = 623
|
||||
}
|
||||
full := []string{"-I", iface, "-H", c.Address, "-p", strconv.Itoa(port), "-U", c.User, "-E"}
|
||||
if v := c.Extra["cipher"]; v != "" { // 17 on newer BMCs that dropped suite 3
|
||||
full = append(full, "-C", v)
|
||||
}
|
||||
if v := c.Extra["privilege"]; v != "" {
|
||||
full = append(full, "-L", v)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, tool, append(full, args...)...)
|
||||
cmd.Env = []string{"IPMI_PASSWORD=" + c.Password, "PATH=/usr/sbin:/usr/bin:/sbin:/bin"}
|
||||
var out bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &out, &out
|
||||
err := cmd.Run()
|
||||
return strings.TrimSpace(out.String()), err
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Redfish
|
||||
|
||||
type rfClient struct {
|
||||
base, user, pass string
|
||||
hc *http.Client
|
||||
etag string
|
||||
}
|
||||
|
||||
func bmcHTTPClient(c *bmcCreds) *http.Client {
|
||||
// BMCs ship self-signed certificates; checking them is opt-in (extra verify_tls=true)
|
||||
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Extra["verify_tls"] != "true"}, //nolint:gosec
|
||||
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext}
|
||||
return &http.Client{Timeout: 45 * time.Second, Transport: tr}
|
||||
}
|
||||
|
||||
// bmcBase builds https://host[:port] from the record; IPMI's port is not Redfish's.
|
||||
func bmcBase(c *bmcCreds, scheme string, defPort int) string {
|
||||
a := strings.TrimRight(c.Address, "/")
|
||||
if strings.Contains(a, "://") {
|
||||
return a
|
||||
}
|
||||
port := c.Port
|
||||
if port == 623 || port == 0 {
|
||||
port = defPort
|
||||
}
|
||||
host := a
|
||||
if strings.Contains(host, ":") && !strings.HasPrefix(host, "[") {
|
||||
host = "[" + host + "]"
|
||||
}
|
||||
if (scheme == "https" && port == 443) || (scheme == "http" && port == 80) {
|
||||
return scheme + "://" + host
|
||||
}
|
||||
return fmt.Sprintf("%s://%s:%d", scheme, host, port)
|
||||
}
|
||||
|
||||
func newRF(c *bmcCreds) *rfClient {
|
||||
return &rfClient{base: bmcBase(c, "https", 443), user: c.User, pass: c.Password, hc: bmcHTTPClient(c)}
|
||||
}
|
||||
|
||||
func (r *rfClient) do(ctx context.Context, method, path string, body, out any) (int, error) {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, r.base+path, rd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if r.user != "" {
|
||||
req.SetBasicAuth(r.user, r.pass)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if method == "PATCH" && r.etag != "" {
|
||||
req.Header.Set("If-Match", r.etag) // some services (Supermicro, newer iDRAC) insist
|
||||
}
|
||||
resp, err := r.hc.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if method == "GET" {
|
||||
if et := resp.Header.Get("ETag"); et != "" {
|
||||
r.etag = et
|
||||
}
|
||||
}
|
||||
if resp.StatusCode >= 300 {
|
||||
return resp.StatusCode, fmt.Errorf("%s %s: %s %s", method, path, resp.Status, redfishMessage(data))
|
||||
}
|
||||
if out != nil && len(data) > 0 {
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
return resp.StatusCode, fmt.Errorf("%s %s: %w", method, path, err)
|
||||
}
|
||||
}
|
||||
return resp.StatusCode, nil
|
||||
}
|
||||
|
||||
// redfishMessage pulls the human message out of a Redfish error body.
|
||||
func redfishMessage(data []byte) string {
|
||||
var e struct {
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
Ext []struct {
|
||||
Message string `json:"Message"`
|
||||
} `json:"@Message.ExtendedInfo"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(data, &e) == nil {
|
||||
if len(e.Error.Ext) > 0 && e.Error.Ext[0].Message != "" {
|
||||
return e.Error.Ext[0].Message
|
||||
}
|
||||
if e.Error.Message != "" {
|
||||
return e.Error.Message
|
||||
}
|
||||
}
|
||||
return oneLine(string(data))
|
||||
}
|
||||
|
||||
// redfishSystemPath: extra system=…, else the vendor's usual one if it exists, else the first member.
|
||||
func (r *rfClient) system(ctx context.Context, c *bmcCreds) (string, obj, error) {
|
||||
var cands []string
|
||||
if s := c.Extra["system"]; s != "" {
|
||||
cands = append(cands, s)
|
||||
}
|
||||
switch c.Driver {
|
||||
case "idrac":
|
||||
cands = append(cands, "/redfish/v1/Systems/System.Embedded.1")
|
||||
case "ilo", "xcc", "supermicro":
|
||||
cands = append(cands, "/redfish/v1/Systems/1")
|
||||
}
|
||||
for _, p := range cands {
|
||||
var sys obj
|
||||
if _, err := r.do(ctx, "GET", p, nil, &sys); err == nil {
|
||||
return p, sys, nil
|
||||
} else if c.Extra["system"] != "" {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
var coll struct {
|
||||
Members []struct {
|
||||
ID string `json:"@odata.id"`
|
||||
} `json:"Members"`
|
||||
}
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/Systems", nil, &coll); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if len(coll.Members) == 0 {
|
||||
return "", nil, errors.New("the BMC lists no systems")
|
||||
}
|
||||
p := coll.Members[0].ID
|
||||
var sys obj
|
||||
_, err := r.do(ctx, "GET", p, nil, &sys)
|
||||
return p, sys, err
|
||||
}
|
||||
|
||||
// resetTypes are the ResetType values the system accepts.
|
||||
func (r *rfClient) resetTypes(ctx context.Context, sys obj) []string {
|
||||
act, _ := dig(sys, "Actions", "#ComputerSystem.Reset").(obj)
|
||||
var out []string
|
||||
for _, v := range toList(act["ResetType@Redfish.AllowableValues"]) {
|
||||
out = append(out, str(v))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if info := str(act["@Redfish.ActionInfo"]); info != "" {
|
||||
var ai struct {
|
||||
Parameters []struct {
|
||||
Name string `json:"Name"`
|
||||
AllowableValues []string `json:"AllowableValues"`
|
||||
} `json:"Parameters"`
|
||||
}
|
||||
if _, err := r.do(ctx, "GET", info, nil, &ai); err == nil {
|
||||
for _, p := range ai.Parameters {
|
||||
if p.Name == "ResetType" {
|
||||
out = p.AllowableValues
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pickReset(allowed []string, prefs ...string) string {
|
||||
if len(allowed) == 0 {
|
||||
return prefs[0]
|
||||
}
|
||||
for _, p := range prefs {
|
||||
if hasTag(allowed, p) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var redfishBootTargets = map[string]string{"pxe": "Pxe", "disk": "Hdd", "bios": "BiosSetup", "cdrom": "Cd", "none": "None", "safe": "None"}
|
||||
|
||||
func redfishOp(ctx context.Context, c *bmcCreds, op, arg string) (string, error) {
|
||||
r := newRF(c)
|
||||
path, sys, err := r.system(ctx, c)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
state := strings.ToLower(str(sys["PowerState"])) // On | Off | PoweringOn | PoweringOff
|
||||
on := state == "on" || state == "poweringon"
|
||||
reset := func(prefs ...string) (string, error) {
|
||||
t := pickReset(r.resetTypes(ctx, sys), prefs...)
|
||||
if t == "" {
|
||||
return "", fmt.Errorf("the BMC accepts none of %v", prefs)
|
||||
}
|
||||
target := str(dig(sys, "Actions", "#ComputerSystem.Reset", "target"))
|
||||
if target == "" {
|
||||
target = path + "/Actions/ComputerSystem.Reset"
|
||||
}
|
||||
if _, err := r.do(ctx, "POST", target, obj{"ResetType": t}, nil); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "Chassis Power Control: " + t + " (redfish)", nil
|
||||
}
|
||||
switch op {
|
||||
case "status":
|
||||
if on {
|
||||
return "Chassis Power is on", nil
|
||||
}
|
||||
return "Chassis Power is off", nil
|
||||
case "on":
|
||||
if on {
|
||||
return "Chassis Power is on already", nil
|
||||
}
|
||||
return reset("On", "PushPowerButton")
|
||||
case "off":
|
||||
if !on {
|
||||
return "Chassis Power is off already", nil
|
||||
}
|
||||
return reset("ForceOff", "PushPowerButton")
|
||||
case "soft":
|
||||
if !on {
|
||||
return "Chassis Power is off already", nil
|
||||
}
|
||||
return reset("GracefulShutdown", "PushPowerButton")
|
||||
case "cycle":
|
||||
if !on {
|
||||
return reset("On", "PushPowerButton")
|
||||
}
|
||||
return reset("PowerCycle", "ForceRestart", "GracefulRestart")
|
||||
case "reset":
|
||||
if !on {
|
||||
return reset("On", "PushPowerButton")
|
||||
}
|
||||
return reset("ForceRestart", "PowerCycle", "GracefulRestart")
|
||||
case "bootdev":
|
||||
boot := obj{"BootSourceOverrideTarget": redfishBootTargets[arg], "BootSourceOverrideEnabled": "Once"}
|
||||
if arg == "none" || arg == "safe" {
|
||||
boot = obj{"BootSourceOverrideEnabled": "Disabled"}
|
||||
}
|
||||
if _, err := r.do(ctx, "PATCH", path, obj{"Boot": boot}, nil); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "Set Boot Device to " + arg + " (redfish " + str(boot["BootSourceOverrideTarget"]) + ", once)", nil
|
||||
}
|
||||
return "", fmt.Errorf("redfish: %s is not supported", op)
|
||||
}
|
||||
|
||||
// redfishProbe asks whether the BMC speaks Redfish with these credentials and
|
||||
// suggests the driver for its vendor. The answer is JSON.
|
||||
func redfishProbe(ctx context.Context, c *bmcCreds) (string, error) {
|
||||
r := newRF(c)
|
||||
var root obj
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/", nil, &root); err != nil {
|
||||
return "", fmt.Errorf("no Redfish service: %w", err)
|
||||
}
|
||||
res := obj{"redfish": str(root["RedfishVersion"]), "vendor": str(root["Vendor"])}
|
||||
if res["vendor"] == "" {
|
||||
for k := range asObj(root["Oem"]) {
|
||||
res["vendor"] = k
|
||||
}
|
||||
}
|
||||
path, sys, err := r.system(ctx, &bmcCreds{Driver: "redfish", Extra: c.Extra, User: c.User, Password: c.Password, Address: c.Address, Port: c.Port})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Redfish %s answers, but not to these credentials: %w", res["redfish"], err)
|
||||
}
|
||||
res["system"], res["manufacturer"], res["model"], res["power"] = path, str(sys["Manufacturer"]), str(sys["Model"]), str(sys["PowerState"])
|
||||
v := strings.ToLower(str(res["vendor"]) + " " + str(res["manufacturer"]))
|
||||
switch {
|
||||
case strings.Contains(v, "dell"):
|
||||
res["driver"] = "idrac"
|
||||
case strings.Contains(v, "hpe"), strings.Contains(v, "hewlett"), strings.Contains(v, "hp "):
|
||||
res["driver"] = "ilo"
|
||||
case strings.Contains(v, "supermicro"):
|
||||
res["driver"] = "supermicro"
|
||||
case strings.Contains(v, "lenovo"):
|
||||
res["driver"] = "xcc"
|
||||
case strings.Contains(v, "openbmc"):
|
||||
res["driver"] = "openbmc"
|
||||
default:
|
||||
res["driver"] = "redfish"
|
||||
}
|
||||
b, _ := json.Marshal(res)
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func asObj(v any) obj {
|
||||
o, _ := v.(obj)
|
||||
return o
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Proxmox VE
|
||||
|
||||
func proxmoxOp(ctx context.Context, c *bmcCreds, op, arg string) (string, error) {
|
||||
node, vmid := c.Extra["node"], c.Extra["vmid"]
|
||||
if node == "" || vmid == "" {
|
||||
return "", errors.New("proxmox driver needs extra node and vmid")
|
||||
}
|
||||
base := bmcBase(c, "https", 8006) + "/api2/json/nodes/" + url.PathEscape(node) + "/qemu/" + url.PathEscape(vmid)
|
||||
hc := bmcHTTPClient(c)
|
||||
call := func(method, path string, form url.Values) (obj, error) {
|
||||
var rd io.Reader
|
||||
if form != nil {
|
||||
rd = strings.NewReader(form.Encode())
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, base+path, rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Authorization", "PVEAPIToken="+c.User+"="+c.Password)
|
||||
if form != nil {
|
||||
req.Header.Set("Content-Type", "application/x-www-form-urlencoded")
|
||||
}
|
||||
resp, err := hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("proxmox %s %s: %s %s", method, path, resp.Status, oneLine(string(data)))
|
||||
}
|
||||
var out obj
|
||||
_ = json.Unmarshal(data, &out)
|
||||
return out, nil
|
||||
}
|
||||
cur, err := call("GET", "/status/current", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
on := str(dig(cur, "data", "status")) == "running"
|
||||
act := func(what string) (string, error) {
|
||||
if _, err := call("POST", "/status/"+what, url.Values{}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "Chassis Power Control: " + what + " (proxmox)", nil
|
||||
}
|
||||
switch op {
|
||||
case "status", "probe":
|
||||
if on {
|
||||
return "Chassis Power is on", nil
|
||||
}
|
||||
return "Chassis Power is off", nil
|
||||
case "on":
|
||||
if on {
|
||||
return "Chassis Power is on already", nil
|
||||
}
|
||||
return act("start")
|
||||
case "off":
|
||||
if !on {
|
||||
return "Chassis Power is off already", nil
|
||||
}
|
||||
return act("stop")
|
||||
case "soft":
|
||||
return act("shutdown")
|
||||
case "cycle", "reset":
|
||||
if !on {
|
||||
return act("start")
|
||||
}
|
||||
return act("reset")
|
||||
case "bootdev":
|
||||
cfg, err := call("GET", "/config", nil)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
d, _ := cfg["data"].(obj)
|
||||
var disks, nets, cds []string
|
||||
keys := make([]string, 0, len(d))
|
||||
for k := range d {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
v := str(d[k])
|
||||
switch {
|
||||
case strings.HasPrefix(k, "net") && len(k) > 3 && k[3] >= '0' && k[3] <= '9':
|
||||
nets = append(nets, k)
|
||||
case strings.Contains(v, "media=cdrom"):
|
||||
cds = append(cds, k)
|
||||
case hasTag([]string{"scsi0", "virtio0", "sata0", "ide0", "nvme0"}, k):
|
||||
disks = append(disks, k)
|
||||
}
|
||||
}
|
||||
var order []string
|
||||
switch arg {
|
||||
case "pxe":
|
||||
order = append(append(append(order, nets...), disks...), cds...)
|
||||
case "cdrom":
|
||||
order = append(append(append(order, cds...), disks...), nets...)
|
||||
default:
|
||||
order = append(append(append(order, disks...), cds...), nets...)
|
||||
}
|
||||
if _, err := call("PUT", "/config", url.Values{"boot": {"order=" + strings.Join(order, ";")}}); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "Set Boot Device to " + arg + " (proxmox boot order " + strings.Join(order, ";") + ")", nil
|
||||
}
|
||||
return "", fmt.Errorf("proxmox: %s is not supported", op)
|
||||
}
|
||||
@@ -0,0 +1,190 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeRedfish is a tiny Redfish service in the shape of one vendor.
|
||||
type fakeRedfish struct {
|
||||
mu sync.Mutex
|
||||
vendor string
|
||||
system string
|
||||
allowed []string
|
||||
power string
|
||||
resets []string
|
||||
boot map[string]any
|
||||
ifMatch []string
|
||||
}
|
||||
|
||||
func (f *fakeRedfish) handler(t *testing.T) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
u, p, ok := r.BasicAuth()
|
||||
if r.URL.Path != "/redfish/v1/" && (!ok || u != "admin" || p != "secret") {
|
||||
w.WriteHeader(401)
|
||||
_, _ = w.Write([]byte(`{"error":{"message":"bad credentials"}}`))
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/redfish/v1/":
|
||||
_ = json.NewEncoder(w).Encode(obj{"RedfishVersion": "1.11.0", "Vendor": f.vendor, "Systems": obj{"@odata.id": "/redfish/v1/Systems"}})
|
||||
case r.URL.Path == "/redfish/v1/Systems":
|
||||
_ = json.NewEncoder(w).Encode(obj{"Members": []obj{{"@odata.id": f.system}}})
|
||||
case r.URL.Path == f.system && r.Method == "GET":
|
||||
w.Header().Set("ETag", `W/"42"`)
|
||||
_ = json.NewEncoder(w).Encode(obj{"PowerState": f.power, "Manufacturer": f.vendor, "Model": "Test 1",
|
||||
"Actions": obj{"#ComputerSystem.Reset": obj{"target": f.system + "/Actions/ComputerSystem.Reset", "ResetType@Redfish.AllowableValues": f.allowed}}})
|
||||
case r.URL.Path == f.system && r.Method == "PATCH":
|
||||
f.ifMatch = append(f.ifMatch, r.Header.Get("If-Match"))
|
||||
var body obj
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
f.boot, _ = body["Boot"].(map[string]any)
|
||||
w.WriteHeader(204)
|
||||
case r.URL.Path == f.system+"/Actions/ComputerSystem.Reset":
|
||||
var body struct{ ResetType string }
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if !hasTag(f.allowed, body.ResetType) {
|
||||
w.WriteHeader(400)
|
||||
_, _ = w.Write([]byte(`{"error":{"@Message.ExtendedInfo":[{"Message":"ResetType not supported"}]}}`))
|
||||
return
|
||||
}
|
||||
f.resets = append(f.resets, body.ResetType)
|
||||
switch body.ResetType {
|
||||
case "On":
|
||||
f.power = "On"
|
||||
case "ForceOff", "GracefulShutdown":
|
||||
f.power = "Off"
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func creds(addr, driver string) *bmcCreds {
|
||||
u, _ := url.Parse(addr)
|
||||
return &bmcCreds{Driver: driver, Address: "https://" + u.Host, User: "admin", Password: "secret", Extra: map[string]string{}}
|
||||
}
|
||||
|
||||
func TestRedfishIDRAC(t *testing.T) {
|
||||
f := &fakeRedfish{vendor: "Dell", system: "/redfish/v1/Systems/System.Embedded.1", power: "Off",
|
||||
allowed: []string{"On", "ForceOff", "ForceRestart", "GracefulRestart", "GracefulShutdown", "PushPowerButton", "Nmi", "PowerCycle"}}
|
||||
srv := httptest.NewTLSServer(f.handler(t))
|
||||
defer srv.Close()
|
||||
c := creds(srv.URL, "idrac")
|
||||
ctx := context.Background()
|
||||
if out, _ := powerRun(ctx, "", c, "status", ""); out != "Chassis Power is off" {
|
||||
t.Errorf("status %q", out)
|
||||
}
|
||||
if _, err := powerRun(ctx, "", c, "cycle", ""); err != nil { // off: cycle means on
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := powerRun(ctx, "", c, "cycle", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := powerRun(ctx, "", c, "soft", ""); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := strings.Join(f.resets, ","); got != "On,PowerCycle,GracefulShutdown" {
|
||||
t.Errorf("resets %s", got)
|
||||
}
|
||||
if _, err := powerRun(ctx, "", c, "bootdev", "pxe"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if f.boot["BootSourceOverrideTarget"] != "Pxe" || f.boot["BootSourceOverrideEnabled"] != "Once" || f.ifMatch[0] != `W/"42"` {
|
||||
t.Errorf("boot override %v if-match %v", f.boot, f.ifMatch)
|
||||
}
|
||||
out, err := powerRun(ctx, "", creds(srv.URL, "redfish"), "probe", "")
|
||||
if err != nil || !strings.Contains(out, `"driver":"idrac"`) || !strings.Contains(out, "System.Embedded.1") {
|
||||
t.Errorf("probe %s %v", out, err)
|
||||
}
|
||||
bad := creds(srv.URL, "redfish")
|
||||
bad.Password = "nope"
|
||||
if _, err := powerRun(ctx, "", bad, "probe", ""); err == nil || !strings.Contains(err.Error(), "credentials") {
|
||||
t.Errorf("probe with a wrong password must fail clearly: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// iLO has no PowerCycle and, on older firmware, no GracefulShutdown.
|
||||
func TestRedfishILOFallbacks(t *testing.T) {
|
||||
f := &fakeRedfish{vendor: "HPE", system: "/redfish/v1/Systems/1", power: "On", allowed: []string{"On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"}}
|
||||
srv := httptest.NewTLSServer(f.handler(t))
|
||||
defer srv.Close()
|
||||
c := creds(srv.URL, "ilo")
|
||||
ctx := context.Background()
|
||||
for _, op := range []string{"cycle", "soft"} {
|
||||
if _, err := powerRun(ctx, "", c, op, ""); err != nil {
|
||||
t.Fatalf("%s: %v", op, err)
|
||||
}
|
||||
}
|
||||
if got := strings.Join(f.resets, ","); got != "ForceRestart,PushPowerButton" {
|
||||
t.Errorf("resets %s", got)
|
||||
}
|
||||
out, _ := powerRun(ctx, "", creds(srv.URL, "redfish"), "probe", "")
|
||||
if !strings.Contains(out, `"driver":"ilo"`) {
|
||||
t.Errorf("probe %s", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxmoxDriver(t *testing.T) {
|
||||
var mu sync.Mutex
|
||||
status, posts, boot := "stopped", []string{}, ""
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
mu.Lock()
|
||||
defer mu.Unlock()
|
||||
if r.Header.Get("Authorization") != "PVEAPIToken=root@pam!rc=s3cret" {
|
||||
w.WriteHeader(401)
|
||||
return
|
||||
}
|
||||
base := "/api2/json/nodes/pve1/qemu/211"
|
||||
switch {
|
||||
case r.URL.Path == base+"/status/current":
|
||||
_ = json.NewEncoder(w).Encode(obj{"data": obj{"status": status}})
|
||||
return
|
||||
case r.URL.Path == base+"/config" && r.Method == "GET":
|
||||
_ = json.NewEncoder(w).Encode(obj{"data": obj{"scsi0": "zfspool:vm-211-disk-0,size=30G", "net0": "virtio=BC:24:11:8E:1E:6C", "ide2": "local:iso/x.iso,media=cdrom"}})
|
||||
return
|
||||
case r.URL.Path == base+"/config" && r.Method == "PUT":
|
||||
_ = r.ParseForm()
|
||||
boot = r.Form.Get("boot")
|
||||
case strings.HasPrefix(r.URL.Path, base+"/status/"):
|
||||
posts = append(posts, strings.TrimPrefix(r.URL.Path, base+"/status/"))
|
||||
status = "running"
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"data":null}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
u, _ := url.Parse(srv.URL)
|
||||
c := &bmcCreds{Driver: "proxmox", Address: "https://" + u.Host, User: "root@pam!rc", Password: "s3cret", Extra: map[string]string{"node": "pve1", "vmid": "211"}}
|
||||
ctx := context.Background()
|
||||
if out, _ := powerRun(ctx, "", c, "status", ""); out != "Chassis Power is off" {
|
||||
t.Errorf("status %q", out)
|
||||
}
|
||||
_, _ = powerRun(ctx, "", c, "cycle", "")
|
||||
_, _ = powerRun(ctx, "", c, "cycle", "")
|
||||
if strings.Join(posts, ",") != "start,reset" {
|
||||
t.Errorf("posts %v", posts)
|
||||
}
|
||||
if _, err := powerRun(ctx, "", c, "bootdev", "pxe"); err != nil || boot != "order=net0;scsi0;ide2" {
|
||||
t.Errorf("boot order %q %v", boot, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestManualDriver(t *testing.T) {
|
||||
c := &bmcCreds{Driver: "manual"}
|
||||
if out, err := powerRun(context.Background(), "", c, "status", ""); err != nil || !strings.Contains(out, "unknown") {
|
||||
t.Errorf("status %q %v", out, err)
|
||||
}
|
||||
if _, err := powerRun(context.Background(), "", c, "on", ""); err != errManualPower {
|
||||
t.Errorf("on: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,931 @@
|
||||
package main
|
||||
|
||||
// Region side of the region/rack split, after MAAS. The region (rollcall
|
||||
// serve) owns the database and computes everything; rack controllers sit next
|
||||
// to the machines and do what has to happen on the machines' networks: DHCP
|
||||
// and TFTP (dnsmasq), the per-MAC PXE files, BMC commands on a BMC network the
|
||||
// region may not reach, and serving boot and OS images from a local cache.
|
||||
//
|
||||
// A rack controller pulls: it long-polls its desired state (PXE files,
|
||||
// dnsmasq configuration, host files) and a queue of commands (ipmitool), so
|
||||
// only rack → region connectivity is needed. The region host itself runs an
|
||||
// embedded rack controller that applies the same state in-process, which is
|
||||
// what a single-server installation uses.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RackIface struct {
|
||||
Name string `json:"name"`
|
||||
MAC string `json:"mac"`
|
||||
Addrs []string `json:"addrs"`
|
||||
}
|
||||
|
||||
type RackController struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1
|
||||
Kind string `json:"kind"` // embedded | remote
|
||||
Version string `json:"version"`
|
||||
Arch string `json:"arch,omitempty"` // GOARCH, for self-update
|
||||
Interfaces []RackIface `json:"interfaces"`
|
||||
DHCPInterfaces []string `json:"dhcp_interfaces"`
|
||||
TFTPRoot string `json:"tftp_root"`
|
||||
LastSeen *time.Time `json:"last_seen"`
|
||||
Registered time.Time `json:"registered"`
|
||||
Online bool `json:"online"`
|
||||
Machines int `json:"machines"`
|
||||
}
|
||||
|
||||
// rackState is everything a rack controller writes to disk.
|
||||
type rackState struct {
|
||||
Gen int64 `json:"gen"`
|
||||
DefaultPXE string `json:"default_pxe"`
|
||||
PXE map[string]string `json:"pxe"` // pxelinux.cfg file name (01-aa-bb-…) → content
|
||||
DnsmasqConf string `json:"dnsmasq_conf"` // "" = rollcall does not manage dnsmasq's configuration here
|
||||
HostsDHCP string `json:"hosts_dhcp"`
|
||||
Hosts string `json:"hosts"`
|
||||
BootFiles []string `json:"boot_files"` // kernels and initrds the rack keeps in its cache for TFTP
|
||||
Update *rackUpdate `json:"update,omitempty"` // the binary the region wants this rack to run
|
||||
}
|
||||
|
||||
type rackJob struct {
|
||||
ID string `json:"id"`
|
||||
Kind string `json:"kind"` // power
|
||||
Driver string `json:"driver"`
|
||||
Address string `json:"address"`
|
||||
Port int `json:"port"`
|
||||
User string `json:"user"`
|
||||
Password string `json:"password"`
|
||||
Extra map[string]string `json:"extra"`
|
||||
Op string `json:"op"`
|
||||
Arg string `json:"arg"`
|
||||
done chan rackResult
|
||||
}
|
||||
|
||||
type rackResult struct {
|
||||
ID string `json:"id"`
|
||||
Output string `json:"output"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- store
|
||||
|
||||
func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []RackController{}
|
||||
for rows.Next() {
|
||||
var r RackController
|
||||
var ifs, dif []byte
|
||||
if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = json.Unmarshal(ifs, &r.Interfaces)
|
||||
_ = json.Unmarshal(dif, &r.DHCPInterfaces)
|
||||
out = append(out, r)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) PutRackController(ctx context.Context, r RackController) error {
|
||||
if r.Interfaces == nil {
|
||||
r.Interfaces = []RackIface{}
|
||||
}
|
||||
if r.DHCPInterfaces == nil {
|
||||
r.DHCPInterfaces = []string{}
|
||||
}
|
||||
ifs, _ := json.Marshal(r.Interfaces)
|
||||
dif, _ := json.Marshal(r.DHCPInterfaces)
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''))
|
||||
ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,'')`,
|
||||
r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) TouchRackController(ctx context.Context, name string) error {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET last_seen=now() WHERE name=$1`, name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) DeleteRackController(ctx context.Context, name string) error {
|
||||
_, err := s.pool.Exec(ctx, `DELETE FROM rack_controllers WHERE name=$1`, name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) SetRackController(ctx context.Context, id, rack string) error {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE machines SET rack_controller=nullif($2,'') WHERE id=$1`, id, rack)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4)
|
||||
ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now()`, mac, machineID, rack, content)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) DeletePXE(ctx context.Context, mac string) error {
|
||||
_, err := s.pool.Exec(ctx, `DELETE FROM pxe_files WHERE mac=$1`, mac)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT mac, content FROM pxe_files WHERE rack=$1`, rack)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := map[string]string{}
|
||||
for rows.Next() {
|
||||
var mac, content string
|
||||
if err := rows.Scan(&mac, &content); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out[pxeFileName(mac)] = content
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") }
|
||||
|
||||
// ---------------------------------------------------------------- hub: state generations and the command queue
|
||||
|
||||
type rackHub struct {
|
||||
mu sync.Mutex
|
||||
gen int64
|
||||
changed chan struct{}
|
||||
work map[string][]*rackJob
|
||||
wake map[string]chan struct{}
|
||||
flight map[string]*rackJob
|
||||
kick chan struct{} // the embedded rack applies on this
|
||||
}
|
||||
|
||||
func newRackHub() *rackHub {
|
||||
return &rackHub{gen: time.Now().UnixNano(), changed: make(chan struct{}), work: map[string][]*rackJob{},
|
||||
wake: map[string]chan struct{}{}, flight: map[string]*rackJob{}, kick: make(chan struct{}, 1)}
|
||||
}
|
||||
|
||||
func (h *rackHub) current() int64 {
|
||||
h.mu.Lock()
|
||||
defer h.mu.Unlock()
|
||||
return h.gen
|
||||
}
|
||||
|
||||
// bump says the desired state of some rack changed.
|
||||
func (h *rackHub) bump() {
|
||||
h.mu.Lock()
|
||||
h.gen++
|
||||
close(h.changed)
|
||||
h.changed = make(chan struct{})
|
||||
h.mu.Unlock()
|
||||
select {
|
||||
case h.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// waitChange blocks until the generation moves past since, or the wait is over.
|
||||
func (h *rackHub) waitChange(ctx context.Context, since int64, wait time.Duration) int64 {
|
||||
t := time.NewTimer(wait)
|
||||
defer t.Stop()
|
||||
for {
|
||||
h.mu.Lock()
|
||||
g, ch := h.gen, h.changed
|
||||
h.mu.Unlock()
|
||||
if g != since {
|
||||
return g
|
||||
}
|
||||
select {
|
||||
case <-ch:
|
||||
case <-t.C:
|
||||
return g
|
||||
case <-ctx.Done():
|
||||
return g
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string, error) {
|
||||
j.ID = randomToken(8)
|
||||
j.done = make(chan rackResult, 1)
|
||||
h.mu.Lock()
|
||||
h.work[rack] = append(h.work[rack], j)
|
||||
if w, ok := h.wake[rack]; ok {
|
||||
close(w)
|
||||
delete(h.wake, rack)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
t := time.NewTimer(90 * time.Second)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case r := <-j.done:
|
||||
if r.Error != "" {
|
||||
return r.Output, errors.New(r.Error)
|
||||
}
|
||||
return r.Output, nil
|
||||
case <-t.C:
|
||||
case <-ctx.Done():
|
||||
}
|
||||
h.mu.Lock()
|
||||
delete(h.flight, j.ID)
|
||||
q := h.work[rack][:0]
|
||||
for _, x := range h.work[rack] {
|
||||
if x != j {
|
||||
q = append(q, x)
|
||||
}
|
||||
}
|
||||
h.work[rack] = q
|
||||
h.mu.Unlock()
|
||||
return "", fmt.Errorf("rack controller %s did not answer in time", rack)
|
||||
}
|
||||
|
||||
func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*rackJob {
|
||||
deadline := time.Now().Add(wait)
|
||||
for {
|
||||
h.mu.Lock()
|
||||
if q := h.work[rack]; len(q) > 0 {
|
||||
h.work[rack] = nil
|
||||
for _, j := range q {
|
||||
h.flight[j.ID] = j
|
||||
}
|
||||
h.mu.Unlock()
|
||||
return q
|
||||
}
|
||||
w, ok := h.wake[rack]
|
||||
if !ok {
|
||||
w = make(chan struct{})
|
||||
h.wake[rack] = w
|
||||
}
|
||||
h.mu.Unlock()
|
||||
left := time.Until(deadline)
|
||||
if left <= 0 {
|
||||
return nil
|
||||
}
|
||||
t := time.NewTimer(left)
|
||||
select {
|
||||
case <-w:
|
||||
case <-t.C:
|
||||
case <-ctx.Done():
|
||||
t.Stop()
|
||||
return nil
|
||||
}
|
||||
t.Stop()
|
||||
}
|
||||
}
|
||||
|
||||
func (h *rackHub) finish(r rackResult) bool {
|
||||
h.mu.Lock()
|
||||
j, ok := h.flight[r.ID]
|
||||
delete(h.flight, r.ID)
|
||||
h.mu.Unlock()
|
||||
if ok {
|
||||
j.done <- r
|
||||
}
|
||||
return ok
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- routing machines to racks
|
||||
|
||||
func (a *App) racksSnapshot() map[string]RackController {
|
||||
a.netMu.RLock()
|
||||
defer a.netMu.RUnlock()
|
||||
out := make(map[string]RackController, len(a.racks))
|
||||
for k, v := range a.racks {
|
||||
out[k] = v
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (a *App) subnetsSnapshot() []Subnet {
|
||||
a.netMu.RLock()
|
||||
defer a.netMu.RUnlock()
|
||||
return a.subnets
|
||||
}
|
||||
|
||||
// reloadNet refreshes the cached subnets and rack controllers after a change.
|
||||
func (a *App) reloadNet(ctx context.Context) {
|
||||
if sn, err := a.store.Subnets(ctx); err == nil {
|
||||
a.netMu.Lock()
|
||||
a.subnets = sn
|
||||
a.netMu.Unlock()
|
||||
}
|
||||
a.reloadRacks(ctx)
|
||||
}
|
||||
|
||||
func (a *App) reloadRacks(ctx context.Context) {
|
||||
rs, err := a.store.RackControllers(ctx)
|
||||
if err != nil {
|
||||
log.Printf("racks: %v", err)
|
||||
return
|
||||
}
|
||||
m := map[string]RackController{}
|
||||
for _, r := range rs {
|
||||
m[r.Name] = r
|
||||
}
|
||||
a.netMu.Lock()
|
||||
a.racks = m
|
||||
a.netMu.Unlock()
|
||||
}
|
||||
|
||||
func rackOnline(r RackController, local string) bool {
|
||||
return r.Name == local || (r.LastSeen != nil && time.Since(*r.LastSeen) < 90*time.Second)
|
||||
}
|
||||
|
||||
// rackOf names the rack controller serving a machine: the one it last came
|
||||
// through, else the one serving the VLAN of its management or boot address,
|
||||
// else the embedded one.
|
||||
func (a *App) rackOf(m *Machine) string {
|
||||
racks := a.racksSnapshot()
|
||||
if m.RackController != "" {
|
||||
if _, ok := racks[m.RackController]; ok {
|
||||
return m.RackController
|
||||
}
|
||||
}
|
||||
if subnets := a.subnetsSnapshot(); len(subnets) > 0 {
|
||||
ips := []string{deref(m.MgmtIP)}
|
||||
if b := bootNIC(machineNICs(m), m.MACs); b != nil {
|
||||
ips = append(ips, b.IPv4...)
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if sn := subnetFor(subnets, ip); sn != nil && sn.PrimaryRack != "" {
|
||||
if _, ok := racks[sn.PrimaryRack]; ok {
|
||||
return sn.PrimaryRack
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return a.cfg.RackName
|
||||
}
|
||||
|
||||
func (a *App) rackURL(name string) string {
|
||||
if name == "" || name == a.cfg.RackName {
|
||||
return a.cfg.ServerURL
|
||||
}
|
||||
if r, ok := a.racksSnapshot()[name]; ok && r.URL != "" {
|
||||
return strings.TrimRight(r.URL, "/")
|
||||
}
|
||||
return a.cfg.ServerURL
|
||||
}
|
||||
|
||||
// urlFor is the base URL a machine uses for images and the node API.
|
||||
func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) }
|
||||
|
||||
// appendFor is commission_append with the server URL of the machine's rack.
|
||||
func (a *App) appendFor(base string) string {
|
||||
if base == a.cfg.ServerURL {
|
||||
return a.cfg.CommissionAppend
|
||||
}
|
||||
return strings.ReplaceAll(a.cfg.CommissionAppend, a.cfg.ServerURL, base)
|
||||
}
|
||||
|
||||
// rackAddresses are the rack controllers' own addresses, never handed to machines.
|
||||
func (a *App) rackAddresses() []string {
|
||||
var out []string
|
||||
for _, r := range a.racksSnapshot() {
|
||||
for _, i := range r.Interfaces {
|
||||
out = append(out, i.Addrs...)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- BMC through the machine's rack
|
||||
|
||||
// ---------------------------------------------------------------- desired state
|
||||
|
||||
const defaultPXETmpl = `# generated by rollcall for rack controller %s
|
||||
# unknown machines land here; known ones get a per-MAC file (01-xx-xx-…)
|
||||
DEFAULT menu.c32
|
||||
PROMPT 0
|
||||
TIMEOUT 80
|
||||
MENU TITLE rollcall · %s · unknown machine: commissioning in 8s
|
||||
|
||||
LABEL commission
|
||||
MENU LABEL Commission this machine (inventory + BMC account)
|
||||
MENU DEFAULT
|
||||
KERNEL rollcall/vmlinuz
|
||||
INITRD rollcall/initrd.img
|
||||
APPEND %s
|
||||
|
||||
LABEL commission-debug
|
||||
MENU LABEL Commission, but stay up afterwards (ssh root@<ip>)
|
||||
KERNEL rollcall/vmlinuz
|
||||
INITRD rollcall/initrd.img
|
||||
APPEND %s rollcall.nopoweroff
|
||||
|
||||
LABEL local
|
||||
MENU LABEL Boot from local disk
|
||||
LOCALBOOT 0
|
||||
`
|
||||
|
||||
func (a *App) rackState(ctx context.Context, name string) (*rackState, error) {
|
||||
gen := a.hub.current()
|
||||
st := &rackState{Gen: gen}
|
||||
if rc, ok := a.racksSnapshot()[name]; ok {
|
||||
st.Update = a.rackUpdateFor(rc)
|
||||
}
|
||||
base := a.rackURL(name)
|
||||
app := a.appendFor(base)
|
||||
st.DefaultPXE = fmt.Sprintf(defaultPXETmpl, name, name, app, app)
|
||||
var err error
|
||||
if st.PXE, err = a.store.PXEFiles(ctx, name); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
subnets, err := a.store.Subnets(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rc := a.racksSnapshot()[name]
|
||||
rc.Name = name
|
||||
dnsDir := a.cfg.DnsmasqDir
|
||||
if dnsDir == "" {
|
||||
dnsDir = "/var/lib/rollcall/dnsmasq"
|
||||
}
|
||||
if conf, ok := dnsmasqConfig(rc, subnets, dnsDir); ok {
|
||||
st.DnsmasqConf = conf
|
||||
}
|
||||
if st.HostsDHCP, st.Hosts, err = a.hostsFiles(ctx, name, subnets); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
st.BootFiles = []string{"rollcall/vmlinuz", "rollcall/initrd.img"}
|
||||
if ents, err := os.ReadDir(a.cfg.ImagesDir); err == nil {
|
||||
for _, e := range ents {
|
||||
if e.IsDir() {
|
||||
st.BootFiles = append(st.BootFiles, "images/"+e.Name()+"/vmlinuz", "images/"+e.Name()+"/initrd.img")
|
||||
}
|
||||
}
|
||||
}
|
||||
return st, nil
|
||||
}
|
||||
|
||||
// hostsFiles: static DHCP leases for the machines this rack serves, and names
|
||||
// for every machine: the addresses on the interface carrying the management
|
||||
// address get the plain name (A and AAAA), others name-<interface>, IB name-ib.
|
||||
func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) {
|
||||
ms, err := a.store.List(ctx)
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
all, _ := a.store.IPs(ctx, "", "")
|
||||
byMachine := map[string][]IPAddr{}
|
||||
for _, x := range all {
|
||||
if x.MachineID != "" {
|
||||
byMachine[x.MachineID] = append(byMachine[x.MachineID], x)
|
||||
}
|
||||
}
|
||||
var dhcp, hosts strings.Builder
|
||||
dhcp.WriteString("# generated by rollcall\n")
|
||||
hosts.WriteString("# generated by rollcall\n")
|
||||
domainOf := func(ip string) string {
|
||||
if sn := subnetFor(subnets, ip); sn != nil && sn.Domain != "" {
|
||||
return sn.Domain
|
||||
}
|
||||
return str(a.cfg.ClusterVars["domain"])
|
||||
}
|
||||
line := func(ip, name string) {
|
||||
if d := domainOf(ip); d != "" {
|
||||
fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name)
|
||||
} else {
|
||||
fmt.Fprintf(&hosts, "%s %s\n", ip, name)
|
||||
}
|
||||
}
|
||||
sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) })
|
||||
for _, m := range ms {
|
||||
if m.NodeName == nil || len(m.MACs) == 0 {
|
||||
continue
|
||||
}
|
||||
name, mgmt, ibip := *m.NodeName, deref(m.MgmtIP), deref(m.IBIP)
|
||||
mine := a.rackOf(m) == rack
|
||||
mgmtIface := ""
|
||||
for _, x := range byMachine[m.ID] {
|
||||
if x.IP == mgmt {
|
||||
mgmtIface = x.Iface
|
||||
}
|
||||
}
|
||||
if mgmt != "" {
|
||||
line(mgmt, name)
|
||||
if mine && familyOf(mgmt+"/32") == 4 {
|
||||
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], mgmt, name)
|
||||
}
|
||||
}
|
||||
for _, x := range byMachine[m.ID] {
|
||||
if x.IP == mgmt || x.IP == ibip {
|
||||
continue
|
||||
}
|
||||
v6 := strings.Contains(x.IP, ":")
|
||||
switch {
|
||||
case x.Iface == mgmtIface && mgmtIface != "":
|
||||
line(x.IP, name)
|
||||
if v6 && mine {
|
||||
fmt.Fprintf(&dhcp, "%s,[%s],%s\n", m.MACs[0], x.IP, name)
|
||||
}
|
||||
case x.Iface != "":
|
||||
line(x.IP, name+"-"+strings.NewReplacer(".", "-", ":", "-").Replace(x.Iface))
|
||||
}
|
||||
}
|
||||
if ibip != "" {
|
||||
fmt.Fprintf(&hosts, "%s %s-ib\n", ibip, name)
|
||||
}
|
||||
}
|
||||
return dhcp.String(), hosts.String(), nil
|
||||
}
|
||||
|
||||
// writeIfChanged replaces a file only when its content differs (path units watch these).
|
||||
func writeIfChanged(p, content string, mode os.FileMode) (bool, error) {
|
||||
if old, err := os.ReadFile(p); err == nil && string(old) == content {
|
||||
return false, nil
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
return false, err
|
||||
}
|
||||
if err := os.WriteFile(p+".tmp", []byte(content), mode); err != nil {
|
||||
return false, err
|
||||
}
|
||||
return true, os.Rename(p+".tmp", p)
|
||||
}
|
||||
|
||||
// applyRackState writes a rack's state: PXE files under tftp/pxelinux.cfg
|
||||
// (stale per-MAC files removed), the dnsmasq configuration and host files.
|
||||
func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err error) {
|
||||
dir := filepath.Join(tftp, "pxelinux.cfg")
|
||||
note := func(p string, ch bool, e error) {
|
||||
if e != nil && err == nil {
|
||||
err = e
|
||||
}
|
||||
if ch {
|
||||
changed = append(changed, p)
|
||||
}
|
||||
}
|
||||
if st.DefaultPXE != "" {
|
||||
ch, e := writeIfChanged(filepath.Join(dir, "default"), st.DefaultPXE, 0o644)
|
||||
note("default", ch, e)
|
||||
}
|
||||
for name, body := range st.PXE {
|
||||
ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644)
|
||||
note(name, ch, e)
|
||||
}
|
||||
if olds, e := filepath.Glob(filepath.Join(dir, "01-*")); e == nil {
|
||||
for _, p := range olds {
|
||||
if _, keep := st.PXE[filepath.Base(p)]; !keep && !strings.HasSuffix(p, ".tmp") {
|
||||
note("-"+filepath.Base(p), true, os.Remove(p))
|
||||
}
|
||||
}
|
||||
}
|
||||
if dnsDir != "" {
|
||||
ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644)
|
||||
note("hosts.dhcp", ch, e)
|
||||
ch, e = writeIfChanged(filepath.Join(dnsDir, "hosts"), st.Hosts, 0o644)
|
||||
note("hosts", ch, e)
|
||||
conf := filepath.Join(dnsDir, "conf.d", "rollcall.conf")
|
||||
if st.DnsmasqConf != "" {
|
||||
ch, e = writeIfChanged(conf, st.DnsmasqConf+dnsmasqUpstream("/run/systemd/resolve/stub-resolv.conf", "/etc/resolv.conf", localAddrs()), 0o644)
|
||||
note("dnsmasq.conf", ch, e)
|
||||
} else if _, e := os.Stat(conf); e == nil {
|
||||
note("-dnsmasq.conf", true, os.Remove(conf))
|
||||
}
|
||||
}
|
||||
return changed, err
|
||||
}
|
||||
|
||||
// localRackLoop is the embedded rack controller: it applies this host's state on every change.
|
||||
func (a *App) localRackLoop() {
|
||||
apply := func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
st, err := a.rackState(ctx, a.cfg.RackName)
|
||||
if err != nil {
|
||||
log.Printf("embedded rack: %v", err)
|
||||
return
|
||||
}
|
||||
if ch, err := applyRackState(st, a.cfg.TFTPRoot, a.cfg.DnsmasqDir); err != nil {
|
||||
log.Printf("embedded rack: %v", err)
|
||||
} else if len(ch) > 0 {
|
||||
log.Printf("embedded rack %s: wrote %s", a.cfg.RackName, strings.Join(ch, " "))
|
||||
}
|
||||
_ = a.store.TouchRackController(ctx, a.cfg.RackName)
|
||||
}
|
||||
apply()
|
||||
tick := time.NewTicker(60 * time.Second)
|
||||
for {
|
||||
select {
|
||||
case <-a.hub.kick:
|
||||
time.Sleep(150 * time.Millisecond) // let a burst of changes settle
|
||||
select {
|
||||
case <-a.hub.kick:
|
||||
default:
|
||||
}
|
||||
case <-tick.C:
|
||||
}
|
||||
apply()
|
||||
}
|
||||
}
|
||||
|
||||
func localInterfaces() []RackIface {
|
||||
var out []RackIface
|
||||
ifs, _ := net.Interfaces()
|
||||
for _, i := range ifs {
|
||||
if i.Flags&net.FlagLoopback != 0 {
|
||||
continue
|
||||
}
|
||||
ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()}
|
||||
addrs, _ := i.Addrs()
|
||||
for _, ad := range addrs {
|
||||
if ipn, ok := ad.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() {
|
||||
ri.Addrs = append(ri.Addrs, ipn.String())
|
||||
}
|
||||
}
|
||||
if len(ri.Addrs) > 0 {
|
||||
out = append(out, ri)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// startRacks registers the embedded rack controller, carries over what an
|
||||
// older version kept on disk or in config, and starts the local apply loop.
|
||||
func (a *App) startRacks(ctx context.Context) error {
|
||||
if err := a.store.PutRackController(ctx, RackController{Name: a.cfg.RackName, URL: a.cfg.ServerURL, Kind: "embedded", Version: version, Arch: runtime.GOARCH,
|
||||
Interfaces: localInterfaces(), TFTPRoot: a.cfg.TFTPRoot}); err != nil {
|
||||
return err
|
||||
}
|
||||
a.reloadNet(ctx)
|
||||
// per-MAC files written before pxe_files existed
|
||||
var n int
|
||||
_ = a.store.pool.QueryRow(ctx, `SELECT count(*) FROM pxe_files`).Scan(&n)
|
||||
if n == 0 {
|
||||
files, _ := filepath.Glob(filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-*"))
|
||||
ms, _ := a.store.List(ctx)
|
||||
for _, f := range files {
|
||||
b, err := os.ReadFile(f)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
mac := strings.ReplaceAll(strings.TrimPrefix(filepath.Base(f), "01-"), "-", ":")
|
||||
owner := ""
|
||||
for _, m := range ms {
|
||||
if hasTag(m.MACs, mac) {
|
||||
owner = m.ID
|
||||
}
|
||||
}
|
||||
_ = a.store.PutPXE(ctx, mac, owner, a.cfg.RackName, string(b))
|
||||
}
|
||||
if len(files) > 0 {
|
||||
log.Printf("imported %d existing PXE files", len(files))
|
||||
}
|
||||
}
|
||||
a.adoptIPs(ctx)
|
||||
go a.localRackLoop()
|
||||
return nil
|
||||
}
|
||||
|
||||
// adoptIPs records addresses machines already have (node pools, before a
|
||||
// subnet was modeled) so IPAM never hands them out twice.
|
||||
func (a *App) adoptIPs(ctx context.Context) {
|
||||
subnets, err := a.store.Subnets(ctx)
|
||||
if err != nil || len(subnets) == 0 {
|
||||
return
|
||||
}
|
||||
ms, _ := a.store.List(ctx)
|
||||
for _, m := range ms {
|
||||
for _, ip := range []string{deref(m.MgmtIP), deref(m.IBIP)} {
|
||||
if sn := subnetFor(subnets, ip); sn != nil {
|
||||
iface := ""
|
||||
if b := bootNIC(machineNICs(m), m.MACs); b != nil && ip == deref(m.MgmtIP) {
|
||||
iface = b.Name
|
||||
}
|
||||
_, _ = a.claimIP(ctx, IPAddr{IP: ip, Subnet: sn.CIDR, MachineID: m.ID, Iface: iface, Kind: "auto", Hostname: deref(m.NodeName)})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- PXE files through the rack
|
||||
|
||||
// writePXEBody records a machine's PXE file for its rack controller.
|
||||
func (a *App) writePXEBody(m *Machine, body string) error {
|
||||
rack := a.rackOf(m)
|
||||
for _, mac := range m.MACs {
|
||||
if err := a.store.PutPXE(context.Background(), normMAC(mac), m.ID, rack, body); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
a.hub.bump()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) removePXE(m *Machine) {
|
||||
for _, mac := range m.MACs {
|
||||
_ = a.store.DeletePXE(context.Background(), normMAC(mac))
|
||||
}
|
||||
a.hub.bump()
|
||||
}
|
||||
|
||||
// exportDnsmasq: host files and dnsmasq configuration follow from the state now.
|
||||
func (a *App) exportDnsmasq(ctx context.Context) error {
|
||||
a.hub.bump()
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- HTTP: rack controllers
|
||||
|
||||
func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if name := a.peerRack(r); name != "" {
|
||||
next(w, r, name)
|
||||
return
|
||||
}
|
||||
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
|
||||
if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" {
|
||||
fail(w, 401, errors.New("rack controller certificate (or legacy secret) required"))
|
||||
return
|
||||
}
|
||||
next(w, r, name)
|
||||
}
|
||||
}
|
||||
|
||||
// fromRack names the rack controller a node request came through (it proxies them).
|
||||
func (a *App) fromRack(r *http.Request) string {
|
||||
if name := a.peerRack(r); name != "" {
|
||||
return name
|
||||
}
|
||||
sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack")
|
||||
if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 {
|
||||
return name
|
||||
}
|
||||
return a.cfg.RackName
|
||||
}
|
||||
|
||||
func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack string) {
|
||||
var rc RackController
|
||||
if err := json.NewDecoder(r.Body).Decode(&rc); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
rc.Name, rc.Kind = rack, "remote"
|
||||
if rack == a.cfg.RackName {
|
||||
fail(w, 409, fmt.Errorf("%s is the embedded rack controller's name", rack))
|
||||
return
|
||||
}
|
||||
if rc.URL == "" {
|
||||
fail(w, 400, errors.New("url is required"))
|
||||
return
|
||||
}
|
||||
if err := a.store.PutRackController(r.Context(), rc); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.reloadRacks(r.Context())
|
||||
a.hub.bump()
|
||||
log.Printf("rack controller %s registered from %s (%s, %d interfaces)", rack, r.RemoteAddr, rc.URL, len(rc.Interfaces))
|
||||
writeJSON(w, 200, obj{"ok": true, "gen": a.hub.current()})
|
||||
}
|
||||
|
||||
func (a *App) handleRackState(w http.ResponseWriter, r *http.Request, rack string) {
|
||||
if _, ok := a.racksSnapshot()[rack]; !ok {
|
||||
fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack))
|
||||
return
|
||||
}
|
||||
since, _ := strconv.ParseInt(r.URL.Query().Get("since"), 10, 64)
|
||||
_ = a.store.TouchRackController(r.Context(), rack)
|
||||
a.hub.waitChange(r.Context(), since, 25*time.Second)
|
||||
_ = a.store.TouchRackController(r.Context(), rack)
|
||||
go a.reloadRacks(context.Background()) // keep last_seen fresh for online checks
|
||||
st, err := a.rackState(r.Context(), rack)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, st)
|
||||
}
|
||||
|
||||
func (a *App) handleRackWork(w http.ResponseWriter, r *http.Request, rack string) {
|
||||
jobs := a.hub.take(r.Context(), rack, 25*time.Second)
|
||||
if jobs == nil {
|
||||
jobs = []*rackJob{}
|
||||
}
|
||||
writeJSON(w, 200, jobs)
|
||||
}
|
||||
|
||||
func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack string) {
|
||||
var res rackResult
|
||||
if err := json.NewDecoder(r.Body).Decode(&res); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, obj{"accepted": a.hub.finish(res)})
|
||||
}
|
||||
|
||||
func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) {
|
||||
rs, err := a.store.RackControllers(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
ms, _ := a.store.List(r.Context())
|
||||
for i := range rs {
|
||||
rs[i].Online = rackOnline(rs[i], a.cfg.RackName)
|
||||
for _, m := range ms {
|
||||
if a.rackOf(m) == rs[i].Name {
|
||||
rs[i].Machines++
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, rs)
|
||||
}
|
||||
|
||||
func (a *App) handleControllerDelete(w http.ResponseWriter, r *http.Request) {
|
||||
name := r.PathValue("name")
|
||||
if name == a.cfg.RackName {
|
||||
fail(w, 409, errors.New("the embedded rack controller cannot be removed"))
|
||||
return
|
||||
}
|
||||
if err := a.store.DeleteRackController(r.Context(), name); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.reloadRacks(r.Context())
|
||||
a.hub.bump()
|
||||
writeJSON(w, 200, obj{"deleted": name})
|
||||
}
|
||||
|
||||
// handleRackDHCP shows a rack's generated dnsmasq configuration (admin).
|
||||
func (a *App) handleRackDHCP(w http.ResponseWriter, r *http.Request) {
|
||||
st, err := a.rackState(r.Context(), r.PathValue("name"))
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "text/plain; charset=utf-8")
|
||||
if st.DnsmasqConf == "" {
|
||||
fmt.Fprintf(w, "# rack controller %s serves no VLAN with dhcp on: rollcall leaves dnsmasq's configuration alone\n", r.PathValue("name"))
|
||||
}
|
||||
fmt.Fprint(w, st.DnsmasqConf, "\n# --- hosts.dhcp\n", st.HostsDHCP)
|
||||
}
|
||||
|
||||
// dnsmasqUpstream tells dnsmasq where to forward names it does not answer
|
||||
// itself. Without it Debian's dnsmasq looks for a resolvconf-made file that
|
||||
// usually does not exist and refuses every outside name, so nodes could not
|
||||
// resolve mirrors or site services. With systemd-resolved the stub is used, so
|
||||
// its per-link routing (split DNS such as ~int.example.com) keeps working;
|
||||
// otherwise the host's resolv.conf servers, never ourselves.
|
||||
func dnsmasqUpstream(stub, resolv string, own []string) string {
|
||||
if _, err := os.Stat(stub); err == nil {
|
||||
return "# upstream: systemd-resolved on this host (keeps its split DNS)\nno-resolv\nserver=127.0.0.53\n"
|
||||
}
|
||||
b, err := os.ReadFile(resolv)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
mine := map[string]bool{}
|
||||
for _, a := range own {
|
||||
mine[stripPrefix(a)] = true
|
||||
}
|
||||
var servers []string
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
f := strings.Fields(line)
|
||||
if len(f) < 2 || f[0] != "nameserver" {
|
||||
continue
|
||||
}
|
||||
ip, ok := parseAddr(f[1])
|
||||
if !ok || mine[ip.String()] || (ip.IsLoopback() && ip.String() != "127.0.0.53") {
|
||||
continue
|
||||
}
|
||||
servers = append(servers, ip.String())
|
||||
}
|
||||
if len(servers) == 0 {
|
||||
return ""
|
||||
}
|
||||
out := "# upstream: this host's resolv.conf\nno-resolv\n"
|
||||
for _, sv := range servers {
|
||||
out += "server=" + sv + "\n"
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// localAddrs: this host's interface addresses (CIDR form).
|
||||
func localAddrs() []string {
|
||||
var out []string
|
||||
addrs, _ := net.InterfaceAddrs()
|
||||
for _, a := range addrs {
|
||||
out = append(out, a.String())
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,642 @@
|
||||
package main
|
||||
|
||||
// `rollcall rack`: a rack controller next to the machines. It registers with
|
||||
// the region, long-polls its desired state and writes it (PXE files, dnsmasq
|
||||
// configuration, host files), runs BMC commands the region queues for it,
|
||||
// serves boot and OS images from a local cache filled from the region on
|
||||
// first use, and proxies the node API (commissioning, deploy jobs, overlays,
|
||||
// phone-home) to the region, marking requests with its name.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
"encoding/base64"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/user"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type RackConfig struct {
|
||||
Name string `json:"name,omitempty"` // default: hostname
|
||||
Region string `json:"region,omitempty"` // the region's URL, e.g. http://10.0.0.234
|
||||
Secret string `json:"secret,omitempty"` // the region's rack_secret
|
||||
URL string `json:"url,omitempty"` // how machines reach this rack, e.g. http://10.213.78.1
|
||||
Listen string `json:"listen,omitempty"` // default ":80"
|
||||
TFTPRoot string `json:"tftp_root,omitempty"` // default /srv/tftp
|
||||
DnsmasqDir string `json:"dnsmasq_dir,omitempty"` // default /var/lib/rollcall/dnsmasq
|
||||
CacheDir string `json:"cache_dir,omitempty"` // default /var/cache/rollcall
|
||||
IPMITool string `json:"ipmitool,omitempty"` // default ipmitool
|
||||
DHCPInterfaces []string `json:"dhcp_interfaces,omitempty"` // default: the interfaces with an address in a served subnet
|
||||
CAFile string `json:"ca_file,omitempty"` // set by rollcall rack join: the region's CA,
|
||||
CertFile string `json:"cert_file,omitempty"` // this rack controller's certificate
|
||||
KeyFile string `json:"key_file,omitempty"` // and key; with them the secret is not needed
|
||||
AutoUpdate *bool `json:"auto_update,omitempty"` // follow the region's version (default true)
|
||||
UpdateDir string `json:"update_dir,omitempty"` // default /var/lib/rollcall/update
|
||||
}
|
||||
|
||||
type rackd struct {
|
||||
cfg RackConfig
|
||||
region *url.URL
|
||||
hc *http.Client // long polls
|
||||
fast *http.Client
|
||||
big *http.Client // image downloads, no timeout
|
||||
gen int64
|
||||
tlsc *tls.Config
|
||||
certMu sync.Mutex
|
||||
upd rackUpdater
|
||||
cert *tls.Certificate
|
||||
|
||||
headMu sync.Mutex
|
||||
heads map[string]headInfo
|
||||
fetch sync.Mutex // one boot-file prefetch at a time
|
||||
}
|
||||
|
||||
type headInfo struct {
|
||||
size int64
|
||||
mod time.Time
|
||||
at time.Time
|
||||
}
|
||||
|
||||
func loadRackConfig(p string) (RackConfig, error) {
|
||||
var c RackConfig
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return c, err
|
||||
}
|
||||
if err := json.Unmarshal(b, &c); err != nil {
|
||||
return c, fmt.Errorf("%s: %w", p, err)
|
||||
}
|
||||
if c.Name == "" {
|
||||
c.Name, _ = os.Hostname()
|
||||
}
|
||||
if c.Listen == "" {
|
||||
c.Listen = ":80"
|
||||
}
|
||||
if c.TFTPRoot == "" {
|
||||
c.TFTPRoot = "/srv/tftp"
|
||||
}
|
||||
if c.DnsmasqDir == "" {
|
||||
c.DnsmasqDir = "/var/lib/rollcall/dnsmasq"
|
||||
}
|
||||
if c.CacheDir == "" {
|
||||
c.CacheDir = "/var/cache/rollcall"
|
||||
}
|
||||
if c.IPMITool == "" {
|
||||
c.IPMITool = "ipmitool"
|
||||
}
|
||||
c.Region, c.URL = strings.TrimRight(c.Region, "/"), strings.TrimRight(c.URL, "/")
|
||||
for _, k := range []struct{ v, n string }{{c.Region, "region"}, {c.URL, "url"}} {
|
||||
if k.v == "" {
|
||||
return c, fmt.Errorf("%s: %s is required", p, k.n)
|
||||
}
|
||||
}
|
||||
if c.Secret == "" && c.CertFile == "" {
|
||||
return c, fmt.Errorf("%s: not enrolled: run rollcall rack join CODE (or set the legacy secret)", p)
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// clients builds the HTTP clients; with a certificate they speak mutual TLS
|
||||
// and pick up a renewed certificate without being rebuilt.
|
||||
func (r *rackd) clients() error {
|
||||
r.tlsc = &tls.Config{MinVersion: tls.VersionTLS12}
|
||||
if r.cfg.CertFile != "" {
|
||||
cab, err := os.ReadFile(r.cfg.CAFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(cab) {
|
||||
return fmt.Errorf("%s: no certificate", r.cfg.CAFile)
|
||||
}
|
||||
pair, err := tls.LoadX509KeyPair(r.cfg.CertFile, r.cfg.KeyFile)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0])
|
||||
r.cert = &pair
|
||||
r.tlsc.RootCAs = pool
|
||||
r.tlsc.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
r.certMu.Lock()
|
||||
defer r.certMu.Unlock()
|
||||
return r.cert, nil
|
||||
}
|
||||
}
|
||||
tr := func() *http.Transport {
|
||||
return &http.Transport{TLSClientConfig: r.tlsc, Proxy: http.ProxyFromEnvironment, MaxIdleConnsPerHost: 8,
|
||||
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext}
|
||||
}
|
||||
r.hc = &http.Client{Timeout: 45 * time.Second, Transport: tr()}
|
||||
r.fast = &http.Client{Timeout: 30 * time.Second, Transport: tr()}
|
||||
r.big = &http.Client{Transport: tr()}
|
||||
return nil
|
||||
}
|
||||
|
||||
// renewLoop asks for a new certificate a month before the current one ends.
|
||||
func (r *rackd) renewLoop() {
|
||||
for {
|
||||
r.certMu.Lock()
|
||||
leaf := r.cert.Leaf
|
||||
r.certMu.Unlock()
|
||||
if leaf != nil && time.Until(leaf.NotAfter) < renewBefore {
|
||||
if err := r.renew(); err != nil {
|
||||
log.Printf("certificate renewal: %v", err)
|
||||
}
|
||||
}
|
||||
time.Sleep(12 * time.Hour)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rackd) renew() error {
|
||||
key, csr, err := newCSR("rack:" + r.cfg.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var resp struct {
|
||||
Cert string `json:"cert"`
|
||||
}
|
||||
if _, err := r.call(r.fast, "POST", "/api/v1/rack/renew", enrollRequest{CSR: csr}, &resp); err != nil {
|
||||
return err
|
||||
}
|
||||
pair, err := tls.X509KeyPair([]byte(resp.Cert), key)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0])
|
||||
if err := os.WriteFile(r.cfg.KeyFile, key, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(r.cfg.CertFile, []byte(resp.Cert), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
r.certMu.Lock()
|
||||
r.cert = &pair
|
||||
r.certMu.Unlock()
|
||||
log.Printf("renewed the rack certificate, valid until %s", pair.Leaf.NotAfter.Format("2006-01-02"))
|
||||
return nil
|
||||
}
|
||||
|
||||
func newCSR(cn string) (keyPEM []byte, csrPEM string, err error) {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
der, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{Subject: pkix.Name{CommonName: cn}}, key)
|
||||
if err != nil {
|
||||
return nil, "", err
|
||||
}
|
||||
kder, _ := x509.MarshalECPrivateKey(key)
|
||||
return pemEncode("EC PRIVATE KEY", kder), string(pemEncode("CERTIFICATE REQUEST", der)), nil
|
||||
}
|
||||
|
||||
// rackJoin enrolls this host as a rack controller with a code from
|
||||
// `rollcall controller join-code` and writes rack.json and the certificate.
|
||||
func rackJoin(args []string) error {
|
||||
if len(args) < 1 || strings.HasPrefix(args[0], "-") {
|
||||
return errors.New("usage: rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] [-config /etc/rollcall/rack.json]")
|
||||
}
|
||||
code, err := parseJoinCode(args[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fs := newFlags("rack join")
|
||||
host, _ := os.Hostname()
|
||||
name := fs.String("name", host, "rack controller name")
|
||||
u := fs.String("url", "", "how machines reach this rack controller, e.g. http://10.213.78.1 (required)")
|
||||
listen := fs.String("listen", "", "listen address (default: the url's host, port 80)")
|
||||
cfgPath := fs.String("config", "/etc/rollcall/rack.json", "configuration to write")
|
||||
dir := fs.String("dir", "/etc/rollcall/rack", "where the certificate goes")
|
||||
_ = fs.Parse(args[1:])
|
||||
if *u == "" {
|
||||
return errors.New("-url is required: the address machines on this segment reach this host at")
|
||||
}
|
||||
cfg := RackConfig{}
|
||||
if b, err := os.ReadFile(*cfgPath); err == nil {
|
||||
_ = json.Unmarshal(b, &cfg)
|
||||
}
|
||||
cfg.Name, cfg.URL, cfg.Region, cfg.Secret = *name, strings.TrimRight(*u, "/"), "https://"+code.Addr, ""
|
||||
if *listen != "" {
|
||||
cfg.Listen = *listen
|
||||
} else if cfg.Listen == "" {
|
||||
if pu, err := url.Parse(cfg.URL); err == nil {
|
||||
cfg.Listen = net.JoinHostPort(pu.Hostname(), "80")
|
||||
}
|
||||
}
|
||||
cfg.CAFile, cfg.CertFile, cfg.KeyFile = filepath.Join(*dir, "ca.pem"), filepath.Join(*dir, "rack.pem"), filepath.Join(*dir, "rack-key.pem")
|
||||
if cfg.TFTPRoot == "" {
|
||||
cfg.TFTPRoot = "/srv/tftp"
|
||||
}
|
||||
key, csr, err := newCSR("rack:" + cfg.Name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req := enrollRequest{Secret: base64.RawURLEncoding.EncodeToString(code.Secret[:]), Name: cfg.Name, CSR: csr,
|
||||
Rack: RackController{URL: cfg.URL, Version: version, Interfaces: localInterfaces(), DHCPInterfaces: cfg.DHCPInterfaces, TFTPRoot: cfg.TFTPRoot}}
|
||||
body, _ := json.Marshal(req)
|
||||
hc := &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{TLSClientConfig: pinnedTLS(code.Fingerprint)}}
|
||||
resp, err := hc.Post(cfg.Region+"/api/v1/rack/enroll", "application/json", bytes.NewReader(body))
|
||||
if err != nil {
|
||||
return fmt.Errorf("cannot reach the region at %s: %w", cfg.Region, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode != 200 {
|
||||
var e apiError
|
||||
_ = json.Unmarshal(data, &e)
|
||||
return fmt.Errorf("the region refused: %s %s", resp.Status, e.Error)
|
||||
}
|
||||
var out struct {
|
||||
Name, Cert, CA string
|
||||
}
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.Name = out.Name
|
||||
if err := os.MkdirAll(*dir, 0o750); err != nil {
|
||||
return err
|
||||
}
|
||||
for _, f := range []struct {
|
||||
p string
|
||||
b []byte
|
||||
mode os.FileMode
|
||||
}{{cfg.CAFile, []byte(out.CA), 0o644}, {cfg.CertFile, []byte(out.Cert), 0o644}, {cfg.KeyFile, key, 0o640}} {
|
||||
if err := os.WriteFile(f.p, f.b, f.mode); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
cb, _ := json.MarshalIndent(cfg, "", " ")
|
||||
if err := os.MkdirAll(filepath.Dir(*cfgPath), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(*cfgPath, append(cb, '\n'), 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
// the service runs as user rollcall: give its group the key and the configuration
|
||||
if g, err := user.LookupGroup("rollcall"); err == nil {
|
||||
if gid, err := strconv.Atoi(g.Gid); err == nil {
|
||||
for _, p := range []string{*dir, cfg.CAFile, cfg.CertFile, cfg.KeyFile, *cfgPath} {
|
||||
_ = os.Chown(p, 0, gid)
|
||||
}
|
||||
}
|
||||
}
|
||||
ca, _ := parseCertPEM([]byte(out.CA))
|
||||
fmt.Printf("joined the region at %s as rack controller %s\n CA %s\n wrote %s and %s\nnext: sh install-rack.sh (first time) or systemctl restart rollcall-rack\n",
|
||||
cfg.Region, cfg.Name, fpString(certFingerprint(ca)), *cfgPath, *dir)
|
||||
return nil
|
||||
}
|
||||
|
||||
func runRack(args []string) error {
|
||||
fs := newFlags("rack")
|
||||
cfgPath := fs.String("config", "/etc/rollcall/rack.json", "rack controller configuration")
|
||||
_ = fs.Parse(args)
|
||||
cfg, err := loadRackConfig(*cfgPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ru, err := url.Parse(cfg.Region)
|
||||
if err != nil {
|
||||
return fmt.Errorf("region: %w", err)
|
||||
}
|
||||
r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}, upd: rackUpdater{tried: map[string]time.Time{}}}
|
||||
if err := r.clients(); err != nil {
|
||||
return fmt.Errorf("tls: %w", err)
|
||||
}
|
||||
if r.cert != nil {
|
||||
go r.renewLoop()
|
||||
} else {
|
||||
log.Printf("warning: using the legacy shared secret; enroll with rollcall rack join for mutual TLS")
|
||||
}
|
||||
for _, d := range []string{filepath.Join(cfg.TFTPRoot, "pxelinux.cfg"), cfg.CacheDir, filepath.Join(cfg.DnsmasqDir, "conf.d")} {
|
||||
if err := os.MkdirAll(d, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
go r.loop()
|
||||
mux := http.NewServeMux()
|
||||
for _, p := range []string{"/rollcall/", "/images/", "/os/"} {
|
||||
mux.HandleFunc("GET "+p, r.serveCached)
|
||||
mux.HandleFunc("HEAD "+p, r.serveCached)
|
||||
}
|
||||
proxy := &httputil.ReverseProxy{
|
||||
Rewrite: func(pr *httputil.ProxyRequest) {
|
||||
pr.SetURL(ru)
|
||||
pr.SetXForwarded()
|
||||
pr.Out.Header.Del("X-Rollcall-Rack-Secret")
|
||||
pr.Out.Header.Set("X-Rollcall-Rack", cfg.Name)
|
||||
if cfg.Secret != "" {
|
||||
pr.Out.Header.Set("X-Rollcall-Rack-Secret", cfg.Secret)
|
||||
}
|
||||
},
|
||||
Transport: r.big.Transport,
|
||||
FlushInterval: 100 * time.Millisecond,
|
||||
}
|
||||
for _, p := range []string{"/api/v1/commission", "/api/v1/bmc-hint", "/api/v1/jobs/self", "/api/v1/jobs/self/progress",
|
||||
"/api/v1/nodes/self/overlay", "/api/v1/phone-home/"} {
|
||||
mux.Handle(p, proxy)
|
||||
}
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, 200, obj{"rack": cfg.Name, "gen": r.gen, "version": version})
|
||||
})
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second}
|
||||
log.Printf("rack controller %s: region %s, serving %s on %s (tftp %s, cache %s)", cfg.Name, cfg.Region, cfg.URL, cfg.Listen, cfg.TFTPRoot, cfg.CacheDir)
|
||||
return srv.ListenAndServe()
|
||||
}
|
||||
|
||||
func (r *rackd) req(ctx context.Context, method, p string, body any) (*http.Request, error) {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, r.cfg.Region+p, rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("X-Rollcall-Rack", r.cfg.Name)
|
||||
if r.cfg.Secret != "" {
|
||||
req.Header.Set("X-Rollcall-Rack-Secret", r.cfg.Secret)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
return req, nil
|
||||
}
|
||||
|
||||
func (r *rackd) call(c *http.Client, method, p string, body, out any) (int, error) {
|
||||
req, err := r.req(context.Background(), method, p, body)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
resp, err := c.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(resp.Body)
|
||||
if resp.StatusCode >= 300 {
|
||||
var e apiError
|
||||
_ = json.Unmarshal(data, &e)
|
||||
if e.Error == "" {
|
||||
e.Error = strings.TrimSpace(string(data))
|
||||
}
|
||||
return resp.StatusCode, fmt.Errorf("%s %s: %s", method, p, e.Error)
|
||||
}
|
||||
if out != nil {
|
||||
return resp.StatusCode, json.Unmarshal(data, out)
|
||||
}
|
||||
return resp.StatusCode, nil
|
||||
}
|
||||
|
||||
func (r *rackd) register() error {
|
||||
_, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version, Arch: runtime.GOARCH,
|
||||
Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil)
|
||||
return err
|
||||
}
|
||||
|
||||
func (r *rackd) loop() {
|
||||
backoff := time.Second
|
||||
for {
|
||||
if err := r.register(); err != nil {
|
||||
log.Printf("register: %v (retry in %s)", err, backoff)
|
||||
time.Sleep(backoff)
|
||||
backoff = min(backoff*2, 30*time.Second)
|
||||
continue
|
||||
}
|
||||
break
|
||||
}
|
||||
log.Printf("registered with %s as %s", r.cfg.Region, r.cfg.Name)
|
||||
go r.workLoop()
|
||||
go func() { // interfaces change; registration also refreshes last_seen
|
||||
for range time.Tick(5 * time.Minute) {
|
||||
if err := r.register(); err != nil {
|
||||
log.Printf("re-register: %v", err)
|
||||
}
|
||||
}
|
||||
}()
|
||||
for {
|
||||
var st rackState
|
||||
code, err := r.call(r.hc, "GET", "/api/v1/rack/state?since="+strconv.FormatInt(r.gen, 10), nil, &st)
|
||||
if err != nil {
|
||||
log.Printf("state: %v", err)
|
||||
if code == 409 || code == 401 {
|
||||
_ = r.register()
|
||||
}
|
||||
time.Sleep(3 * time.Second)
|
||||
continue
|
||||
}
|
||||
go r.maybeUpdate(st.Update)
|
||||
if st.Gen == r.gen {
|
||||
continue
|
||||
}
|
||||
ch, err := applyRackState(&st, r.cfg.TFTPRoot, r.cfg.DnsmasqDir)
|
||||
if err != nil {
|
||||
log.Printf("apply: %v", err)
|
||||
} else if len(ch) > 0 {
|
||||
log.Printf("state %d: wrote %s", st.Gen, strings.Join(ch, " "))
|
||||
}
|
||||
r.gen = st.Gen
|
||||
go r.prefetch(st.BootFiles)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *rackd) workLoop() {
|
||||
for {
|
||||
var jobs []rackJob
|
||||
if _, err := r.call(r.hc, "GET", "/api/v1/rack/work", nil, &jobs); err != nil {
|
||||
log.Printf("work: %v", err)
|
||||
time.Sleep(3 * time.Second)
|
||||
continue
|
||||
}
|
||||
for _, j := range jobs {
|
||||
go func(j rackJob) {
|
||||
res := rackResult{ID: j.ID}
|
||||
switch j.Kind {
|
||||
case "power":
|
||||
out, err := powerRun(context.Background(), r.cfg.IPMITool, &bmcCreds{Driver: j.Driver, Address: j.Address, Port: j.Port,
|
||||
User: j.User, Password: j.Password, Extra: j.Extra}, j.Op, j.Arg)
|
||||
res.Output = out
|
||||
if err != nil {
|
||||
res.Error = err.Error()
|
||||
if out != "" {
|
||||
res.Error = out + " (" + err.Error() + ")"
|
||||
}
|
||||
}
|
||||
default:
|
||||
res.Error = "unknown job kind " + j.Kind + " (update this rack controller)"
|
||||
}
|
||||
log.Printf("job %s %s %s %s %s: %s", j.ID, j.Kind, j.Driver, j.Op, j.Arg, oneLine(res.Output+" "+res.Error))
|
||||
if _, err := r.call(r.fast, "POST", "/api/v1/rack/result", res, nil); err != nil {
|
||||
log.Printf("result %s: %v", j.ID, err)
|
||||
}
|
||||
}(j)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- image cache
|
||||
|
||||
// head asks the region for a file's size and modification time (cached briefly).
|
||||
func (r *rackd) head(rel string) (headInfo, error) {
|
||||
r.headMu.Lock()
|
||||
if h, ok := r.heads[rel]; ok && time.Since(h.at) < 30*time.Second {
|
||||
r.headMu.Unlock()
|
||||
return h, nil
|
||||
}
|
||||
r.headMu.Unlock()
|
||||
req, _ := http.NewRequest("HEAD", r.cfg.Region+rel, nil)
|
||||
resp, err := r.fast.Do(req)
|
||||
if err != nil {
|
||||
return headInfo{}, err
|
||||
}
|
||||
resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return headInfo{}, fmt.Errorf("%s: %s", rel, resp.Status)
|
||||
}
|
||||
mod, _ := http.ParseTime(resp.Header.Get("Last-Modified"))
|
||||
h := headInfo{size: resp.ContentLength, mod: mod, at: time.Now()}
|
||||
r.headMu.Lock()
|
||||
r.heads[rel] = h
|
||||
r.headMu.Unlock()
|
||||
return h, nil
|
||||
}
|
||||
|
||||
// fresh says whether the cached copy matches the region's file. OS images live
|
||||
// under versioned paths and never change; the rest is checked by size and time.
|
||||
func (r *rackd) fresh(rel string, fi os.FileInfo) bool {
|
||||
if strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/") {
|
||||
return true
|
||||
}
|
||||
h, err := r.head(rel)
|
||||
if err != nil {
|
||||
return true // region unreachable: serve what we have
|
||||
}
|
||||
return h.size == fi.Size() && h.mod.Unix() == fi.ModTime().Unix()
|
||||
}
|
||||
|
||||
func (r *rackd) localPath(rel string) (string, bool) {
|
||||
clean := path.Clean("/" + rel)
|
||||
if clean == "/" || strings.Contains(clean, "..") {
|
||||
return "", false
|
||||
}
|
||||
return filepath.Join(r.cfg.CacheDir, filepath.FromSlash(clean)), true
|
||||
}
|
||||
|
||||
// fill downloads rel from the region into the cache, copying to w on the way when given.
|
||||
func (r *rackd) fill(rel string, w http.ResponseWriter) error {
|
||||
local, ok := r.localPath(rel)
|
||||
if !ok {
|
||||
return errors.New("bad path")
|
||||
}
|
||||
resp, err := r.big.Get(r.cfg.Region + rel) // no timeout: images are large
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
if w != nil {
|
||||
w.WriteHeader(resp.StatusCode)
|
||||
}
|
||||
return fmt.Errorf("%s: %s", rel, resp.Status)
|
||||
}
|
||||
if err := os.MkdirAll(filepath.Dir(local), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp, err := os.CreateTemp(filepath.Dir(local), ".fill-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
var dst io.Writer = tmp
|
||||
if w != nil {
|
||||
for _, h := range []string{"Content-Length", "Content-Type", "Last-Modified"} {
|
||||
if v := resp.Header.Get(h); v != "" {
|
||||
w.Header().Set(h, v)
|
||||
}
|
||||
}
|
||||
w.WriteHeader(200)
|
||||
dst = io.MultiWriter(tmp, w)
|
||||
}
|
||||
n, err := io.Copy(dst, resp.Body)
|
||||
if cerr := tmp.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if resp.ContentLength >= 0 && n != resp.ContentLength {
|
||||
return fmt.Errorf("%s: short read %d/%d", rel, n, resp.ContentLength)
|
||||
}
|
||||
if mod, err := http.ParseTime(resp.Header.Get("Last-Modified")); err == nil {
|
||||
_ = os.Chtimes(tmp.Name(), mod, mod)
|
||||
}
|
||||
_ = os.Chmod(tmp.Name(), 0o644)
|
||||
if err := os.Rename(tmp.Name(), local); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("cached %s (%s)", rel, human(float64(n)))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *rackd) serveCached(w http.ResponseWriter, req *http.Request) {
|
||||
rel := path.Clean(req.URL.Path)
|
||||
local, ok := r.localPath(rel)
|
||||
if !ok {
|
||||
http.NotFound(w, req)
|
||||
return
|
||||
}
|
||||
if fi, err := os.Stat(local); err == nil && !fi.IsDir() && r.fresh(rel, fi) {
|
||||
http.ServeFile(w, req, local)
|
||||
return
|
||||
}
|
||||
if req.Method == http.MethodHead {
|
||||
h, err := r.head(rel)
|
||||
if err != nil {
|
||||
http.NotFound(w, req)
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Length", strconv.FormatInt(h.size, 10))
|
||||
w.Header().Set("Last-Modified", h.mod.UTC().Format(http.TimeFormat))
|
||||
return
|
||||
}
|
||||
if err := r.fill(rel, w); err != nil {
|
||||
log.Printf("cache %s: %v", rel, err)
|
||||
}
|
||||
}
|
||||
|
||||
// prefetch keeps kernels and initrds cached: TFTP cannot fetch on demand.
|
||||
func (r *rackd) prefetch(files []string) {
|
||||
r.fetch.Lock()
|
||||
defer r.fetch.Unlock()
|
||||
for _, f := range files {
|
||||
rel := "/" + strings.TrimPrefix(f, "/")
|
||||
local, ok := r.localPath(rel)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
if fi, err := os.Stat(local); err == nil && r.fresh(rel, fi) {
|
||||
continue
|
||||
}
|
||||
if err := r.fill(rel, nil); err != nil {
|
||||
log.Printf("prefetch %s: %v", rel, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,213 @@
|
||||
package main
|
||||
|
||||
// Rack controllers follow the region's version. The region names the binary
|
||||
// it wants (version, architecture, SHA-256) in each rack's state and serves it
|
||||
// over the rack's mutual-TLS channel. The rack (unprivileged) downloads it into
|
||||
// its update directory, checks the checksum and that the binary reports the
|
||||
// expected version, and leaves a "ready" marker. A root path unit
|
||||
// (rollcall-rack-update.path) installs it, restarts the rack and puts the
|
||||
// previous binary back if the rack does not register again within a minute.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type rackUpdate struct {
|
||||
Version string `json:"version"`
|
||||
Arch string `json:"arch"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- region
|
||||
|
||||
type binCache struct {
|
||||
mu sync.Mutex
|
||||
key map[string]string // path -> "size/mtime"
|
||||
sums map[string]string // path -> sha256
|
||||
}
|
||||
|
||||
var rackBins = &binCache{key: map[string]string{}, sums: map[string]string{}}
|
||||
|
||||
// rackBinary: the binary a rack of this architecture should run: the region's
|
||||
// own for the same architecture, else rack_binaries/rollcall-linux-<arch>.
|
||||
func (a *App) rackBinary(arch string) (path, sum string, ok bool) {
|
||||
if arch == "" {
|
||||
return "", "", false
|
||||
}
|
||||
if arch == runtime.GOARCH && runtime.GOOS == "linux" {
|
||||
p, err := os.Executable()
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
path = p
|
||||
} else if a.cfg.RackBinaries != "" {
|
||||
path = filepath.Join(a.cfg.RackBinaries, "rollcall-linux-"+arch)
|
||||
} else {
|
||||
return "", "", false
|
||||
}
|
||||
st, err := os.Stat(path)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
k := fmt.Sprintf("%d/%d", st.Size(), st.ModTime().UnixNano())
|
||||
rackBins.mu.Lock()
|
||||
defer rackBins.mu.Unlock()
|
||||
if rackBins.key[path] == k {
|
||||
return path, rackBins.sums[path], true
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
defer f.Close()
|
||||
h := sha256.New()
|
||||
if _, err := io.Copy(h, f); err != nil {
|
||||
return "", "", false
|
||||
}
|
||||
rackBins.key[path], rackBins.sums[path] = k, hex.EncodeToString(h.Sum(nil))
|
||||
return path, rackBins.sums[path], true
|
||||
}
|
||||
|
||||
// rackUpdateFor: what the rack should run, nil when nothing to offer.
|
||||
func (a *App) rackUpdateFor(rc RackController) *rackUpdate {
|
||||
if rc.Kind != "remote" || version == "dev" || strings.HasSuffix(version, "-dirty") {
|
||||
return nil
|
||||
}
|
||||
if _, sum, ok := a.rackBinary(rc.Arch); ok {
|
||||
return &rackUpdate{Version: version, Arch: rc.Arch, SHA256: sum}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) handleRackBinary(w http.ResponseWriter, r *http.Request, rack string) {
|
||||
arch := r.URL.Query().Get("arch")
|
||||
path, sum, ok := a.rackBinary(arch)
|
||||
if !ok {
|
||||
fail(w, 404, fmt.Errorf("no rollcall binary for linux/%s here (set rack_binaries)", arch))
|
||||
return
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
defer f.Close()
|
||||
st, _ := f.Stat()
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
w.Header().Set("X-Rollcall-Version", version)
|
||||
w.Header().Set("X-Rollcall-SHA256", sum)
|
||||
w.Header().Set("Content-Length", fmt.Sprint(st.Size()))
|
||||
log.Printf("rack controller %s: sending rollcall %s (linux/%s)", rack, version, arch)
|
||||
_, _ = io.Copy(w, f)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- rack
|
||||
|
||||
type rackUpdater struct {
|
||||
mu sync.Mutex
|
||||
tried map[string]time.Time // version -> last attempt
|
||||
}
|
||||
|
||||
func (r *rackd) updateDir() string {
|
||||
if r.cfg.UpdateDir != "" {
|
||||
return r.cfg.UpdateDir
|
||||
}
|
||||
return "/var/lib/rollcall/update"
|
||||
}
|
||||
|
||||
func (r *rackd) autoUpdate() bool { return r.cfg.AutoUpdate == nil || *r.cfg.AutoUpdate }
|
||||
|
||||
// maybeUpdate fetches and stages the region's binary when it differs from ours.
|
||||
func (r *rackd) maybeUpdate(up *rackUpdate) {
|
||||
if up == nil || up.Version == "" || up.Version == version || !r.autoUpdate() || up.Arch != runtime.GOARCH {
|
||||
return
|
||||
}
|
||||
r.upd.mu.Lock()
|
||||
if t, ok := r.upd.tried[up.Version]; ok && time.Since(t) < time.Hour {
|
||||
r.upd.mu.Unlock()
|
||||
return
|
||||
}
|
||||
r.upd.tried[up.Version] = time.Now()
|
||||
r.upd.mu.Unlock()
|
||||
dir := r.updateDir()
|
||||
if b, err := os.ReadFile(filepath.Join(dir, "last-result")); err == nil &&
|
||||
strings.Contains(string(b), up.Version) && strings.Contains(string(b), "rolled back") {
|
||||
if st, err := os.Stat(filepath.Join(dir, "last-result")); err == nil && time.Since(st.ModTime()) < 6*time.Hour {
|
||||
log.Printf("update to %s was rolled back %s ago; not retrying yet", up.Version, time.Since(st.ModTime()).Round(time.Minute))
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := r.stageUpdate(up); err != nil {
|
||||
log.Printf("update to %s: %v", up.Version, err)
|
||||
return
|
||||
}
|
||||
log.Printf("update to %s staged in %s; rollcall-rack-update installs it", up.Version, dir)
|
||||
}
|
||||
|
||||
func (r *rackd) stageUpdate(up *rackUpdate) error {
|
||||
dir := r.updateDir()
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
tmp := filepath.Join(dir, "rollcall.new.tmp")
|
||||
defer os.Remove(tmp)
|
||||
req, err := r.req(context.Background(), "GET", "/api/v1/rack/binary?arch="+up.Arch, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resp, err := r.big.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return fmt.Errorf("download: %s %s", resp.Status, strings.TrimSpace(string(b)))
|
||||
}
|
||||
f, err := os.OpenFile(tmp, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0o755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h := sha256.New()
|
||||
_, err = io.Copy(io.MultiWriter(f, h), resp.Body)
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if got := hex.EncodeToString(h.Sum(nil)); got != up.SHA256 {
|
||||
return fmt.Errorf("checksum %s, region announced %s", got[:12], up.SHA256[:12])
|
||||
}
|
||||
out, err := exec.Command(tmp, "version").Output()
|
||||
if err != nil {
|
||||
return fmt.Errorf("new binary does not run: %w", err)
|
||||
}
|
||||
if f := strings.Fields(string(out)); len(f) < 2 || f[1] != up.Version {
|
||||
return fmt.Errorf("new binary says %q, expected %s", strings.TrimSpace(string(out)), up.Version)
|
||||
}
|
||||
if err := os.Rename(tmp, filepath.Join(dir, "rollcall.new")); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "rollcall.new.sha256"), []byte(up.SHA256+" rollcall.new\n"), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
ready := filepath.Join(dir, "ready")
|
||||
if err := os.WriteFile(ready+".tmp", []byte(up.Version+"\n"), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(ready+".tmp", ready)
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"runtime"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestRackStageUpdate(t *testing.T) {
|
||||
bin := []byte("#!/bin/sh\necho rollcall v9.9.9\n")
|
||||
sum := sha256.Sum256(bin)
|
||||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.URL.Path != "/api/v1/rack/binary" || r.Header.Get("X-Rollcall-Rack") != "r2" {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
_, _ = w.Write(bin)
|
||||
}))
|
||||
defer srv.Close()
|
||||
dir := t.TempDir()
|
||||
r := &rackd{cfg: RackConfig{Name: "r2", Region: srv.URL, UpdateDir: dir}, big: srv.Client(), upd: rackUpdater{tried: map[string]time.Time{}}}
|
||||
good := &rackUpdate{Version: "v9.9.9", Arch: runtime.GOARCH, SHA256: hex.EncodeToString(sum[:])}
|
||||
|
||||
if err := r.stageUpdate(&rackUpdate{Version: "v9.9.9", Arch: runtime.GOARCH, SHA256: "00" + good.SHA256[2:]}); err == nil {
|
||||
t.Fatal("checksum mismatch accepted")
|
||||
}
|
||||
if err := r.stageUpdate(&rackUpdate{Version: "v1.0.0", Arch: runtime.GOARCH, SHA256: good.SHA256}); err == nil {
|
||||
t.Fatal("version mismatch accepted")
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "ready")); err == nil {
|
||||
t.Fatal("ready marker after a rejected update")
|
||||
}
|
||||
if err := r.stageUpdate(good); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, f := range []string{"rollcall.new", "rollcall.new.sha256", "ready"} {
|
||||
if _, err := os.Stat(filepath.Join(dir, f)); err != nil {
|
||||
t.Errorf("%s missing: %v", f, err)
|
||||
}
|
||||
}
|
||||
if b, _ := os.ReadFile(filepath.Join(dir, "ready")); string(b) != "v9.9.9\n" {
|
||||
t.Errorf("ready: %q", b)
|
||||
}
|
||||
// a rolled-back version is not fetched again right away
|
||||
_ = os.Remove(filepath.Join(dir, "ready"))
|
||||
_ = os.WriteFile(filepath.Join(dir, "last-result"), []byte("2026-10-28 v9.9.9 did not register within 60s; rolled back\n"), 0o644)
|
||||
r.maybeUpdate(good)
|
||||
if _, err := os.Stat(filepath.Join(dir, "ready")); err == nil {
|
||||
t.Error("retried a rolled-back version")
|
||||
}
|
||||
}
|
||||
+228
-55
@@ -3,6 +3,8 @@ package main
|
||||
import (
|
||||
"context"
|
||||
"crypto/subtle"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"embed"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
@@ -10,11 +12,12 @@ import (
|
||||
"io"
|
||||
"io/fs"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
@@ -25,6 +28,17 @@ type App struct {
|
||||
cfg *Config
|
||||
store *Store
|
||||
sealer *Sealer
|
||||
hub *rackHub
|
||||
ca *CA
|
||||
|
||||
netMu sync.RWMutex
|
||||
racks map[string]RackController
|
||||
subnets []Subnet
|
||||
|
||||
ipam ipamProvider // nil = built-in IPAM only
|
||||
ipamSt *ipamState
|
||||
|
||||
adminCC adminCertCache
|
||||
}
|
||||
|
||||
type apiError struct {
|
||||
@@ -52,11 +66,65 @@ func tokenEqual(a, b string) bool {
|
||||
|
||||
func (a *App) isAdmin(r *http.Request) bool {
|
||||
h := r.Header.Get("Authorization")
|
||||
return strings.HasPrefix(h, "Bearer ") && tokenEqual(strings.TrimPrefix(h, "Bearer "), a.cfg.AdminToken)
|
||||
if strings.HasPrefix(h, "Bearer ") && tokenEqual(strings.TrimPrefix(h, "Bearer "), a.cfg.AdminToken) {
|
||||
return true
|
||||
}
|
||||
return a.peerAdmin(r) != "" // an admin client certificate (admincert.go)
|
||||
}
|
||||
|
||||
// adminOverHTTP applies admin_http to requests that did not come over TLS.
|
||||
func (a *App) adminOverHTTP(r *http.Request) error {
|
||||
if r.TLS != nil || a.cfg.AdminHTTP == "allow" {
|
||||
return nil
|
||||
}
|
||||
host, _, _ := net.SplitHostPort(r.RemoteAddr)
|
||||
if ip := net.ParseIP(host); a.cfg.AdminHTTP == "loopback" && ip != nil && ip.IsLoopback() {
|
||||
return nil
|
||||
}
|
||||
_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
|
||||
h := r.Host
|
||||
if hh, _, err := net.SplitHostPort(h); err == nil {
|
||||
h = hh
|
||||
}
|
||||
return fmt.Errorf("the admin API is HTTPS-only here: https://%s (trust the CA from /ca.pem)", net.JoinHostPort(strings.Trim(h, "[]"), port))
|
||||
}
|
||||
|
||||
// httpsURL is the region's HTTPS address for the host the client used.
|
||||
func (a *App) httpsURL(r *http.Request) string {
|
||||
_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
|
||||
h := r.Host
|
||||
if hh, _, err := net.SplitHostPort(h); err == nil {
|
||||
h = hh
|
||||
}
|
||||
h = strings.Trim(h, "[]")
|
||||
if port == "" || port == "443" {
|
||||
if strings.Contains(h, ":") {
|
||||
return "https://[" + h + "]"
|
||||
}
|
||||
return "https://" + h
|
||||
}
|
||||
return "https://" + net.JoinHostPort(h, port)
|
||||
}
|
||||
|
||||
// uiOverHTTPS sends browsers that reach the web UI over plain HTTP to HTTPS when
|
||||
// the admin API is not allowed over HTTP anyway (the page could not work).
|
||||
// Only the UI is redirected: nodes keep using HTTP for images and the agent.
|
||||
func (a *App) uiOverHTTPS(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if r.TLS == nil && a.cfg.AdminHTTP != "" && a.cfg.AdminHTTP != "allow" && a.cfg.TLSListen != "" && a.adminOverHTTP(r) != nil {
|
||||
http.Redirect(w, r, a.httpsURL(r)+r.URL.RequestURI(), http.StatusPermanentRedirect)
|
||||
return
|
||||
}
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
func (a *App) adminAuth(next http.HandlerFunc) http.HandlerFunc {
|
||||
return func(w http.ResponseWriter, r *http.Request) {
|
||||
if err := a.adminOverHTTP(r); err != nil {
|
||||
fail(w, 403, err)
|
||||
return
|
||||
}
|
||||
if !a.isAdmin(r) {
|
||||
w.Header().Set("WWW-Authenticate", `Bearer realm="rollcall"`)
|
||||
fail(w, http.StatusUnauthorized, errors.New("admin token required"))
|
||||
@@ -107,6 +175,9 @@ func (a *App) routes() http.Handler {
|
||||
mux.HandleFunc("GET /api/v1/machines/{id}/power", a.adminAuth(a.handlePower))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/power", a.adminAuth(a.handlePower))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/bootdev", a.adminAuth(a.handleBootdev))
|
||||
mux.HandleFunc("GET /api/v1/machines/{id}/bmc", a.adminAuth(a.handleBMC))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/bmc", a.adminAuth(a.handleBMC))
|
||||
mux.HandleFunc("GET /api/v1/power-drivers", a.adminAuth(func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, powerDrivers) }))
|
||||
mux.HandleFunc("GET /api/v1/whoami", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 200, map[string]any{"ok": true})
|
||||
}))
|
||||
@@ -129,6 +200,73 @@ func (a *App) routes() http.Handler {
|
||||
mux.HandleFunc("GET /api/v1/jobs/self", a.handleJobSelf) // job token auth inside
|
||||
mux.HandleFunc("POST /api/v1/jobs/self/progress", a.handleJobProgress) // job token auth inside
|
||||
mux.HandleFunc("POST /api/v1/phone-home/{token}", a.handlePhoneHome)
|
||||
// fleet-wide views for the UI
|
||||
mux.HandleFunc("GET /api/v1/events", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
evs, err := a.store.RecentEvents(r.Context(), 100)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, evs)
|
||||
}))
|
||||
mux.HandleFunc("GET /api/v1/jobs", a.adminAuth(func(w http.ResponseWriter, r *http.Request) {
|
||||
jobs, err := a.store.RecentJobs(r.Context(), 100)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, jobs)
|
||||
}))
|
||||
// network model and rack controllers
|
||||
mux.HandleFunc("GET /api/v1/fabrics", a.adminAuth(a.handleFabrics))
|
||||
mux.HandleFunc("POST /api/v1/fabrics", a.adminAuth(a.handleFabrics))
|
||||
mux.HandleFunc("DELETE /api/v1/fabrics/{name}", a.adminAuth(a.handleFabricDelete))
|
||||
mux.HandleFunc("GET /api/v1/vlans", a.adminAuth(a.handleVLANs))
|
||||
mux.HandleFunc("POST /api/v1/vlans", a.adminAuth(a.handleVLANs))
|
||||
mux.HandleFunc("DELETE /api/v1/vlans/{fabric}/{vid}", a.adminAuth(a.handleVLANDelete))
|
||||
mux.HandleFunc("GET /api/v1/subnets", a.adminAuth(a.handleSubnets))
|
||||
mux.HandleFunc("POST /api/v1/subnets", a.adminAuth(a.handleSubnets))
|
||||
mux.HandleFunc("DELETE /api/v1/subnets/{cidr...}", a.adminAuth(a.handleSubnetDelete))
|
||||
mux.HandleFunc("POST /api/v1/ranges", a.adminAuth(a.handleRanges))
|
||||
mux.HandleFunc("DELETE /api/v1/ranges/{id}", a.adminAuth(a.handleRangeDelete))
|
||||
mux.HandleFunc("GET /api/v1/ips", a.adminAuth(a.handleIPs))
|
||||
mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs))
|
||||
mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete))
|
||||
mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM))
|
||||
mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))
|
||||
mux.HandleFunc("POST /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))
|
||||
mux.HandleFunc("DELETE /api/v1/admin-certs/{ref}", a.adminAuth(a.handleAdminCertRevoke))
|
||||
mux.HandleFunc("POST /api/v1/ipam/sync", a.adminAuth(a.handleIPAMSync))
|
||||
mux.HandleFunc("GET /api/v1/ipam/prefixes", a.adminAuth(a.handleIPAMPrefixes))
|
||||
mux.HandleFunc("POST /api/v1/ipam/prefixes", a.adminAuth(a.handleIPAMPrefixes))
|
||||
mux.HandleFunc("GET /api/v1/layouts", a.adminAuth(a.handleLayouts))
|
||||
mux.HandleFunc("POST /api/v1/layouts", a.adminAuth(a.handleLayouts))
|
||||
mux.HandleFunc("DELETE /api/v1/layouts/{name}", a.adminAuth(a.handleLayoutDelete))
|
||||
mux.HandleFunc("GET /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/network", a.adminAuth(a.handleMachineNetwork))
|
||||
mux.HandleFunc("GET /api/v1/controllers", a.adminAuth(a.handleControllers))
|
||||
mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete))
|
||||
mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP))
|
||||
mux.HandleFunc("GET /ca.pem", a.handleCAPEM)
|
||||
mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode))
|
||||
mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only
|
||||
mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew))
|
||||
mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister))
|
||||
mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState))
|
||||
mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork))
|
||||
mux.HandleFunc("POST /api/v1/rack/result", a.rackAuth(a.handleRackResult))
|
||||
mux.HandleFunc("GET /api/v1/rack/binary", a.rackAuth(a.handleRackBinary))
|
||||
// physical location
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/location", a.adminAuth(a.handleLocation))
|
||||
mux.HandleFunc("POST /api/v1/locations/import", a.adminAuth(a.handleLocationImport))
|
||||
mux.HandleFunc("GET /api/v1/racks", a.adminAuth(a.handleRacks))
|
||||
mux.HandleFunc("POST /api/v1/racks", a.adminAuth(a.handleRacks))
|
||||
mux.HandleFunc("DELETE /api/v1/racks", a.adminAuth(a.handleRacks))
|
||||
// tags
|
||||
mux.HandleFunc("GET /api/v1/tags", a.adminAuth(a.handleTags))
|
||||
mux.HandleFunc("POST /api/v1/tags", a.adminAuth(a.handleTags))
|
||||
mux.HandleFunc("DELETE /api/v1/tags/{name}", a.adminAuth(a.handleTagDelete))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/tags", a.adminAuth(a.handleMachineTags))
|
||||
// image files for the live boot (no auth: the kernel/initrd/squashfs must be fetchable)
|
||||
imgDir := a.cfg.ImageDir
|
||||
mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir))))
|
||||
@@ -136,7 +274,7 @@ func (a *App) routes() http.Handler {
|
||||
mux.Handle("GET /os/", http.StripPrefix("/os/", http.FileServer(http.Dir(a.cfg.OSDir))))
|
||||
// embedded web UI
|
||||
sub, _ := fs.Sub(webFS, "web")
|
||||
mux.Handle("GET /", http.FileServer(http.FS(sub)))
|
||||
mux.Handle("GET /", a.uiOverHTTPS(http.FileServer(http.FS(sub))))
|
||||
return logging(mux)
|
||||
}
|
||||
|
||||
@@ -239,6 +377,12 @@ type Summary struct {
|
||||
OSVersion *string `json:"os_version"`
|
||||
DeployedAt *time.Time `json:"deployed_at"`
|
||||
Job *Job `json:"job,omitempty"` // active deployment job, if any
|
||||
Tags []string `json:"tags"` // effective (manual + auto)
|
||||
ManualTags []string `json:"manual_tags"`
|
||||
AutoTags []string `json:"auto_tags"`
|
||||
Location Location `json:"location"`
|
||||
Rack string `json:"rack_controller"`
|
||||
NetLayout string `json:"net_layout"`
|
||||
}
|
||||
|
||||
func num(v any) float64 {
|
||||
@@ -254,13 +398,19 @@ func summarize(m *Machine) Summary {
|
||||
CPU: str(dig(inv, "cpu", "Model name")), CPUs: str(dig(inv, "cpu", "CPU(s)")),
|
||||
MemoryBytes: num(dig(inv, "memory", "total_bytes")), Disks: num(dig(inv, "storage", "disk_count")),
|
||||
StorageBytes: num(dig(inv, "storage", "total_bytes")), NICs: listLen(inv["network"]),
|
||||
IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: listLen(inv["gpus"]),
|
||||
IBHCAs: listLen(dig(inv, "infiniband", "hcas")), GPUs: nodeCtx(m).GPUs, // real accelerators only
|
||||
Profile: m.Profile, NodeName: m.NodeName, MgmtIP: m.MgmtIP, IBIP: m.IBIP, BootedAt: m.BootedAt,
|
||||
OS: m.OS, OSVersion: m.OSVersion, DeployedAt: m.DeployedAt}
|
||||
OS: m.OS, OSVersion: m.OSVersion, DeployedAt: m.DeployedAt,
|
||||
Tags: effectiveTags(m), ManualTags: m.Tags, AutoTags: m.AutoTags, Location: m.Location,
|
||||
Rack: m.RackController, NetLayout: m.NetLayout}
|
||||
if m.BMC != nil {
|
||||
var b obj
|
||||
if json.Unmarshal(m.BMC, &b) == nil {
|
||||
s.BMC = &obj{"source": b["source"], "address": b["address"], "user": b["user"],
|
||||
drv := str(b["driver"])
|
||||
if drv == "" {
|
||||
drv = "ipmi"
|
||||
}
|
||||
s.BMC = &obj{"source": b["source"], "address": b["address"], "user": b["user"], "driver": drv,
|
||||
"configured": b["configured"], "verified_oob": b["verified_oob"]}
|
||||
}
|
||||
}
|
||||
@@ -292,10 +442,6 @@ LABEL local
|
||||
LOCALBOOT 0
|
||||
`
|
||||
|
||||
func (a *App) pxePath(mac string) string {
|
||||
return filepath.Join(a.cfg.TFTPRoot, "pxelinux.cfg", "01-"+strings.ReplaceAll(normMAC(mac), ":", "-"))
|
||||
}
|
||||
|
||||
func deref(p *string) string {
|
||||
if p == nil {
|
||||
return ""
|
||||
@@ -306,32 +452,10 @@ func deref(p *string) string {
|
||||
func (a *App) writePXE(m *Machine) error {
|
||||
man, prod, ser := deref(m.Manufacturer), deref(m.Product), deref(m.Serial)
|
||||
body := fmt.Sprintf(commissionedTmpl, time.Now().Format(time.RFC3339), m.ID, man, prod, ser,
|
||||
m.ID, man, prod, ser, a.cfg.CommissionAppend)
|
||||
m.ID, man, prod, ser, a.appendFor(a.urlFor(m)))
|
||||
return a.writePXEBody(m, body)
|
||||
}
|
||||
|
||||
func (a *App) writePXEBody(m *Machine, body string) error {
|
||||
for _, mac := range m.MACs {
|
||||
p := a.pxePath(mac)
|
||||
if err := os.WriteFile(p+".tmp", []byte(body), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(p+".tmp", p); err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("wrote %s", p)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) removePXE(m *Machine) {
|
||||
for _, mac := range m.MACs {
|
||||
if err := os.Remove(a.pxePath(mac)); err == nil {
|
||||
log.Printf("removed %s", a.pxePath(mac))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- handlers
|
||||
|
||||
func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -388,15 +512,35 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
m.Inventory, _ = json.Marshal(inv)
|
||||
|
||||
// a BMC an operator configured (driver_set) survives commissioning; what the agent
|
||||
// found is kept next to it for reference
|
||||
if old, err := a.store.Get(r.Context(), m.ID); err == nil && old.BMC != nil {
|
||||
var ob obj
|
||||
if json.Unmarshal(old.BMC, &ob) == nil && ob["driver_set"] == true {
|
||||
var found obj
|
||||
_ = json.Unmarshal(m.BMC, &found)
|
||||
delete(found, "password")
|
||||
ob["commissioning"] = found
|
||||
m.BMC, _ = json.Marshal(ob)
|
||||
secret = old.bmcSecret
|
||||
detail += " (operator BMC settings kept)"
|
||||
}
|
||||
}
|
||||
|
||||
isNew, err := a.store.UpsertCommissioned(r.Context(), m, secret)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail)
|
||||
_ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r))
|
||||
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
|
||||
go a.probeBMC(full)
|
||||
}
|
||||
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
|
||||
m = full // carries profile / node token when this is a re-commission of a stateless node
|
||||
}
|
||||
a.retag(r.Context(), []*Machine{m})
|
||||
if err := a.writePXEAny(r.Context(), m); err != nil {
|
||||
log.Printf("pxe write failed for %s: %v", m.ID, err)
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "pxe-write-failed", err.Error())
|
||||
@@ -431,8 +575,15 @@ func (a *App) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
active, _ := a.store.ActiveJobs(r.Context())
|
||||
wantTag := r.URL.Query().Get("tag")
|
||||
out := make([]Summary, 0, len(ms))
|
||||
for _, m := range ms {
|
||||
if wantTag != "" && !hasTag(effectiveTags(m), wantTag) {
|
||||
continue
|
||||
}
|
||||
if !locationMatch(m, r.URL.Query()) {
|
||||
continue
|
||||
}
|
||||
s := summarize(m)
|
||||
s.Job = active[m.ID]
|
||||
out = append(out, s)
|
||||
@@ -466,6 +617,7 @@ func (a *App) handleDelete(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
a.removePXE(m)
|
||||
defer a.kickIPAM()
|
||||
if err := a.store.Delete(r.Context(), m.ID); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
@@ -486,17 +638,7 @@ func (a *App) handleRecommission(w http.ResponseWriter, r *http.Request) {
|
||||
// optional: power-cycle it right away when the caller asks (?now=1)
|
||||
out := obj{"id": m.ID, "state": "recommission"}
|
||||
if r.URL.Query().Get("now") == "1" {
|
||||
if c, err := a.credsFor(m); err == nil {
|
||||
o, ierr := a.ipmi(r.Context(), c, "chassis", "bootdev", "pxe")
|
||||
o2, ierr2 := a.ipmi(r.Context(), c, "chassis", "power", "cycle")
|
||||
if ierr2 != nil && strings.Contains(o2, "off") { // cycle fails when off; power on instead
|
||||
o2, ierr2 = a.ipmi(r.Context(), c, "chassis", "power", "on")
|
||||
}
|
||||
out["ipmi"] = strings.TrimSpace(o + "\n" + o2)
|
||||
out["ipmi_ok"] = ierr == nil && ierr2 == nil
|
||||
} else {
|
||||
out["ipmi"] = err.Error()
|
||||
}
|
||||
out["ipmi"], out["ipmi_ok"] = a.pxeBoot(r.Context(), m)
|
||||
}
|
||||
writeJSON(w, 200, out)
|
||||
}
|
||||
@@ -518,17 +660,15 @@ func (a *App) handlePower(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
action = req.Action
|
||||
}
|
||||
args, ok := powerActions[action]
|
||||
if !ok {
|
||||
if !powerOps[action] || action == "bootdev" || action == "probe" {
|
||||
fail(w, 400, fmt.Errorf("unknown power action %q", action))
|
||||
return
|
||||
}
|
||||
c, err := a.credsFor(m)
|
||||
if err != nil {
|
||||
out, err := a.power(r.Context(), m, action, "")
|
||||
if errors.Is(err, errNoBMC) {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
out, err := a.ipmi(r.Context(), c, args...)
|
||||
if action != "status" {
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "power "+action, out)
|
||||
}
|
||||
@@ -548,12 +688,11 @@ func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 400, fmt.Errorf("device must be one of pxe, disk, cdrom, bios, none, safe"))
|
||||
return
|
||||
}
|
||||
c, err := a.credsFor(m)
|
||||
if err != nil {
|
||||
out, err := a.power(r.Context(), m, "bootdev", req.Device)
|
||||
if errors.Is(err, errNoBMC) {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
out, err := a.ipmi(r.Context(), c, "chassis", "bootdev", req.Device)
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "bootdev "+req.Device, out)
|
||||
writeJSON(w, 200, obj{"id": m.ID, "device": req.Device, "ok": err == nil, "output": out})
|
||||
}
|
||||
@@ -573,8 +712,42 @@ func serve(cfgPath string) error {
|
||||
if err != nil {
|
||||
return fmt.Errorf("database: %w", err)
|
||||
}
|
||||
app := &App{cfg: cfg, store: store, sealer: sealer}
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: app.routes(), ReadHeaderTimeout: 10 * time.Second}
|
||||
app := &App{cfg: cfg, store: store, sealer: sealer, hub: newRackHub()}
|
||||
if app.ca, err = loadOrCreateCA(cfg.CADir, "rollcall CA"); err != nil {
|
||||
return fmt.Errorf("certificate authority: %w", err)
|
||||
}
|
||||
ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
err = app.startRacks(ctx2)
|
||||
cancel2()
|
||||
if err != nil {
|
||||
return fmt.Errorf("rack controllers: %w", err)
|
||||
}
|
||||
if err := app.startIPAM(); err != nil {
|
||||
return fmt.Errorf("ipam: %w", err)
|
||||
}
|
||||
go app.jobWatchdog()
|
||||
handler := app.routes()
|
||||
if cfg.TLSListen != "" {
|
||||
names := append([]string{}, cfg.TLSNames...)
|
||||
if u, err := url.Parse(cfg.ServerURL); err == nil && u.Hostname() != "" {
|
||||
names = append(names, u.Hostname())
|
||||
}
|
||||
pair, err := regionCert(app.ca, cfg.CADir, names)
|
||||
if err != nil {
|
||||
return fmt.Errorf("region certificate: %w", err)
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(app.ca.Cert)
|
||||
tsrv := &http.Server{Addr: cfg.TLSListen, Handler: handler, ReadHeaderTimeout: 10 * time.Second,
|
||||
TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{pair}, ClientCAs: pool, ClientAuth: tls.VerifyClientCertIfGiven}}
|
||||
go func() {
|
||||
log.Printf("rollcall HTTPS on %s (CA %s)", cfg.TLSListen, app.ca.Fingerprint())
|
||||
if err := tsrv.ListenAndServeTLS("", ""); err != nil {
|
||||
log.Printf("HTTPS listener: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: handler, ReadHeaderTimeout: 10 * time.Second}
|
||||
log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "")
|
||||
return srv.ListenAndServe()
|
||||
}
|
||||
+111
-70
@@ -23,6 +23,7 @@ import (
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -185,7 +186,29 @@ func parsePool(spec string) ([]string, error) {
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (a *App) allocate(ctx context.Context, nameFormat, wantName string) (name, mgmt, ib string, err error) {
|
||||
// allocate picks a node name and addresses: from IPAM when the machine's boot
|
||||
// NIC sits in a modeled subnet, else from the legacy node_pools.
|
||||
func (a *App) allocate(ctx context.Context, m *Machine, nameFormat, wantName string) (name, mgmt, ib string, err error) {
|
||||
if name, _, _, err = a.allocatePools(ctx, nameFormat, wantName, false); err != nil {
|
||||
return
|
||||
}
|
||||
if b := bootNIC(machineNICs(m), m.MACs); b != nil {
|
||||
for _, ip := range b.IPv4 {
|
||||
if sn := subnetFor(a.subnetsSnapshot(), ip); sn != nil {
|
||||
have, _ := a.store.IPs(ctx, sn.CIDR, m.ID)
|
||||
for _, x := range have {
|
||||
return name, x.IP, "", nil
|
||||
}
|
||||
mgmt, err = a.allocIP(ctx, *sn, m.ID, b.Name, name)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
_, mgmt, ib, err = a.allocatePools(ctx, nameFormat, name, true)
|
||||
return
|
||||
}
|
||||
|
||||
func (a *App) allocatePools(ctx context.Context, nameFormat, wantName string, pools bool) (name, mgmt, ib string, err error) {
|
||||
ms, err := a.store.List(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
@@ -214,10 +237,13 @@ func (a *App) allocate(ctx context.Context, nameFormat, wantName string) (name,
|
||||
break
|
||||
}
|
||||
}
|
||||
} else if usedName[name] {
|
||||
} else if usedName[name] && !pools {
|
||||
err = fmt.Errorf("node name %s already in use", name)
|
||||
return
|
||||
}
|
||||
if !pools {
|
||||
return
|
||||
}
|
||||
pick := func(pool string) (string, error) {
|
||||
spec := a.cfg.NodePools[pool]
|
||||
if spec == "" {
|
||||
@@ -255,15 +281,26 @@ type NodeCtx struct {
|
||||
GPUs int
|
||||
IBGUIDs []string
|
||||
HostKey, HostKeyPub string // only filled for the node being rendered
|
||||
Tags []string
|
||||
Manufacturer, Product string
|
||||
Disks int
|
||||
NVMe bool
|
||||
PCIVendors []string // vendor ids seen on the PCI bus (e.g. 10de = NVIDIA, 15b3 = Mellanox)
|
||||
OS string
|
||||
Building, Room, Rack string
|
||||
Unit int
|
||||
AssetTag string
|
||||
}
|
||||
|
||||
type overlayCtx struct {
|
||||
Node NodeCtx
|
||||
Nodes []NodeCtx // all machines with a profile, sorted by name
|
||||
Profile Profile
|
||||
Vars map[string]any // cluster_vars overlaid with profile vars
|
||||
Server string
|
||||
Now string
|
||||
Node NodeCtx
|
||||
Nodes []NodeCtx // all machines with a profile, sorted by name
|
||||
Profile Profile
|
||||
Overlays []string // profile overlays plus those of the node's tags
|
||||
CA string // the installation's CA certificate (PEM)
|
||||
Vars map[string]any // cluster_vars overlaid with profile vars
|
||||
Server string
|
||||
Now string
|
||||
}
|
||||
|
||||
func atoi(v any) int {
|
||||
@@ -308,6 +345,22 @@ func nodeCtx(m *Machine) NodeCtx {
|
||||
n.IBGUIDs = append(n.IBGUIDs, g)
|
||||
}
|
||||
}
|
||||
n.Tags = effectiveTags(m)
|
||||
n.Manufacturer, n.Product, n.OS = deref(m.Manufacturer), deref(m.Product), deref(m.OS)
|
||||
n.Building, n.Room, n.Rack, n.Unit, n.AssetTag = m.Location.Building, m.Location.Room, m.Location.Rack, m.Location.Unit, m.Location.AssetTag
|
||||
n.Disks = int(num(dig(inv, "storage", "disk_count")))
|
||||
for _, d := range toList(dig(inv, "storage", "disks")) {
|
||||
if strings.HasPrefix(str(dig(d, "name")), "nvme") || str(dig(d, "tran")) == "nvme" {
|
||||
n.NVMe = true
|
||||
}
|
||||
}
|
||||
seen := map[string]bool{}
|
||||
for _, p := range toList(inv["pci"]) {
|
||||
if v := str(dig(p, "vendor_id")); v != "" && !seen[v] {
|
||||
seen[v] = true
|
||||
n.PCIVendors = append(n.PCIVendors, v)
|
||||
}
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
@@ -332,7 +385,12 @@ var tmplFuncs = template.FuncMap{
|
||||
}
|
||||
return s
|
||||
},
|
||||
"json": func(v any) string { b, _ := json.Marshal(v); return string(b) },
|
||||
"json": func(v any) string { b, _ := json.Marshal(v); return string(b) },
|
||||
"has": hasTag,
|
||||
"contains": func(s, sub string) bool { return strings.Contains(s, sub) },
|
||||
"hasPrefix": strings.HasPrefix,
|
||||
"matches": func(pattern, s string) bool { ok, _ := regexp.MatchString(pattern, s); return ok },
|
||||
"gte": func(a, b int) bool { return a >= b },
|
||||
}
|
||||
|
||||
func (a *App) hostKeyFor(ctx context.Context, m *Machine) (priv, pub string, err error) {
|
||||
@@ -397,7 +455,23 @@ func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) {
|
||||
for k, v := range p.Vars {
|
||||
vars[k] = v
|
||||
}
|
||||
return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Vars: vars, Server: a.cfg.ServerURL,
|
||||
overlays := append([]string{}, p.Overlays...)
|
||||
if defs, err := a.store.Tags(ctx); err == nil {
|
||||
for _, t := range defs {
|
||||
if hasTag(me.Tags, t.Name) {
|
||||
for _, o := range t.Overlays {
|
||||
if !hasTag(overlays, o) {
|
||||
overlays = append(overlays, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
caPEM := ""
|
||||
if a.ca != nil {
|
||||
caPEM = string(a.ca.PEM)
|
||||
}
|
||||
return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, CA: caPEM, Vars: vars, Server: a.urlFor(m),
|
||||
Now: time.Now().Format(time.RFC3339)}, nil
|
||||
}
|
||||
|
||||
@@ -413,7 +487,7 @@ func (a *App) renderOverlay(ctx context.Context, m *Machine, stage string) ([]by
|
||||
var buf bytes.Buffer
|
||||
tw := tar.NewWriter(&buf)
|
||||
seen := map[string]bool{}
|
||||
for _, name := range oc.Profile.Overlays {
|
||||
for _, name := range oc.Overlays {
|
||||
root := filepath.Join(a.cfg.OverlaysDir, name, stage)
|
||||
if st, err := os.Stat(root); err != nil || !st.IsDir() {
|
||||
continue
|
||||
@@ -518,7 +592,7 @@ func (a *App) renderFile(ctx context.Context, profile, stage, rel string) ([]byt
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, name := range p.Overlays {
|
||||
for _, name := range oc.Overlays {
|
||||
for _, cand := range []string{rel + ".tmpl", rel} {
|
||||
fp := filepath.Join(a.cfg.OverlaysDir, name, stage, cand)
|
||||
data, err := os.ReadFile(fp)
|
||||
@@ -570,11 +644,12 @@ LABEL local
|
||||
`
|
||||
|
||||
func (a *App) statelessAppend(m *Machine, p *Profile, token string) string {
|
||||
base := a.urlFor(m)
|
||||
parts := []string{"boot=live", "components",
|
||||
"fetch=" + a.cfg.ServerURL + "/images/" + p.Image + "/filesystem.squashfs",
|
||||
"fetch=" + base + "/images/" + p.Image + "/filesystem.squashfs",
|
||||
"hostname=" + deref(m.NodeName),
|
||||
"live-config.nocomponents=user-setup,sudo",
|
||||
"rollcall.server=" + a.cfg.ServerURL, "rollcall.node=" + token}
|
||||
"rollcall.server=" + base, "rollcall.node=" + token}
|
||||
if p.Append != "" {
|
||||
parts = append(parts, p.Append)
|
||||
}
|
||||
@@ -584,7 +659,7 @@ func (a *App) statelessAppend(m *Machine, p *Profile, token string) string {
|
||||
func (a *App) writePXEStateless(m *Machine, p *Profile, token string) error {
|
||||
body := fmt.Sprintf(statelessTmpl, time.Now().Format(time.RFC3339), deref(m.NodeName), m.ID, p.Name,
|
||||
deref(m.NodeName), p.Name, deref(m.NodeName), p.Name, p.Image, p.Image, a.statelessAppend(m, p, token),
|
||||
a.cfg.CommissionAppend)
|
||||
a.appendFor(a.urlFor(m)))
|
||||
return a.writePXEBody(m, body)
|
||||
}
|
||||
|
||||
@@ -607,51 +682,6 @@ func (a *App) writePXEAny(ctx context.Context, m *Machine) error {
|
||||
return a.writePXE(m)
|
||||
}
|
||||
|
||||
// exportDnsmasq writes dhcp-hostsfile and addn-hosts files; a systemd path unit HUPs dnsmasq.
|
||||
func (a *App) exportDnsmasq(ctx context.Context) error {
|
||||
if a.cfg.DnsmasqDir == "" {
|
||||
return nil
|
||||
}
|
||||
ms, err := a.store.List(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
domain := str(a.cfg.ClusterVars["domain"])
|
||||
var dhcp, hosts bytes.Buffer
|
||||
fmt.Fprintf(&dhcp, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339))
|
||||
fmt.Fprintf(&hosts, "# generated by rollcall %s\n", time.Now().Format(time.RFC3339))
|
||||
for _, m := range ms {
|
||||
if m.NodeName == nil || len(m.MACs) == 0 {
|
||||
continue
|
||||
}
|
||||
name := *m.NodeName
|
||||
if m.MgmtIP != nil {
|
||||
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], *m.MgmtIP, name)
|
||||
if domain != "" {
|
||||
fmt.Fprintf(&hosts, "%s %s.%s %s\n", *m.MgmtIP, name, domain, name)
|
||||
} else {
|
||||
fmt.Fprintf(&hosts, "%s %s\n", *m.MgmtIP, name)
|
||||
}
|
||||
}
|
||||
if m.IBIP != nil {
|
||||
fmt.Fprintf(&hosts, "%s %s-ib\n", *m.IBIP, name)
|
||||
}
|
||||
}
|
||||
if err := os.MkdirAll(a.cfg.DnsmasqDir, 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
for fn, b := range map[string]*bytes.Buffer{"hosts.dhcp": &dhcp, "hosts": &hosts} {
|
||||
p := filepath.Join(a.cfg.DnsmasqDir, fn)
|
||||
if err := os.WriteFile(p+".tmp", b.Bytes(), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.Rename(p+".tmp", p); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- handlers
|
||||
|
||||
func (a *App) handleProfiles(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -725,7 +755,7 @@ func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) {
|
||||
// keep the existing name/IPs when re-assigning an already named node
|
||||
name, mgmt, ib := deref(m.NodeName), deref(m.MgmtIP), deref(m.IBIP)
|
||||
if name == "" || (req.Name != "" && req.Name != name) {
|
||||
if name, mgmt, ib, err = a.allocate(r.Context(), p.NameFormat, req.Name); err != nil {
|
||||
if name, mgmt, ib, err = a.allocate(r.Context(), m, p.NameFormat, req.Name); err != nil {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
@@ -761,21 +791,30 @@ func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) {
|
||||
|
||||
// pxeBoot sets bootdev pxe and power-cycles (or powers on) through the BMC.
|
||||
func (a *App) pxeBoot(ctx context.Context, m *Machine) (string, bool) {
|
||||
c, err := a.credsFor(m)
|
||||
if err != nil {
|
||||
return err.Error(), false
|
||||
o1, e1 := a.power(ctx, m, "bootdev", "pxe")
|
||||
if errors.Is(e1, errNoBMC) {
|
||||
return e1.Error(), false
|
||||
}
|
||||
o1, e1 := a.ipmi(ctx, c, "chassis", "bootdev", "pxe")
|
||||
st, _ := a.ipmi(ctx, c, "chassis", "power", "status")
|
||||
st, _ := a.power(ctx, m, "status", "")
|
||||
var o2 string
|
||||
var e2 error
|
||||
if strings.Contains(st, "off") {
|
||||
o2, e2 = a.ipmi(ctx, c, "chassis", "power", "on")
|
||||
o2, e2 = a.power(ctx, m, "on", "")
|
||||
} else {
|
||||
o2, e2 = a.ipmi(ctx, c, "chassis", "power", "cycle")
|
||||
o2, e2 = a.power(ctx, m, "cycle", "")
|
||||
}
|
||||
_ = a.store.AddEvent(ctx, m.ID, "pxe boot", strings.TrimSpace(o1+"\n"+o2))
|
||||
return strings.TrimSpace(o1 + "\n" + o2), e1 == nil && e2 == nil
|
||||
detail := strings.TrimSpace(o1 + "\n" + o2)
|
||||
ev := "pxe boot"
|
||||
if e1 != nil || e2 != nil { // a lost or failed power command must not look like success
|
||||
ev = "pxe boot failed"
|
||||
for _, e := range []error{e1, e2} {
|
||||
if e != nil {
|
||||
detail = strings.TrimSpace(detail + "\n" + e.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
_ = a.store.AddEvent(ctx, m.ID, ev, detail)
|
||||
return detail, e1 == nil && e2 == nil
|
||||
}
|
||||
|
||||
func (a *App) handleUnassign(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -784,6 +823,8 @@ func (a *App) handleUnassign(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
_ = a.store.FreeMachineIPs(r.Context(), m.ID, "auto")
|
||||
a.kickIPAM()
|
||||
if err := a.store.Unassign(r.Context(), m.ID); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
|
||||
+158
-4
@@ -75,6 +75,111 @@ CREATE TABLE IF NOT EXISTS jobs (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS jobs_machine ON jobs(machine_id, created);
|
||||
CREATE INDEX IF NOT EXISTS jobs_token ON jobs(token_hash);
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
name text PRIMARY KEY,
|
||||
comment text,
|
||||
match text,
|
||||
cloud_config text,
|
||||
overlays jsonb NOT NULL DEFAULT '[]',
|
||||
created timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS tags jsonb NOT NULL DEFAULT '[]';
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS auto_tags jsonb NOT NULL DEFAULT '[]';
|
||||
CREATE INDEX IF NOT EXISTS machines_tags ON machines USING gin(tags);
|
||||
CREATE INDEX IF NOT EXISTS machines_auto_tags ON machines USING gin(auto_tags);
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS building text;
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS room text;
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS rack text;
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS rack_unit int;
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS asset_tag text;
|
||||
CREATE TABLE IF NOT EXISTS racks (
|
||||
building text NOT NULL,
|
||||
room text NOT NULL,
|
||||
name text NOT NULL,
|
||||
height int NOT NULL DEFAULT 42,
|
||||
comment text,
|
||||
PRIMARY KEY (building, room, name)
|
||||
);
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS rack_controller text;
|
||||
ALTER TABLE machines ADD COLUMN IF NOT EXISTS net_layout text;
|
||||
ALTER TABLE tags ADD COLUMN IF NOT EXISTS net_layout text;
|
||||
CREATE TABLE IF NOT EXISTS fabrics (
|
||||
name text PRIMARY KEY,
|
||||
kind text NOT NULL DEFAULT 'ethernet',
|
||||
comment text
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS vlans (
|
||||
id bigserial PRIMARY KEY,
|
||||
fabric text NOT NULL REFERENCES fabrics(name) ON DELETE CASCADE,
|
||||
vid int NOT NULL DEFAULT 0,
|
||||
name text,
|
||||
mtu int NOT NULL DEFAULT 1500,
|
||||
dhcp boolean NOT NULL DEFAULT false,
|
||||
primary_rack text,
|
||||
secondary_rack text,
|
||||
UNIQUE (fabric, vid)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS subnets (
|
||||
cidr text PRIMARY KEY,
|
||||
name text,
|
||||
vlan_id bigint REFERENCES vlans(id) ON DELETE SET NULL,
|
||||
purpose text NOT NULL DEFAULT 'provisioning',
|
||||
gateway text,
|
||||
dns jsonb NOT NULL DEFAULT '[]',
|
||||
domain text,
|
||||
comment text
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ip_ranges (
|
||||
id bigserial PRIMARY KEY,
|
||||
subnet text NOT NULL REFERENCES subnets(cidr) ON DELETE CASCADE,
|
||||
kind text NOT NULL,
|
||||
start_ip text NOT NULL,
|
||||
end_ip text NOT NULL,
|
||||
comment text
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS ip_addresses (
|
||||
ip text PRIMARY KEY,
|
||||
subnet text NOT NULL REFERENCES subnets(cidr) ON DELETE CASCADE,
|
||||
machine_id text,
|
||||
iface text,
|
||||
kind text NOT NULL,
|
||||
hostname text,
|
||||
created timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS ip_addresses_machine ON ip_addresses(machine_id);
|
||||
CREATE TABLE IF NOT EXISTS net_layouts (
|
||||
name text PRIMARY KEY,
|
||||
spec jsonb NOT NULL,
|
||||
comment text
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS rack_controllers (
|
||||
name text PRIMARY KEY,
|
||||
url text,
|
||||
kind text NOT NULL DEFAULT 'remote',
|
||||
version text,
|
||||
interfaces jsonb NOT NULL DEFAULT '[]',
|
||||
dhcp_interfaces jsonb NOT NULL DEFAULT '[]',
|
||||
tftp_root text,
|
||||
last_seen timestamptz,
|
||||
registered timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text;
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz;
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS arch text;
|
||||
CREATE TABLE IF NOT EXISTS join_tokens (
|
||||
hash text PRIMARY KEY,
|
||||
name text,
|
||||
expires timestamptz NOT NULL,
|
||||
used timestamptz,
|
||||
created timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS pxe_files (
|
||||
mac text PRIMARY KEY,
|
||||
machine_id text,
|
||||
rack text,
|
||||
content text NOT NULL,
|
||||
updated timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
`
|
||||
|
||||
var errNotFound = errors.New("not found")
|
||||
@@ -103,6 +208,11 @@ type Machine struct {
|
||||
OS *string `json:"os"`
|
||||
OSVersion *string `json:"os_version"`
|
||||
DeployedAt *time.Time `json:"deployed_at"`
|
||||
Tags []string `json:"tags"` // manual
|
||||
AutoTags []string `json:"auto_tags"` // from tag match rules, recomputed at commissioning
|
||||
Location Location `json:"location"`
|
||||
RackController string `json:"rack_controller"`
|
||||
NetLayout string `json:"net_layout"`
|
||||
bmcSecret []byte
|
||||
nodeToken []byte
|
||||
hostKey []byte
|
||||
@@ -122,20 +232,27 @@ func openStore(ctx context.Context, url string) (*Store, error) {
|
||||
if _, err := pool.Exec(ctx, schema); err != nil {
|
||||
return nil, fmt.Errorf("schema: %w", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, adminCertSchema); err != nil {
|
||||
return nil, fmt.Errorf("schema: %w", err)
|
||||
}
|
||||
return &Store{pool: pool}, nil
|
||||
}
|
||||
|
||||
const machineCols = `id, uuid, serial, hostname, manufacturer, product, state, first_seen, last_seen,
|
||||
commissioned_at, macs, bmc, bmc_secret, inventory, profile, node_name, mgmt_ip, ib_ip, node_token, host_key, booted_at,
|
||||
os, os_version, deployed_at`
|
||||
os, os_version, deployed_at, tags, auto_tags,
|
||||
coalesce(building,''), coalesce(room,''), coalesce(rack,''), coalesce(rack_unit,0), coalesce(asset_tag,''),
|
||||
coalesce(rack_controller,''), coalesce(net_layout,'')`
|
||||
|
||||
func scanMachine(row pgx.Row, full bool) (*Machine, error) {
|
||||
m := &Machine{}
|
||||
var macs []byte
|
||||
var macs, tags, auto []byte
|
||||
err := row.Scan(&m.ID, &m.UUID, &m.Serial, &m.Hostname, &m.Manufacturer, &m.Product, &m.State,
|
||||
&m.FirstSeen, &m.LastSeen, &m.CommissionedAt, &macs, &m.BMC, &m.bmcSecret, &m.Inventory,
|
||||
&m.Profile, &m.NodeName, &m.MgmtIP, &m.IBIP, &m.nodeToken, &m.hostKey, &m.BootedAt,
|
||||
&m.OS, &m.OSVersion, &m.DeployedAt)
|
||||
&m.OS, &m.OSVersion, &m.DeployedAt, &tags, &auto,
|
||||
&m.Location.Building, &m.Location.Room, &m.Location.Rack, &m.Location.Unit, &m.Location.AssetTag,
|
||||
&m.RackController, &m.NetLayout)
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
return nil, errNotFound
|
||||
}
|
||||
@@ -143,6 +260,14 @@ func scanMachine(row pgx.Row, full bool) (*Machine, error) {
|
||||
return nil, err
|
||||
}
|
||||
_ = json.Unmarshal(macs, &m.MACs)
|
||||
_ = json.Unmarshal(tags, &m.Tags)
|
||||
_ = json.Unmarshal(auto, &m.AutoTags)
|
||||
if m.Tags == nil {
|
||||
m.Tags = []string{}
|
||||
}
|
||||
if m.AutoTags == nil {
|
||||
m.AutoTags = []string{}
|
||||
}
|
||||
if !full {
|
||||
m.Inventory = nil
|
||||
}
|
||||
@@ -190,6 +315,29 @@ func (s *Store) Events(ctx context.Context, id string) ([]Event, error) {
|
||||
return evs, rows.Err()
|
||||
}
|
||||
|
||||
type MachineEvent struct {
|
||||
Event
|
||||
MachineID string `json:"machine_id"`
|
||||
}
|
||||
|
||||
// RecentEvents lists the newest events across all machines.
|
||||
func (s *Store) RecentEvents(ctx context.Context, limit int) ([]MachineEvent, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT ts, machine_id, event, coalesce(detail,'') FROM events ORDER BY ts DESC LIMIT $1`, limit)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
evs := []MachineEvent{}
|
||||
for rows.Next() {
|
||||
var e MachineEvent
|
||||
if err := rows.Scan(&e.TS, &e.MachineID, &e.Event.Event, &e.Detail); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
evs = append(evs, e)
|
||||
}
|
||||
return evs, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) AddEvent(ctx context.Context, id, event, detail string) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO events(machine_id, event, detail) VALUES ($1,$2,$3)`, id, event, detail)
|
||||
return err
|
||||
@@ -215,7 +363,7 @@ func (s *Store) UpsertCommissioned(ctx context.Context, m *Machine, secret []byt
|
||||
}
|
||||
// a node that already has a profile goes back to 'stateless', everything else to 'ready'
|
||||
_, err = tx.Exec(ctx, `UPDATE machines SET uuid=$2, serial=$3, hostname=$4, manufacturer=$5, product=$6,
|
||||
state=CASE WHEN profile IS NULL THEN 'ready' ELSE 'stateless' END, booted_at=NULL,
|
||||
state=CASE WHEN profile IS NULL THEN 'ready' ELSE 'stateless' END, booted_at=NULL, os=NULL, os_version=NULL, deployed_at=NULL,
|
||||
last_seen=now(), commissioned_at=now(), macs=$7, bmc=$8, bmc_secret=$9, inventory=$10 WHERE id=$1`,
|
||||
m.ID, m.UUID, m.Serial, m.Hostname, m.Manufacturer, m.Product, macs, m.BMC, secret, m.Inventory)
|
||||
if err != nil {
|
||||
@@ -247,6 +395,12 @@ func (s *Store) Delete(ctx context.Context, id string) error {
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM jobs WHERE machine_id=$1`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM ip_addresses WHERE machine_id=$1`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(ctx, `DELETE FROM pxe_files WHERE machine_id=$1`, id); err != nil {
|
||||
return err
|
||||
}
|
||||
tag, err := tx.Exec(ctx, `DELETE FROM machines WHERE id=$1`, id)
|
||||
if err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,363 @@
|
||||
package main
|
||||
|
||||
// Machine tags, MAAS-style. A tag is a name plus optional
|
||||
// match Go template over the node context that renders "true" for machines
|
||||
// that should carry the tag automatically (re-evaluated at commissioning
|
||||
// and whenever a tag definition changes);
|
||||
// cloud_config a #cloud-config (or #!/bin/sh) fragment merged into the user-data of
|
||||
// every deployment of a tagged machine (e.g. gpu -> NVIDIA packages);
|
||||
// overlays extra overlay directories applied to tagged stateless nodes.
|
||||
// Manual tags are set per machine; effective tags = manual ∪ auto.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
)
|
||||
|
||||
type TagDef struct {
|
||||
Name string `json:"name"`
|
||||
Comment string `json:"comment"`
|
||||
Match string `json:"match"`
|
||||
CloudConfig string `json:"cloud_config"`
|
||||
Overlays []string `json:"overlays"`
|
||||
NetLayout string `json:"net_layout"`
|
||||
Created time.Time `json:"created"`
|
||||
Machines int `json:"machines,omitempty"` // filled by the list handler
|
||||
}
|
||||
|
||||
var tagNameRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,63}$`)
|
||||
|
||||
// ---------------------------------------------------------------- store
|
||||
|
||||
func (s *Store) Tags(ctx context.Context) ([]TagDef, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(comment,''), coalesce(match,''), coalesce(cloud_config,''), overlays, coalesce(net_layout,''), created FROM tags ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
out := []TagDef{}
|
||||
for rows.Next() {
|
||||
var t TagDef
|
||||
var ov []byte
|
||||
if err := rows.Scan(&t.Name, &t.Comment, &t.Match, &t.CloudConfig, &ov, &t.NetLayout, &t.Created); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = json.Unmarshal(ov, &t.Overlays)
|
||||
if t.Overlays == nil {
|
||||
t.Overlays = []string{}
|
||||
}
|
||||
out = append(out, t)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
func (s *Store) PutTag(ctx context.Context, t TagDef) error {
|
||||
if t.Overlays == nil {
|
||||
t.Overlays = []string{}
|
||||
}
|
||||
ov, _ := json.Marshal(t.Overlays)
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO tags(name, comment, match, cloud_config, overlays, net_layout) VALUES ($1,$2,$3,$4,$5,nullif($6,''))
|
||||
ON CONFLICT (name) DO UPDATE SET comment=$2, match=$3, cloud_config=$4, overlays=$5, net_layout=nullif($6,'')`,
|
||||
t.Name, t.Comment, t.Match, t.CloudConfig, ov, t.NetLayout)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) DeleteTag(ctx context.Context, name string) error {
|
||||
tag, err := s.pool.Exec(ctx, `DELETE FROM tags WHERE name=$1`, name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return errNotFound
|
||||
}
|
||||
// drop it from every machine (manual and auto)
|
||||
_, err = s.pool.Exec(ctx, `UPDATE machines SET tags = tags - $1, auto_tags = auto_tags - $1`, name)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) SetMachineTags(ctx context.Context, id string, manual, auto []string) error {
|
||||
mb, _ := json.Marshal(sortedUnique(manual))
|
||||
ab, _ := json.Marshal(sortedUnique(auto))
|
||||
tag, err := s.pool.Exec(ctx, `UPDATE machines SET tags=$2, auto_tags=$3 WHERE id=$1`, id, mb, ab)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if tag.RowsAffected() == 0 {
|
||||
return errNotFound
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func sortedUnique(l []string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := []string{}
|
||||
for _, s := range l {
|
||||
s = strings.TrimSpace(s)
|
||||
if s != "" && !seen[s] {
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func hasTag(list []string, t string) bool {
|
||||
for _, x := range list {
|
||||
if x == t {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// effectiveTags is manual ∪ auto, sorted.
|
||||
func effectiveTags(m *Machine) []string {
|
||||
return sortedUnique(append(append([]string{}, m.Tags...), m.AutoTags...))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- matching
|
||||
|
||||
// matchTag renders the tag's match template against the node context; "true" (after
|
||||
// trimming) means the tag applies.
|
||||
func matchTag(t TagDef, n NodeCtx) (bool, error) {
|
||||
if strings.TrimSpace(t.Match) == "" {
|
||||
return false, nil
|
||||
}
|
||||
tmpl, err := template.New(t.Name).Funcs(tmplFuncs).Option("missingkey=zero").Parse(t.Match)
|
||||
if err != nil {
|
||||
return false, fmt.Errorf("tag %s: %w", t.Name, err)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := tmpl.Execute(&out, map[string]any{"Node": n}); err != nil {
|
||||
return false, fmt.Errorf("tag %s: %w", t.Name, err)
|
||||
}
|
||||
return strings.TrimSpace(out.String()) == "true", nil
|
||||
}
|
||||
|
||||
// autoTagsFor evaluates every tag with a match rule against a machine.
|
||||
func autoTagsFor(defs []TagDef, m *Machine) ([]string, error) {
|
||||
n := nodeCtx(m)
|
||||
var out []string
|
||||
var firstErr error
|
||||
for _, t := range defs {
|
||||
ok, err := matchTag(t, n)
|
||||
if err != nil && firstErr == nil {
|
||||
firstErr = err
|
||||
}
|
||||
if ok {
|
||||
out = append(out, t.Name)
|
||||
}
|
||||
}
|
||||
return sortedUnique(out), firstErr
|
||||
}
|
||||
|
||||
// retag recomputes auto tags for the given machines (all when nil).
|
||||
func (a *App) retag(ctx context.Context, only []*Machine) {
|
||||
defs, err := a.store.Tags(ctx)
|
||||
if err != nil {
|
||||
log.Printf("retag: %v", err)
|
||||
return
|
||||
}
|
||||
ms := only
|
||||
if ms == nil {
|
||||
if ms, err = a.store.List(ctx); err != nil {
|
||||
log.Printf("retag: %v", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, m := range ms {
|
||||
auto, err := autoTagsFor(defs, m)
|
||||
if err != nil {
|
||||
log.Printf("retag %s: %v", m.ID, err)
|
||||
}
|
||||
if strings.Join(auto, ",") == strings.Join(m.AutoTags, ",") {
|
||||
continue
|
||||
}
|
||||
if err := a.store.SetMachineTags(ctx, m.ID, m.Tags, auto); err != nil {
|
||||
log.Printf("retag %s: %v", m.ID, err)
|
||||
continue
|
||||
}
|
||||
_ = a.store.AddEvent(ctx, m.ID, "auto-tags", strings.Join(auto, " "))
|
||||
m.AutoTags = auto
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- cloud-init fragments
|
||||
|
||||
// tagFragments returns the cloud_config fragments of the machine's effective tags.
|
||||
func (a *App) tagFragments(ctx context.Context, m *Machine) ([]TagDef, error) {
|
||||
defs, err := a.store.Tags(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
eff := effectiveTags(m)
|
||||
var out []TagDef
|
||||
for _, t := range defs {
|
||||
if hasTag(eff, t.Name) && strings.TrimSpace(t.CloudConfig) != "" {
|
||||
out = append(out, t)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// multipartUserData packs the base cloud-config and tag fragments into the
|
||||
// multipart/mixed user-data cloud-init expects. Cloud-config parts get a merge
|
||||
// directive so lists (packages, runcmd, write_files) append instead of replace.
|
||||
func multipartUserData(base string, frags []TagDef) string {
|
||||
var buf bytes.Buffer
|
||||
w := multipart.NewWriter(&buf)
|
||||
add := func(name, body string) {
|
||||
ctype := "text/cloud-config"
|
||||
if strings.HasPrefix(body, "#!") {
|
||||
ctype = "text/x-shellscript"
|
||||
} else if strings.HasPrefix(body, "#cloud-boothook") {
|
||||
ctype = "text/cloud-boothook"
|
||||
} else if !strings.HasPrefix(body, "#cloud-config") {
|
||||
body = "#cloud-config\n" + body
|
||||
}
|
||||
if ctype == "text/cloud-config" && !strings.Contains(body, "merge_how") && !strings.Contains(body, "merge_type") {
|
||||
body = strings.Replace(body, "#cloud-config", "#cloud-config\nmerge_how: \"list(append)+dict(recurse_array,no_replace)+str()\"", 1)
|
||||
}
|
||||
h := textproto.MIMEHeader{}
|
||||
h.Set("Content-Type", ctype+`; charset="us-ascii"`)
|
||||
h.Set("Content-Disposition", `attachment; filename="`+name+`"`)
|
||||
h.Set("MIME-Version", "1.0")
|
||||
p, _ := w.CreatePart(h)
|
||||
_, _ = p.Write([]byte(body))
|
||||
if !strings.HasSuffix(body, "\n") {
|
||||
_, _ = p.Write([]byte("\n"))
|
||||
}
|
||||
}
|
||||
add("rollcall-base.cfg", base)
|
||||
for _, t := range frags {
|
||||
add("tag-"+t.Name+".cfg", t.CloudConfig)
|
||||
}
|
||||
_ = w.Close()
|
||||
head := "Content-Type: multipart/mixed; boundary=\"" + w.Boundary() + "\"\nMIME-Version: 1.0\nNumber-Attachments: " + fmt.Sprint(len(frags)+1) + "\n\n"
|
||||
return head + buf.String()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- handlers
|
||||
|
||||
func (a *App) handleTags(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
defs, err := a.store.Tags(r.Context())
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
ms, _ := a.store.List(r.Context())
|
||||
for i := range defs {
|
||||
for _, m := range ms {
|
||||
if hasTag(effectiveTags(m), defs[i].Name) {
|
||||
defs[i].Machines++
|
||||
}
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, defs)
|
||||
case http.MethodPost:
|
||||
var t TagDef
|
||||
if err := json.NewDecoder(r.Body).Decode(&t); err != nil || !tagNameRe.MatchString(t.Name) {
|
||||
fail(w, 400, fmt.Errorf("name is required: lowercase letters, digits, . _ -"))
|
||||
return
|
||||
}
|
||||
if _, err := template.New(t.Name).Funcs(tmplFuncs).Parse(t.Match); err != nil {
|
||||
fail(w, 400, fmt.Errorf("match: %w", err))
|
||||
return
|
||||
}
|
||||
for _, o := range t.Overlays {
|
||||
if strings.ContainsAny(o, "/ \t") {
|
||||
fail(w, 400, fmt.Errorf("overlay names must be plain identifiers"))
|
||||
return
|
||||
}
|
||||
}
|
||||
if err := a.store.PutTag(r.Context(), t); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.retag(r.Context(), nil)
|
||||
a.rewriteTaggedPXE(r.Context(), t.Name)
|
||||
writeJSON(w, 200, t)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleTagDelete(w http.ResponseWriter, r *http.Request) {
|
||||
if err := a.store.DeleteTag(r.Context(), r.PathValue("name")); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, obj{"deleted": r.PathValue("name")})
|
||||
}
|
||||
|
||||
// handleMachineTags: {"add":["gpu"],"remove":["staging"]} or {"set":[...]} for manual tags.
|
||||
func (a *App) handleMachineTags(w http.ResponseWriter, r *http.Request) {
|
||||
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Add, Remove, Set []string
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
manual := m.Tags
|
||||
if req.Set != nil {
|
||||
manual = req.Set
|
||||
}
|
||||
manual = append(append([]string{}, manual...), req.Add...)
|
||||
for _, name := range append(req.Add, req.Set...) {
|
||||
if !tagNameRe.MatchString(name) {
|
||||
fail(w, 400, fmt.Errorf("bad tag name %q", name))
|
||||
return
|
||||
}
|
||||
}
|
||||
var kept []string
|
||||
for _, t := range manual {
|
||||
if !hasTag(req.Remove, t) {
|
||||
kept = append(kept, t)
|
||||
}
|
||||
}
|
||||
if err := a.store.SetMachineTags(r.Context(), m.ID, kept, m.AutoTags); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
// tags without a definition are fine (free-form), but create the definition so they show up in lists
|
||||
defs, _ := a.store.Tags(r.Context())
|
||||
for _, t := range kept {
|
||||
known := false
|
||||
for _, d := range defs {
|
||||
if d.Name == t {
|
||||
known = true
|
||||
}
|
||||
}
|
||||
if !known {
|
||||
_ = a.store.PutTag(r.Context(), TagDef{Name: t, Comment: "created from machine " + m.ID})
|
||||
}
|
||||
}
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "tags", strings.Join(effectiveTags(m), " "))
|
||||
if m.Profile != nil { // overlays may have changed
|
||||
_ = a.writePXEAny(r.Context(), m)
|
||||
}
|
||||
writeJSON(w, 200, obj{"id": m.ID, "tags": m.Tags, "auto_tags": m.AutoTags, "effective": effectiveTags(m)})
|
||||
}
|
||||
|
||||
// rewriteTaggedPXE is a no-op placeholder for PXE entries (tags do not change the kernel
|
||||
// line today); kept so overlay changes can hook in later.
|
||||
func (a *App) rewriteTaggedPXE(ctx context.Context, tag string) {}
|
||||
+2281
File diff suppressed because it is too large.
Load diff
@@ -0,0 +1,362 @@
|
||||
package main
|
||||
|
||||
// The frame every full-screen rollcall screen is drawn in, the same design as
|
||||
// llmgw's: a title bar, a sidebar on the left, the content in a bordered panel
|
||||
// on the right, the keys along the bottom, and a dialog over it all when one is
|
||||
// open. Keyboard only: these screens run on serial consoles and BMC KVMs as
|
||||
// often as in a desktop terminal.
|
||||
//
|
||||
// The two widgets are written here rather than taken from bubbles: its
|
||||
// packages ask the terminal for its colours as they load, which would make
|
||||
// every CLI subcommand wait on a terminal that never answers.
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/lipgloss"
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
)
|
||||
|
||||
var (
|
||||
tuiAccent = lipgloss.AdaptiveColor{Light: "#0B7F7D", Dark: "#3FC2BC"}
|
||||
tuiOK = lipgloss.AdaptiveColor{Light: "#1F8A4C", Dark: "#6FD3A6"}
|
||||
tuiWarn = lipgloss.AdaptiveColor{Light: "#A15C00", Dark: "#E9B35A"}
|
||||
tuiBad = lipgloss.AdaptiveColor{Light: "#B42318", Dark: "#F07C6C"}
|
||||
tuiDim = lipgloss.AdaptiveColor{Light: "#56696C", Dark: "#8FA5A4"}
|
||||
|
||||
sTitle = lipgloss.NewStyle().Bold(true).Foreground(tuiAccent)
|
||||
sBold = lipgloss.NewStyle().Bold(true)
|
||||
sDim = lipgloss.NewStyle().Foreground(tuiDim)
|
||||
sOK = lipgloss.NewStyle().Foreground(tuiOK)
|
||||
sWarn = lipgloss.NewStyle().Foreground(tuiWarn)
|
||||
sBad = lipgloss.NewStyle().Foreground(tuiBad)
|
||||
|
||||
fBar = lipgloss.NewStyle().Bold(true).Foreground(lipgloss.AdaptiveColor{Light: "#FFFFFF", Dark: "#0B1B1B"}).Background(tuiAccent)
|
||||
fBorder = lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(tuiDim)
|
||||
fFocus = lipgloss.NewStyle().Border(lipgloss.RoundedBorder()).BorderForeground(tuiAccent)
|
||||
fKeys = lipgloss.NewStyle().Foreground(tuiDim)
|
||||
fSelected = lipgloss.NewStyle().Bold(true).Foreground(tuiAccent)
|
||||
fDialog = lipgloss.NewStyle().Border(lipgloss.DoubleBorder()).BorderForeground(tuiAccent).Padding(1, 2)
|
||||
)
|
||||
|
||||
const navWidth = 22
|
||||
|
||||
// navItem is one line of the sidebar.
|
||||
type navItem struct {
|
||||
Label string
|
||||
State string // "done", "current", "todo" for steps; "" for menu entries
|
||||
Current bool
|
||||
Dim bool
|
||||
Heading bool // a title between groups, not an entry
|
||||
}
|
||||
|
||||
type frame struct {
|
||||
Width, Height int
|
||||
Title string // left of the title bar
|
||||
Where string // right of the title bar
|
||||
Nav []navItem
|
||||
PanelTitle string
|
||||
Body string
|
||||
Scroll int // body lines scrolled off the top
|
||||
Keys string
|
||||
Dialog string
|
||||
DialogTitle string
|
||||
}
|
||||
|
||||
// render draws the whole screen, exactly Width by Height.
|
||||
func (f frame) render() string {
|
||||
w, h := max(f.Width, 60), max(f.Height, 16)
|
||||
|
||||
title := " " + f.Title
|
||||
right := version + " "
|
||||
if f.Where != "" {
|
||||
right = f.Where + " · " + right
|
||||
}
|
||||
gap := max(1, w-ansi.StringWidth(title)-ansi.StringWidth(right))
|
||||
bar := fBar.Render(truncateWidth(title+strings.Repeat(" ", gap)+right, w))
|
||||
|
||||
ph := h - 2
|
||||
inner := ph - 2
|
||||
|
||||
var navLines []string
|
||||
cur := 0
|
||||
for _, it := range f.Nav {
|
||||
mark := " "
|
||||
label := it.Label
|
||||
if it.Heading {
|
||||
navLines = append(navLines, sDim.Render(label))
|
||||
continue
|
||||
}
|
||||
if it.Current || it.State == "current" {
|
||||
cur = len(navLines)
|
||||
}
|
||||
switch {
|
||||
case it.State == "done":
|
||||
mark = sOK.Render("✓ ")
|
||||
case it.State == "current", it.Current:
|
||||
mark = fSelected.Render("› ")
|
||||
label = fSelected.Render(label)
|
||||
case it.Dim || it.State == "todo":
|
||||
label = sDim.Render(label)
|
||||
}
|
||||
navLines = append(navLines, truncateWidth(mark+label, navWidth-2))
|
||||
}
|
||||
if len(navLines) > inner {
|
||||
total := len(navLines)
|
||||
top := min(max(0, cur-inner/2), total-inner)
|
||||
navLines = append([]string{}, navLines[top:top+inner]...)
|
||||
if top > 0 {
|
||||
navLines[0] = sDim.Render(" ↑ more")
|
||||
}
|
||||
if top+inner < total {
|
||||
navLines[inner-1] = sDim.Render(" ↓ more")
|
||||
}
|
||||
}
|
||||
left := box(fBorder, "", strings.Join(navLines, "\n"), navWidth, inner)
|
||||
|
||||
pw := w - navWidth - 2 - 2
|
||||
lines := strings.Split(strings.TrimRight(f.Body, "\n"), "\n")
|
||||
scroll := min(max(f.Scroll, 0), max(0, len(lines)-inner))
|
||||
visible := lines[scroll:]
|
||||
if len(visible) > inner {
|
||||
visible = visible[:inner]
|
||||
visible[inner-1] = sDim.Render(" ↓ more — PgDn")
|
||||
}
|
||||
for i := range visible {
|
||||
visible[i] = truncateWidth(visible[i], pw)
|
||||
}
|
||||
panelTitle := f.PanelTitle
|
||||
if scroll > 0 {
|
||||
panelTitle += " ↑"
|
||||
}
|
||||
main := box(fFocus, panelTitle, strings.Join(visible, "\n"), pw, inner)
|
||||
|
||||
keys := fKeys.Render(" " + truncateWidth(f.Keys, w-2))
|
||||
screen := bar + "\n" + lipgloss.JoinHorizontal(lipgloss.Top, left, main) + "\n" + keys
|
||||
|
||||
if f.Dialog != "" {
|
||||
dw := min(w-8, max(50, maxLineWidth(f.Dialog)+6))
|
||||
d := fDialog.Width(dw).Render(f.Dialog)
|
||||
if f.DialogTitle != "" {
|
||||
d = titled(d, f.DialogTitle)
|
||||
}
|
||||
screen = overlay(screen, d, w, h)
|
||||
}
|
||||
return screen
|
||||
}
|
||||
|
||||
func box(style lipgloss.Style, title, content string, width, height int) string {
|
||||
b := style.Width(width).Height(height).Render(content)
|
||||
if title == "" {
|
||||
return b
|
||||
}
|
||||
return titled(b, title)
|
||||
}
|
||||
|
||||
// titled sets " title " into a box's top border, after its corner.
|
||||
func titled(b, title string) string {
|
||||
lines := strings.Split(b, "\n")
|
||||
if len(lines) == 0 {
|
||||
return b
|
||||
}
|
||||
top := lines[0]
|
||||
t := " " + title + " "
|
||||
tw := ansi.StringWidth(t)
|
||||
if ansi.StringWidth(top) < tw+4 {
|
||||
return b
|
||||
}
|
||||
lines[0] = ansi.Truncate(top, 2, "") + sTitle.Render(t) + ansi.TruncateLeft(top, 2+tw, "")
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
// overlay draws top over the middle of base, both w by h.
|
||||
func overlay(base, top string, w, h int) string {
|
||||
bl := strings.Split(base, "\n")
|
||||
tl := strings.Split(top, "\n")
|
||||
tw := maxLineWidth(top)
|
||||
x := max(0, (w-tw)/2)
|
||||
y := max(0, (h-len(tl))/2)
|
||||
for i, l := range tl {
|
||||
row := y + i
|
||||
if row >= len(bl) {
|
||||
break
|
||||
}
|
||||
under := bl[row]
|
||||
left := ansi.Truncate(under, x, "")
|
||||
if pad := x - ansi.StringWidth(left); pad > 0 {
|
||||
left += strings.Repeat(" ", pad)
|
||||
}
|
||||
right := ansi.TruncateLeft(under, x+ansi.StringWidth(l), "")
|
||||
bl[row] = left + l + right
|
||||
}
|
||||
return strings.Join(bl, "\n")
|
||||
}
|
||||
|
||||
func maxLineWidth(s string) int {
|
||||
n := 0
|
||||
for _, l := range strings.Split(s, "\n") {
|
||||
n = max(n, ansi.StringWidth(l))
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// truncateWidth cuts a line, colours and all, to w cells.
|
||||
func truncateWidth(s string, w int) string {
|
||||
if ansi.StringWidth(s) <= w {
|
||||
return s
|
||||
}
|
||||
return ansi.Truncate(s, max(w-1, 0), "…")
|
||||
}
|
||||
|
||||
// table draws rows under a header in columns as wide as they need, cut to
|
||||
// width; the selected row is shown reversed when the panel has the focus.
|
||||
func table(headers []string, rows [][]string, selected int, width int, focused bool) string {
|
||||
widths := make([]int, len(headers))
|
||||
for i, h := range headers {
|
||||
widths[i] = ansi.StringWidth(h)
|
||||
}
|
||||
for _, r := range rows {
|
||||
for i := range headers {
|
||||
if i < len(r) {
|
||||
widths[i] = min(max(widths[i], ansi.StringWidth(r[i])), 40)
|
||||
}
|
||||
}
|
||||
}
|
||||
line := func(cells []string) string {
|
||||
var b strings.Builder
|
||||
for i := range headers {
|
||||
c := ""
|
||||
if i < len(cells) {
|
||||
c = truncateWidth(cells[i], widths[i])
|
||||
}
|
||||
b.WriteString(c)
|
||||
if i < len(headers)-1 {
|
||||
b.WriteString(strings.Repeat(" ", widths[i]-ansi.StringWidth(c)+2))
|
||||
}
|
||||
}
|
||||
return truncateWidth(b.String(), width)
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString(" " + lipgloss.NewStyle().Bold(true).Foreground(tuiDim).Render(line(headers)) + "\n")
|
||||
for i, r := range rows {
|
||||
l := line(r)
|
||||
switch {
|
||||
case i == selected && focused:
|
||||
plain := ansi.Strip(l)
|
||||
pad := max(0, width-ansi.StringWidth(plain))
|
||||
l = fBar.Render(plain + strings.Repeat(" ", pad))
|
||||
case i == selected:
|
||||
l = sBold.Render(ansi.Strip(l))
|
||||
}
|
||||
b.WriteString(" " + l + "\n")
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
b.WriteString(" " + sDim.Render("nothing yet") + "\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// wrap breaks text at spaces to width; continuation lines start with indent.
|
||||
func wrap(s string, width int, indent string) string {
|
||||
if width < 10 {
|
||||
return s
|
||||
}
|
||||
var out []string
|
||||
for _, para := range strings.Split(s, "\n") {
|
||||
line := ""
|
||||
for _, word := range strings.Fields(para) {
|
||||
if line != "" && ansi.StringWidth(line)+1+ansi.StringWidth(word) > width {
|
||||
out = append(out, line)
|
||||
line = indent + word
|
||||
continue
|
||||
}
|
||||
if line == "" {
|
||||
line = word
|
||||
} else {
|
||||
line += " " + word
|
||||
}
|
||||
}
|
||||
out = append(out, line)
|
||||
}
|
||||
return strings.Join(out, "\n")
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- widgets
|
||||
|
||||
type spinTickMsg struct{}
|
||||
|
||||
var spinFrames = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"}
|
||||
|
||||
type spinnerModel struct{ frame int }
|
||||
|
||||
func (s spinnerModel) Tick() tea.Msg { return spinTickMsg{} }
|
||||
|
||||
func (s spinnerModel) Update(msg tea.Msg) (spinnerModel, tea.Cmd) {
|
||||
if _, ok := msg.(spinTickMsg); !ok {
|
||||
return s, nil
|
||||
}
|
||||
s.frame = (s.frame + 1) % len(spinFrames)
|
||||
return s, tea.Tick(100*time.Millisecond, func(time.Time) tea.Msg { return spinTickMsg{} })
|
||||
}
|
||||
|
||||
func (s spinnerModel) View() string { return sTitle.Render(spinFrames[s.frame]) }
|
||||
|
||||
// textField is one line of text typed at its end: all a form here needs.
|
||||
type textField struct {
|
||||
value []rune
|
||||
focused bool
|
||||
Prompt string
|
||||
Placeholder string
|
||||
CharLimit int
|
||||
Mask bool // a password
|
||||
}
|
||||
|
||||
func newTextField() textField { return textField{} }
|
||||
|
||||
func (f *textField) SetValue(s string) { f.value = []rune(s) }
|
||||
func (f textField) Value() string { return string(f.value) }
|
||||
func (f *textField) Focus() { f.focused = true }
|
||||
func (f *textField) Blur() { f.focused = false }
|
||||
|
||||
func (f textField) Update(msg tea.Msg) (textField, tea.Cmd) {
|
||||
k, ok := msg.(tea.KeyMsg)
|
||||
if !ok || !f.focused {
|
||||
return f, nil
|
||||
}
|
||||
switch k.Type {
|
||||
case tea.KeyRunes, tea.KeySpace:
|
||||
r := k.Runes
|
||||
if k.Type == tea.KeySpace {
|
||||
r = []rune{' '}
|
||||
}
|
||||
for _, c := range r {
|
||||
if f.CharLimit == 0 || len(f.value) < f.CharLimit {
|
||||
f.value = append(f.value, c)
|
||||
}
|
||||
}
|
||||
case tea.KeyBackspace:
|
||||
if len(f.value) > 0 {
|
||||
f.value = f.value[:len(f.value)-1]
|
||||
}
|
||||
case tea.KeyCtrlU:
|
||||
f.value = nil
|
||||
}
|
||||
return f, nil
|
||||
}
|
||||
|
||||
func (f textField) View() string {
|
||||
text := string(f.value)
|
||||
if f.Mask {
|
||||
text = strings.Repeat("•", len(f.value))
|
||||
}
|
||||
shown := text
|
||||
if text == "" && f.Placeholder != "" {
|
||||
shown = sDim.Render(f.Placeholder)
|
||||
}
|
||||
if f.focused {
|
||||
shown += "█"
|
||||
}
|
||||
return f.Prompt + shown
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
tea "github.com/charmbracelet/bubbletea"
|
||||
"github.com/charmbracelet/x/ansi"
|
||||
)
|
||||
|
||||
// fakeModel is a console with a small fleet loaded, no server needed.
|
||||
func fakeModel(t *testing.T, w, h int) tuiModel {
|
||||
t.Helper()
|
||||
now := time.Now()
|
||||
sp, _ := json.Marshal(DeploySpec{OS: "rocky-9", Version: "20261027-92c206", Disk: "/dev/sda"})
|
||||
prog := json.RawMessage(`{"phase":"writing","pct":42,"msg":"42% of 552 MiB"}`)
|
||||
job := &Job{ID: "j-1", MachineID: "m-a", Kind: "deploy", State: "running", Spec: sp, Progress: prog, Created: now}
|
||||
name, os, ip := "node001", "rocky-9", "10.0.0.50"
|
||||
bmc := obj{"address": "10.0.0.11", "configured": true}
|
||||
m := newTUIModel(&client{base: "http://test"})
|
||||
m.w, m.h = w, h
|
||||
m.loaded = true
|
||||
m.d = tuiData{at: now,
|
||||
machines: []Summary{
|
||||
{ID: "m-a", State: "deploying", NodeName: &name, OS: &os, MgmtIP: &ip, CPUs: "64", MemoryBytes: 512 << 30, Disks: 2, StorageBytes: 8 << 40, GPUs: 4, IBHCAs: 2,
|
||||
BMC: &bmc, Tags: []string{"gpu", "rack-3"}, ManualTags: []string{"rack-3"}, AutoTags: []string{"gpu"}, Job: job, LastSeen: &now,
|
||||
Location: Location{Building: "DC1", Room: "Hall A", Rack: "R12", Unit: 24}},
|
||||
{ID: "m-b", State: "ready", CPUs: "32", MemoryBytes: 256 << 30, LastSeen: &now},
|
||||
},
|
||||
racks: []Rack{{Building: "DC1", Room: "Hall A", Name: "R12", Height: 42}},
|
||||
jobs: []*Job{job},
|
||||
events: []MachineEvent{{Event: Event{TS: now, Event: "deploy started", Detail: "job=j-1"}, MachineID: "m-a"}},
|
||||
tags: []TagDef{{Name: "gpu", Match: "{{ gt .Node.GPUs 0 }}", CloudConfig: "#cloud-config\npackages: [x]", Machines: 1}},
|
||||
os: []OSImage{{Name: "rocky-9", Current: &OSMeta{Version: "20261027-92c206", Size: 553 << 20, RawSize: 10 << 30, Imported: now}}},
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestTUIPagesRender(t *testing.T) {
|
||||
for _, size := range [][2]int{{140, 40}, {80, 24}, {60, 16}} {
|
||||
for page := 0; page < pCount; page++ {
|
||||
m := fakeModel(t, size[0], size[1])
|
||||
m.menu, m.inPanel = page, true
|
||||
out := ansi.Strip(m.View())
|
||||
lines := strings.Split(out, "\n")
|
||||
if len(lines) != max(size[1], 16) {
|
||||
t.Errorf("%dx%d %s: %d lines", size[0], size[1], pageNames[page], len(lines))
|
||||
}
|
||||
for i, l := range lines {
|
||||
if w := ansi.StringWidth(l); w > max(size[0], 60) {
|
||||
t.Errorf("%dx%d %s: line %d is %d wide", size[0], size[1], pageNames[page], i, w)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(out, pageLabels[page]) {
|
||||
t.Errorf("%s: label missing", pageNames[page])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestTUIMachinesListAndFilter(t *testing.T) {
|
||||
m := fakeModel(t, 160, 30)
|
||||
m.menu, m.inPanel = pMachines, true
|
||||
out := ansi.Strip(m.View())
|
||||
for _, want := range []string{"node001", "deploying 42%", "Hall A/R12 U24", "64c 512.0G 2×8.0T 4GPU 2IB", "m-b"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("machines page lacks %q", want)
|
||||
}
|
||||
}
|
||||
m.filter.SetValue("r12")
|
||||
if got := len(m.filtered()); got != 1 {
|
||||
t.Errorf("filter r12: %d machines, want 1", got)
|
||||
}
|
||||
m.filter.SetValue("")
|
||||
m.stateF = "ready"
|
||||
if got := m.filtered(); len(got) != 1 || got[0].ID != "m-b" {
|
||||
t.Errorf("state filter: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTUIKeysSelectionAndActions(t *testing.T) {
|
||||
m := fakeModel(t, 140, 40)
|
||||
press := func(s string) {
|
||||
var k tea.KeyMsg
|
||||
switch s {
|
||||
case "enter":
|
||||
k = tea.KeyMsg{Type: tea.KeyEnter}
|
||||
case "esc":
|
||||
k = tea.KeyMsg{Type: tea.KeyEsc}
|
||||
case " ":
|
||||
k = tea.KeyMsg{Type: tea.KeySpace}
|
||||
default:
|
||||
k = tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(s)}
|
||||
}
|
||||
mm, _ := m.Update(k)
|
||||
m = mm.(tuiModel)
|
||||
}
|
||||
press("j") // Machines
|
||||
press("enter")
|
||||
if !m.inPanel || m.menu != pMachines {
|
||||
t.Fatalf("not in the machines panel: menu=%d inPanel=%v", m.menu, m.inPanel)
|
||||
}
|
||||
press(" ")
|
||||
press(" ")
|
||||
if len(m.sel) != 2 {
|
||||
t.Fatalf("selected %d, want 2", len(m.sel))
|
||||
}
|
||||
press("a")
|
||||
if m.dlg == nil || !strings.Contains(m.dlg.title, "2 machines") {
|
||||
t.Fatalf("actions dialog for the selection not open: %+v", m.dlg)
|
||||
}
|
||||
out := ansi.Strip(m.View())
|
||||
for _, want := range []string{"Power cycle", "Deploy OS…", "Location…", "Delete record"} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("actions menu lacks %q", want)
|
||||
}
|
||||
}
|
||||
// Deploy OS… opens a form with the OS image choice
|
||||
for i, it := range m.dlg.menu {
|
||||
if it.label == "Deploy OS…" {
|
||||
m.dlg.focus = i
|
||||
}
|
||||
}
|
||||
press("enter")
|
||||
if m.dlg == nil || len(m.dlg.fields) == 0 || m.dlg.fields[0].value() != "rocky-9" {
|
||||
t.Fatalf("deploy form not open with rocky-9: %+v", m.dlg)
|
||||
}
|
||||
press("esc")
|
||||
press("esc") // clears the selection
|
||||
if len(m.sel) != 0 {
|
||||
t.Errorf("Esc did not clear the selection")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTUIBatchedRunes(t *testing.T) {
|
||||
m := fakeModel(t, 140, 40)
|
||||
mm, _ := m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("jj")})
|
||||
if got := mm.(tuiModel).menu; got != pRacks {
|
||||
t.Errorf("\"jj\" in one message moved to page %d, want %d (Racks)", got, pRacks)
|
||||
}
|
||||
m.filtering = true
|
||||
m.filter.Focus()
|
||||
mm, _ = m.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("r12")})
|
||||
if got := mm.(tuiModel).filter.Value(); got != "r12" {
|
||||
t.Errorf("typed into the filter: %q, want r12", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTUIRackElevation(t *testing.T) {
|
||||
m := fakeModel(t, 120, 60)
|
||||
m.menu, m.inPanel, m.rackOpen = pRacks, true, 0
|
||||
out := ansi.Strip(m.View())
|
||||
if !strings.Contains(out, "24 │ node001") {
|
||||
t.Errorf("elevation lacks node001 at U24:\n%s", out)
|
||||
}
|
||||
}
|
||||
+773
-220
File diff suppressed because it is too large.
Load diff
+33
-90
@@ -3,109 +3,52 @@
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>rollcall</title>
|
||||
<title>Rollcall</title>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
</head>
|
||||
<body>
|
||||
<header>
|
||||
<h1>rollcall <span class="sub">hardware commissioning</span></h1>
|
||||
<div class="hdr-right">
|
||||
<span id="status" class="muted"></span>
|
||||
<button id="refresh" class="btn">Refresh</button>
|
||||
<button id="logout" class="btn ghost" title="forget token">Token</button>
|
||||
</div>
|
||||
</header>
|
||||
<div class="shell">
|
||||
<nav class="side">
|
||||
<div class="brand">Rollcall<span class="muted">bare-metal fleet</span></div>
|
||||
<a href="#/overview" data-nav="overview">Overview</a>
|
||||
<a href="#/machines" data-nav="machines">Machines <span class="count" id="n-machines"></span></a>
|
||||
<a href="#/racks" data-nav="racks">Racks <span class="count" id="n-racks"></span></a>
|
||||
<a href="#/network" data-nav="network">Network <span class="count" id="n-net"></span></a>
|
||||
<a href="#/jobs" data-nav="jobs">Jobs <span class="count" id="n-jobs"></span></a>
|
||||
<a href="#/tags" data-nav="tags">Tags</a>
|
||||
<a href="#/profiles" data-nav="profiles">Profiles</a>
|
||||
<a href="#/images" data-nav="images">OS images</a>
|
||||
<div class="side-foot">
|
||||
<span id="status" class="muted"></span>
|
||||
<button id="logout" class="btn ghost" title="forget the admin token">Token</button>
|
||||
</div>
|
||||
</nav>
|
||||
|
||||
<main id="main"></main>
|
||||
</div>
|
||||
|
||||
<dialog id="login">
|
||||
<form method="dialog">
|
||||
<h2>Admin token</h2>
|
||||
<p class="muted">The <code>admin_token</code> from <code>/etc/rollcall/config.json</code>. Stored only in this browser.</p>
|
||||
<p class="muted">The <code>admin_token</code> from the server's <code>config.json</code>. Stored only in this browser. Or use an admin client certificate (<code>rollcall admin-cert request NAME</code>) and the browser signs you in by itself.</p>
|
||||
<p id="http-warn" class="warn" hidden>This page came over plain HTTP: the token crosses the network unencrypted. Use <a id="https-link" href="#">HTTPS</a> after trusting the installation's <a href="/ca.pem">CA certificate</a> <span id="ca-fp" class="mono muted"></span>.</p>
|
||||
<input id="token" type="password" autocomplete="off" placeholder="token" required>
|
||||
<menu><button class="btn primary" value="ok">Sign in</button></menu>
|
||||
</form>
|
||||
</dialog>
|
||||
|
||||
<main>
|
||||
<section id="list">
|
||||
<div class="toolbar">
|
||||
<input id="filter" type="search" placeholder="filter: id, model, serial, mac, state…">
|
||||
<span id="count" class="muted"></span>
|
||||
</div>
|
||||
<table id="machines">
|
||||
<thead><tr>
|
||||
<th>ID</th><th>State</th><th>Node</th><th>Profile / OS</th><th>Model</th><th>Serial</th><th class="num">CPU</th><th class="num">RAM</th>
|
||||
<th class="num">Disks</th><th class="num">Storage</th><th class="num">NIC</th><th class="num">IB</th><th class="num">GPU</th>
|
||||
<th>BMC</th><th>Last seen</th>
|
||||
</tr></thead>
|
||||
<tbody></tbody>
|
||||
</table>
|
||||
<p id="empty" class="muted" hidden>No machines yet. PXE-boot an unknown server: it commissions itself and appears here.</p>
|
||||
<!-- generic action dialog: filled by JS -->
|
||||
<dialog id="dlg">
|
||||
<form method="dialog" id="dlg-form">
|
||||
<h2 id="dlg-title"></h2>
|
||||
<div id="dlg-body" class="kv form"></div>
|
||||
<p id="dlg-note" class="muted"></p>
|
||||
<menu><button class="btn ghost" value="cancel" type="button" id="dlg-cancel">Cancel</button><button class="btn primary" value="ok" id="dlg-ok">OK</button></menu>
|
||||
</form>
|
||||
</dialog>
|
||||
|
||||
<details id="profiles-box" style="margin-top:18px">
|
||||
<summary>Provisioning: OS images (deployment), profiles & root images (stateless)</summary>
|
||||
<div class="grid" style="margin-top:10px">
|
||||
<div class="card wide"><h3>OS images for deployment <span class="muted" style="text-transform:none">· refreshed nightly by <code>rollcall os sync</code></span></h3><div id="osimages"></div></div>
|
||||
<div class="card"><h3>Profiles</h3><div id="profiles"></div></div>
|
||||
<div class="card"><h3>Stateless root images</h3><div id="images"></div></div>
|
||||
<div class="card wide"><h3>Add / update profile</h3>
|
||||
<form id="profile-form" class="kv" style="grid-template-columns:max-content 1fr;align-items:center">
|
||||
<label>Name</label><input name="name" required placeholder="compute">
|
||||
<label>Image</label><input name="image" required placeholder="compute-debian13">
|
||||
<label>Overlays</label><input name="overlays" placeholder="compute">
|
||||
<label>Name format</label><input name="name_format" value="node%03d">
|
||||
<label>Kernel append</label><input name="append" placeholder="extra kernel parameters">
|
||||
<label>Vars (JSON)</label><input name="vars" value="{}">
|
||||
<span></span><span><button class="btn primary" type="submit">Save profile</button> <span id="profile-out" class="muted"></span></span>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</details>
|
||||
</section>
|
||||
|
||||
<section id="detail" hidden>
|
||||
<div class="detail-head">
|
||||
<button id="back" class="btn ghost">← Machines</button>
|
||||
<h2 id="d-title"></h2>
|
||||
<span id="d-state" class="pill"></span>
|
||||
</div>
|
||||
<div class="actions">
|
||||
<span class="muted">Power:</span>
|
||||
<button class="btn" data-power="status">Status</button>
|
||||
<button class="btn" data-power="on">On</button>
|
||||
<button class="btn" data-power="off">Off</button>
|
||||
<button class="btn" data-power="soft">Soft off</button>
|
||||
<button class="btn" data-power="cycle">Cycle</button>
|
||||
<button class="btn" data-power="reset">Reset</button>
|
||||
<span class="sep"></span>
|
||||
<span class="muted">Boot:</span>
|
||||
<button class="btn" data-bootdev="pxe">PXE</button>
|
||||
<button class="btn" data-bootdev="disk">Disk</button>
|
||||
<span class="sep"></span>
|
||||
<span class="muted">Stateless:</span>
|
||||
<select id="assign-profile"><option value="">profile…</option></select>
|
||||
<input id="assign-name" placeholder="node name (auto)" style="width:150px;padding:5px 8px;border:1px solid var(--line);border-radius:6px;background:var(--panel);color:var(--fg)">
|
||||
<button id="assign" class="btn">Assign</button>
|
||||
<button id="assign-boot" class="btn" title="assign, bootdev pxe, power-cycle">Assign + boot</button>
|
||||
<button id="unassign" class="btn warn">Unassign</button>
|
||||
<span class="sep"></span>
|
||||
<span class="muted">Deploy:</span>
|
||||
<select id="deploy-os"><option value="">OS image…</option></select>
|
||||
<select id="deploy-disk"><option value="">disk: auto (smallest that fits)</option></select>
|
||||
<input id="deploy-host" type="text" placeholder="hostname (auto)" style="width:140px">
|
||||
<label><input id="deploy-upgrade" type="checkbox"> upgrade on first boot</label>
|
||||
<button id="deploy" class="btn primary" title="write the OS image to the disk, seed cloud-init, bootdev pxe, power-cycle">Deploy + boot</button>
|
||||
<button id="release" class="btn warn" title="wipe the disk signatures and return the machine to ready">Release</button>
|
||||
<button id="job-cancel" class="btn" hidden>Cancel job</button>
|
||||
<span class="sep"></span>
|
||||
<button id="recommission" class="btn warn">Re-commission</button>
|
||||
<button id="recommission-now" class="btn warn" title="remove per-MAC PXE config, set bootdev pxe and power-cycle via BMC">Re-commission now</button>
|
||||
<button id="delete" class="btn danger">Delete</button>
|
||||
<span id="action-out" class="mono"></span>
|
||||
</div>
|
||||
<div id="d-body" class="grid"></div>
|
||||
<details><summary>Raw JSON</summary><pre id="d-json" class="mono"></pre></details>
|
||||
</section>
|
||||
</main>
|
||||
<div id="toast" hidden></div>
|
||||
<script src="topo.js"></script>
|
||||
<script src="app.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
+163
-45
@@ -1,60 +1,178 @@
|
||||
:root {
|
||||
--bg: #f6f7f9; --panel: #ffffff; --fg: #1d2129; --muted: #6b7280; --line: #e3e6ea;
|
||||
--bg: #f4f5f7; --panel: #ffffff; --fg: #1c2027; --muted: #687078; --line: #e1e4e8; --line2: #ccd1d6;
|
||||
--accent: #2563eb; --accent-fg: #ffffff; --warn: #b45309; --danger: #b91c1c; --ok: #15803d;
|
||||
--pill: #e8eefc; --hover: #f0f4ff; --mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
--pill: #eef1f5; --hover: #f0f4ff; --sel: #e8efff; --side: #1b1f27; --side-fg: #c9ced6; --side-act: #2b3242;
|
||||
--mono: ui-monospace, SFMono-Regular, Menlo, Consolas, monospace;
|
||||
}
|
||||
@media (prefers-color-scheme: dark) {
|
||||
:root:not([data-theme="light"]) {
|
||||
--bg: #0f1216; --panel: #171b21; --fg: #e6e8eb; --muted: #9aa3ad; --line: #2a3139;
|
||||
--accent: #3b82f6; --pill: #1e2a44; --hover: #1b2230; --warn: #f59e0b; --danger: #ef4444; --ok: #22c55e;
|
||||
--bg: #0f1216; --panel: #171b21; --fg: #e6e8eb; --muted: #9aa3ad; --line: #2a3139; --line2: #3a424c;
|
||||
--accent: #3b82f6; --pill: #1f2733; --hover: #1b2230; --sel: #1e2a44; --warn: #f59e0b; --danger: #ef4444; --ok: #22c55e;
|
||||
--side: #0b0e12; --side-fg: #aeb6c0; --side-act: #1c2430;
|
||||
}
|
||||
}
|
||||
:root[data-theme="dark"] {
|
||||
--bg: #0f1216; --panel: #171b21; --fg: #e6e8eb; --muted: #9aa3ad; --line: #2a3139;
|
||||
--accent: #3b82f6; --pill: #1e2a44; --hover: #1b2230; --warn: #f59e0b; --danger: #ef4444; --ok: #22c55e;
|
||||
--bg: #0f1216; --panel: #171b21; --fg: #e6e8eb; --muted: #9aa3ad; --line: #2a3139; --line2: #3a424c;
|
||||
--accent: #3b82f6; --pill: #1f2733; --hover: #1b2230; --sel: #1e2a44; --warn: #f59e0b; --danger: #ef4444; --ok: #22c55e;
|
||||
--side: #0b0e12; --side-fg: #aeb6c0; --side-act: #1c2430;
|
||||
}
|
||||
* { box-sizing: border-box; }
|
||||
body { margin: 0; background: var(--bg); color: var(--fg); font: 14px/1.45 system-ui, -apple-system, Segoe UI, Roboto, sans-serif; }
|
||||
header { display: flex; align-items: center; justify-content: space-between; padding: 12px 16px; border-bottom: 1px solid var(--line); background: var(--panel); position: sticky; top: 0; z-index: 2; gap: 12px; flex-wrap: wrap; }
|
||||
h1 { font-size: 18px; margin: 0; } h1 .sub { font-weight: 400; color: var(--muted); font-size: 13px; margin-left: 8px; }
|
||||
h2 { font-size: 16px; margin: 0; }
|
||||
main { padding: 16px; max-width: 1400px; margin: 0 auto; }
|
||||
.muted { color: var(--muted); } .mono { font-family: var(--mono); font-size: 12px; white-space: pre-wrap; }
|
||||
.btn { border: 1px solid var(--line); background: var(--panel); color: var(--fg); padding: 6px 10px; border-radius: 6px; cursor: pointer; font: inherit; font-size: 13px; }
|
||||
html, body { height: 100%; }
|
||||
body { margin: 0; background: var(--bg); color: var(--fg); font: 13px/1.45 system-ui, -apple-system, Segoe UI, Roboto, sans-serif; }
|
||||
a { color: var(--accent); }
|
||||
.muted { color: var(--muted); } .mono { font-family: var(--mono); font-size: 12px; }
|
||||
.ok { color: var(--ok); } .bad { color: var(--danger); } .warn { color: var(--warn); }
|
||||
.num { text-align: right; font-variant-numeric: tabular-nums; }
|
||||
|
||||
/* layout */
|
||||
.shell { display: grid; grid-template-columns: 200px 1fr; min-height: 100vh; }
|
||||
.side { background: var(--side); color: var(--side-fg); padding: 14px 10px; display: flex; flex-direction: column; gap: 2px; position: sticky; top: 0; height: 100vh; }
|
||||
.brand { font-weight: 700; font-size: 16px; color: #fff; padding: 4px 10px 14px; }
|
||||
.brand .muted { display: block; font-weight: 400; font-size: 11px; color: var(--side-fg); }
|
||||
.side a { color: var(--side-fg); text-decoration: none; padding: 7px 10px; border-radius: 6px; display: flex; justify-content: space-between; }
|
||||
.side a:hover { background: var(--side-act); } .side a.active { background: var(--side-act); color: #fff; }
|
||||
.side .count { font-size: 11px; opacity: .8; }
|
||||
.side-foot { margin-top: auto; display: flex; flex-direction: column; gap: 6px; padding: 10px; font-size: 11px; }
|
||||
main { padding: 18px 22px; max-width: 1500px; width: 100%; }
|
||||
.page-head { display: flex; align-items: center; gap: 12px; margin-bottom: 14px; flex-wrap: wrap; }
|
||||
.page-head h1 { font-size: 18px; margin: 0; } .page-head .sub { color: var(--muted); }
|
||||
.page-head .right { margin-left: auto; display: flex; gap: 6px; align-items: center; flex-wrap: wrap; }
|
||||
|
||||
/* controls */
|
||||
.btn { border: 1px solid var(--line2); background: var(--panel); color: var(--fg); padding: 5px 10px; border-radius: 6px; cursor: pointer; font: inherit; font-size: 12.5px; white-space: nowrap; }
|
||||
.btn:hover { background: var(--hover); } .btn.primary { background: var(--accent); color: var(--accent-fg); border-color: var(--accent); }
|
||||
.btn.ghost { border-color: transparent; } .btn.warn { color: var(--warn); } .btn.danger { color: var(--danger); }
|
||||
.btn:disabled { opacity: .5; cursor: progress; }
|
||||
.hdr-right { display: flex; gap: 8px; align-items: center; }
|
||||
.toolbar { display: flex; gap: 12px; align-items: center; margin-bottom: 10px; }
|
||||
input[type=search], input[type=password] { padding: 7px 10px; border: 1px solid var(--line); border-radius: 6px; background: var(--panel); color: var(--fg); font: inherit; min-width: 280px; }
|
||||
.btn.ghost { border-color: transparent; background: transparent; color: inherit; } .btn.warn { color: var(--warn); } .btn.danger { color: var(--danger); }
|
||||
.btn:disabled { opacity: .5; cursor: progress; } .btn.sm { padding: 2px 7px; font-size: 11.5px; }
|
||||
input, select, textarea { padding: 6px 9px; border: 1px solid var(--line2); border-radius: 6px; background: var(--panel); color: var(--fg); font: inherit; }
|
||||
textarea { font-family: var(--mono); font-size: 12px; width: 100%; }
|
||||
input[type=search] { min-width: 260px; }
|
||||
.chips { display: flex; gap: 6px; flex-wrap: wrap; align-items: center; }
|
||||
.chip { border: 1px solid var(--line2); border-radius: 999px; padding: 2px 10px; cursor: pointer; background: var(--panel); font-size: 12px; }
|
||||
.chip:hover { background: var(--hover); } .chip.on { background: var(--accent); border-color: var(--accent); color: var(--accent-fg); }
|
||||
.chip .n { opacity: .7; margin-left: 4px; }
|
||||
.menu { position: relative; display: inline-block; }
|
||||
.menu > .list { display: none; position: absolute; top: 100%; left: 0; z-index: 5; background: var(--panel); border: 1px solid var(--line2); border-radius: 8px; padding: 4px; min-width: 190px; box-shadow: 0 8px 24px rgba(0,0,0,.18); }
|
||||
.menu.open > .list { display: block; }
|
||||
.menu .list button { display: block; width: 100%; text-align: left; border: none; background: none; padding: 7px 10px; border-radius: 5px; color: var(--fg); font: inherit; cursor: pointer; }
|
||||
.menu .list button:hover { background: var(--hover); } .menu .list button.danger { color: var(--danger); } .menu .list hr { border: 0; border-top: 1px solid var(--line); margin: 4px 0; }
|
||||
|
||||
/* tables */
|
||||
.tw { overflow-x: auto; border-radius: 8px; }
|
||||
.tw table { min-width: 1100px; }
|
||||
table { width: 100%; border-collapse: collapse; background: var(--panel); border: 1px solid var(--line); border-radius: 8px; overflow: hidden; }
|
||||
th, td { padding: 8px 10px; text-align: left; border-bottom: 1px solid var(--line); white-space: nowrap; }
|
||||
th { font-weight: 600; color: var(--muted); font-size: 12px; text-transform: uppercase; letter-spacing: .03em; }
|
||||
th.num, td.num { text-align: right; font-variant-numeric: tabular-nums; }
|
||||
tbody tr { cursor: pointer; } tbody tr:hover { background: var(--hover); }
|
||||
.pill { display: inline-block; padding: 2px 8px; border-radius: 999px; background: var(--pill); font-size: 12px; }
|
||||
.pill.ready, .pill.booted, .pill.deployed { color: var(--ok); } .pill.recommission, .pill.new, .pill.stateless, .pill.deploying, .pill.releasing { color: var(--warn); }
|
||||
.pill.deploy-failed, .pill.release-failed { color: var(--danger); }
|
||||
.bar { position: relative; height: 10px; background: var(--pill); border-radius: 999px; overflow: hidden; margin: 6px 0; }
|
||||
.bar > span { position: absolute; inset: 0 auto 0 0; background: var(--accent); transition: width .5s; }
|
||||
.actions select, .actions input[type=text] { padding: 5px 8px; border: 1px solid var(--line); border-radius: 6px; background: var(--panel); color: var(--fg); font: inherit; }
|
||||
.actions label { display: inline-flex; gap: 4px; align-items: center; color: var(--muted); font-size: 13px; }
|
||||
.card form input { padding: 5px 8px; border: 1px solid var(--line); border-radius: 6px; background: var(--panel); color: var(--fg); font: inherit; min-width: 0; width: 100%; }
|
||||
#assign-profile { padding: 5px 8px; border: 1px solid var(--line); border-radius: 6px; background: var(--panel); color: var(--fg); font: inherit; }
|
||||
.ok { color: var(--ok); } .bad { color: var(--danger); }
|
||||
.detail-head { display: flex; gap: 12px; align-items: center; margin-bottom: 10px; }
|
||||
.actions { display: flex; gap: 6px; align-items: center; flex-wrap: wrap; padding: 10px; background: var(--panel); border: 1px solid var(--line); border-radius: 8px; margin-bottom: 14px; }
|
||||
.sep { width: 1px; height: 20px; background: var(--line); margin: 0 6px; }
|
||||
#action-out { margin-left: auto; color: var(--muted); max-width: 50%; }
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(420px, 1fr)); gap: 12px; }
|
||||
.card { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 12px; overflow: auto; }
|
||||
th, td { padding: 7px 10px; text-align: left; border-bottom: 1px solid var(--line); white-space: nowrap; vertical-align: middle; }
|
||||
th { font-weight: 600; color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .04em; position: sticky; top: 0; background: var(--panel); z-index: 1; }
|
||||
th.sort { cursor: pointer; } th.sort:hover { color: var(--fg); }
|
||||
tbody tr.row { cursor: pointer; } tbody tr.row:hover { background: var(--hover); } tbody tr.selected { background: var(--sel); }
|
||||
td.check, th.check { width: 28px; padding-right: 0; }
|
||||
.pill { display: inline-block; padding: 1px 8px; border-radius: 999px; background: var(--pill); font-size: 11.5px; line-height: 18px; }
|
||||
.pill.ready, .pill.booted, .pill.deployed, .pill.done { color: var(--ok); }
|
||||
.pill.recommission, .pill.new, .pill.stateless, .pill.deploying, .pill.releasing, .pill.running, .pill.pending { color: var(--warn); }
|
||||
.pill.deploy-failed, .pill.release-failed, .pill.failed { color: var(--danger); }
|
||||
.pill.tag { background: var(--hover); border: 1px solid var(--line); font-family: var(--mono); font-size: 11px; }
|
||||
.pill.tag.auto { border-style: dashed; color: var(--muted); } .pill.tag a { color: var(--danger); text-decoration: none; margin-left: 2px; }
|
||||
.dot { display: inline-block; width: 8px; height: 8px; border-radius: 50%; background: var(--line2); margin-right: 5px; }
|
||||
.dot.ok { background: var(--ok); } .dot.bad { background: var(--danger); } .dot.warn { background: var(--warn); }
|
||||
.hw { color: var(--muted); font-size: 12px; } .hw b { color: var(--fg); font-weight: 500; }
|
||||
.pager { display: flex; gap: 8px; align-items: center; justify-content: flex-end; margin-top: 8px; color: var(--muted); }
|
||||
.bulk { display: flex; gap: 6px; align-items: center; padding: 8px 10px; background: var(--sel); border: 1px solid var(--line2); border-radius: 8px; margin-bottom: 10px; flex-wrap: wrap; }
|
||||
.empty { padding: 40px; text-align: center; color: var(--muted); background: var(--panel); border: 1px dashed var(--line2); border-radius: 8px; }
|
||||
|
||||
/* cards / grid */
|
||||
.grid { display: grid; grid-template-columns: repeat(auto-fit, minmax(380px, 1fr)); gap: 12px; }
|
||||
.card { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 12px 14px; overflow: auto; }
|
||||
.card.wide { grid-column: 1 / -1; }
|
||||
.card h3 { margin: 0 0 8px; font-size: 13px; text-transform: uppercase; letter-spacing: .03em; color: var(--muted); }
|
||||
.card h3 { margin: 0 0 8px; font-size: 11.5px; text-transform: uppercase; letter-spacing: .04em; color: var(--muted); }
|
||||
.card table { border: none; } .card th, .card td { padding: 4px 8px; font-size: 12.5px; } .card th { position: static; }
|
||||
.kv { display: grid; grid-template-columns: max-content 1fr; gap: 3px 14px; }
|
||||
.kv dt { color: var(--muted); } .kv dd { margin: 0; word-break: break-all; }
|
||||
.card table { border: none; } .card th, .card td { padding: 4px 8px; font-size: 13px; }
|
||||
details { margin-top: 14px; } summary { cursor: pointer; color: var(--muted); }
|
||||
dialog { border: 1px solid var(--line); border-radius: 10px; background: var(--panel); color: var(--fg); padding: 20px; min-width: 360px; }
|
||||
dialog::backdrop { background: rgba(0,0,0,.45); } dialog menu { padding: 0; margin: 12px 0 0; display: flex; justify-content: flex-end; }
|
||||
dialog input { width: 100%; margin-top: 8px; }
|
||||
@media (max-width: 720px) { main { padding: 16px 16px; } .grid { grid-template-columns: 1fr; } input[type=search] { min-width: 0; width: 100%; } table { display: block; overflow-x: auto; } }
|
||||
.kv.form { gap: 8px 12px; align-items: center; } .kv.form label { color: var(--muted); } .kv.form input, .kv.form select { width: 100%; }
|
||||
.stats { display: grid; grid-template-columns: repeat(auto-fit, minmax(150px, 1fr)); gap: 10px; margin-bottom: 14px; }
|
||||
.stat { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 12px 14px; cursor: pointer; }
|
||||
.stat:hover { background: var(--hover); } .stat .v { font-size: 24px; font-weight: 600; line-height: 1.1; } .stat .l { color: var(--muted); font-size: 12px; }
|
||||
.bar { position: relative; height: 8px; background: var(--pill); border-radius: 999px; overflow: hidden; min-width: 120px; }
|
||||
.bar > span { position: absolute; inset: 0 auto 0 0; background: var(--accent); transition: width .5s; }
|
||||
|
||||
/* racks */
|
||||
.racks { display: grid; grid-template-columns: repeat(auto-fill, minmax(260px, 1fr)); gap: 12px; }
|
||||
.rack { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 10px 12px; }
|
||||
.rack h4 { margin: 0 0 6px; font-size: 13px; display: flex; justify-content: space-between; } .rack h4 .muted { font-weight: 400; }
|
||||
.elev { display: grid; grid-template-columns: 26px 1fr; gap: 0; border: 1px solid var(--line2); border-radius: 4px; overflow: hidden; font-size: 11px; }
|
||||
.elev .u { color: var(--muted); text-align: right; padding: 0 4px; border-bottom: 1px solid var(--line); border-right: 1px solid var(--line); line-height: 16px; font-variant-numeric: tabular-nums; }
|
||||
.elev .slot { border-bottom: 1px solid var(--line); line-height: 16px; padding: 0 6px; white-space: nowrap; overflow: hidden; text-overflow: ellipsis; cursor: default; }
|
||||
.elev .slot.m { cursor: pointer; background: var(--sel); border-left: 3px solid var(--accent); }
|
||||
.elev .slot.m.deployed, .elev .slot.m.booted, .elev .slot.m.ready { border-left-color: var(--ok); }
|
||||
.elev .slot.m.deploying, .elev .slot.m.releasing, .elev .slot.m.stateless { border-left-color: var(--warn); }
|
||||
.elev .slot.m.deploy-failed, .elev .slot.m.release-failed { border-left-color: var(--danger); }
|
||||
.elev .slot.m:hover { filter: brightness(1.08); }
|
||||
.tree { display: flex; gap: 10px; flex-wrap: wrap; margin-bottom: 12px; }
|
||||
.tree .chip.on { background: var(--accent); color: var(--accent-fg); }
|
||||
|
||||
/* detail */
|
||||
.d-head { display: flex; align-items: center; gap: 12px; flex-wrap: wrap; margin-bottom: 10px; }
|
||||
.d-head h1 { margin: 0; font-size: 20px; } .d-head .id { color: var(--muted); font-family: var(--mono); font-size: 12px; }
|
||||
.d-meta { color: var(--muted); margin-bottom: 10px; display: flex; gap: 14px; flex-wrap: wrap; }
|
||||
.tabs { display: flex; gap: 2px; border-bottom: 1px solid var(--line); margin: 12px 0; }
|
||||
.tabs a { padding: 7px 12px; color: var(--muted); text-decoration: none; border-bottom: 2px solid transparent; margin-bottom: -1px; }
|
||||
.tabs a.active { color: var(--fg); border-color: var(--accent); }
|
||||
pre.json { background: var(--panel); border: 1px solid var(--line); border-radius: 8px; padding: 12px; overflow: auto; max-height: 70vh; font-size: 11.5px; }
|
||||
.act-out { color: var(--muted); font-family: var(--mono); font-size: 12px; white-space: pre-wrap; margin-top: 6px; }
|
||||
|
||||
/* dialogs */
|
||||
dialog { border: 1px solid var(--line2); border-radius: 10px; background: var(--panel); color: var(--fg); padding: 20px; min-width: 420px; max-width: 640px; }
|
||||
dialog::backdrop { background: rgba(0,0,0,.45); } dialog h2 { margin: 0 0 10px; font-size: 16px; }
|
||||
dialog menu { padding: 0; margin: 14px 0 0; display: flex; justify-content: flex-end; gap: 6px; }
|
||||
dialog input { width: 100%; }
|
||||
#toast { position: fixed; right: 18px; bottom: 18px; background: var(--side); color: #fff; padding: 10px 14px; border-radius: 8px; max-width: 480px; font-size: 12.5px; white-space: pre-wrap; z-index: 20; box-shadow: 0 8px 24px rgba(0,0,0,.3); }
|
||||
#toast.bad { background: var(--danger); }
|
||||
|
||||
@media (max-width: 860px) {
|
||||
.shell { grid-template-columns: 1fr; }
|
||||
.side { position: static; height: auto; flex-direction: row; flex-wrap: wrap; align-items: center; }
|
||||
.brand { padding: 4px 10px; } .brand .muted { display: none; } .side-foot { margin-left: auto; flex-direction: row; }
|
||||
main { padding: 14px 16px; } .grid { grid-template-columns: 1fr; } input[type=search] { min-width: 0; width: 100%; }
|
||||
table { display: block; overflow-x: auto; } dialog { min-width: 0; width: calc(100vw - 32px); }
|
||||
}
|
||||
|
||||
/* topology */
|
||||
.topo-wrap { overflow-x: auto; }
|
||||
svg.topo { display: block; font: 12px system-ui, -apple-system, Segoe UI, Roboto, sans-serif; }
|
||||
svg.topo.compact { font-size: 11px; }
|
||||
svg.topo text { fill: var(--fg); }
|
||||
svg.topo .t1 { font-weight: 600; } svg.topo .t2, svg.topo .t3, svg.topo .n2 { fill: var(--muted); font-size: 11px; }
|
||||
svg.topo.compact .n2, svg.topo.compact .t3 { font-size: 10px; }
|
||||
svg.topo .seg line { stroke: var(--line2); stroke-width: 3; stroke-linecap: round; }
|
||||
svg.topo .seg.dhcp line { stroke: var(--accent); stroke-width: 4; }
|
||||
svg.topo .seg.p-storage line, svg.topo .seg.p-ib line { stroke-dasharray: 8 5; }
|
||||
svg.topo .seg { cursor: pointer; } svg.topo .seg:hover line { stroke-width: 6; }
|
||||
svg.topo .wire { stroke: var(--line2); stroke-width: 1.5; fill: none; }
|
||||
svg.topo .wire.ctl { stroke: var(--accent); stroke-width: 1.5; }
|
||||
svg.topo .wire.ctl.down { stroke: var(--danger); stroke-dasharray: 5 4; }
|
||||
svg.topo .wire.ext { stroke: var(--muted); stroke-dasharray: 3 3; }
|
||||
svg.topo .tap { fill: var(--panel); stroke: var(--fg); stroke-width: 1.5; }
|
||||
svg.topo .tap.rack { fill: var(--accent); stroke: var(--accent); } svg.topo .wire.rackdrop { stroke: var(--accent); stroke-opacity: .55; stroke-width: 2; }
|
||||
svg.topo .node rect { fill: var(--panel); stroke: var(--line2); stroke-width: 1.2; }
|
||||
svg.topo .node .n1 { font-weight: 600; }
|
||||
svg.topo .region rect { stroke: var(--accent); stroke-width: 2; fill: var(--sel); }
|
||||
svg.topo .ipam rect { stroke-dasharray: 4 3; } svg.topo .ipam.bad rect { stroke: var(--danger); }
|
||||
svg.topo .rack.ok .led { fill: var(--ok); } svg.topo .rack.bad .led { fill: var(--danger); } svg.topo .rack.bad rect { stroke: var(--danger); }
|
||||
svg.topo .mach { cursor: pointer; } svg.topo .mach:hover rect { fill: var(--hover); }
|
||||
svg.topo .mach .stripe { stroke: none; fill: var(--line2); }
|
||||
svg.topo .mach.ok .stripe { fill: var(--ok); } svg.topo .mach.warn .stripe { fill: var(--warn); } svg.topo .mach.bad .stripe { fill: var(--danger); }
|
||||
svg.topo .mach.group rect { stroke-width: 1.2; } svg.topo .mach.group .stack { fill: none; stroke: var(--line2); }
|
||||
svg.topo .mach.group.ok rect:first-child { stroke: var(--ok); } svg.topo .mach.group.warn rect:first-child { stroke: var(--warn); } svg.topo .mach.group.bad rect:first-child { stroke: var(--danger); }
|
||||
.legend { display: flex; gap: 14px; flex-wrap: wrap; color: var(--muted); font-size: 11.5px; margin-top: 8px; align-items: center; }
|
||||
.legend i { display: inline-block; width: 22px; height: 0; border-top: 4px solid var(--accent); vertical-align: middle; margin-right: 5px; border-radius: 2px; }
|
||||
.legend i.drop { border-top: 2px solid var(--accent); opacity: .55; } .legend b.sq { display: inline-block; width: 8px; height: 8px; background: var(--accent); margin-right: 5px; } .legend b.ci { display: inline-block; width: 7px; height: 7px; border: 1.5px solid var(--fg); border-radius: 50%; margin-right: 5px; }
|
||||
.legend i.plain { border-top: 3px solid var(--line2); } .legend i.dash { border-top: 3px dashed var(--line2); } .legend i.ctl { border-top: 2px solid var(--accent); } .legend i.down { border-top: 2px dashed var(--danger); }
|
||||
|
||||
/* overview */
|
||||
.ov { display: grid; grid-template-columns: repeat(12, 1fr); gap: 12px; }
|
||||
.ov > .card { grid-column: span 4; } .ov > .card.w6 { grid-column: span 6; } .ov > .card.w8 { grid-column: span 8; } .ov > .card.wide { grid-column: 1 / -1; }
|
||||
.big { display: grid; grid-template-columns: repeat(auto-fit, minmax(90px, 1fr)); gap: 10px; }
|
||||
.big div b { display: block; font-size: 20px; font-weight: 600; line-height: 1.15; font-variant-numeric: tabular-nums; } .big div span { color: var(--muted); font-size: 11.5px; }
|
||||
.hbar { display: grid; grid-template-columns: minmax(90px, max-content) 1fr 64px; gap: 4px 10px; align-items: center; font-size: 12.5px; }
|
||||
.hbar .bar { min-width: 0; } .hbar .n { text-align: right; color: var(--muted); font-variant-numeric: tabular-nums; }
|
||||
.statebar { display: flex; height: 12px; border-radius: 999px; overflow: hidden; background: var(--pill); margin: 4px 0 10px; }
|
||||
.statebar span { height: 100%; } .statebar .ok { background: var(--ok); } .statebar .warn { background: var(--warn); } .statebar .bad { background: var(--danger); } .statebar .idle { background: var(--line2); }
|
||||
@media (max-width: 1100px) { .ov > .card, .ov > .card.w6, .ov > .card.w8 { grid-column: 1 / -1; } }
|
||||
@@ -0,0 +1,194 @@
|
||||
'use strict';
|
||||
// Network topology drawn from the model: region (and the external IPAM) on
|
||||
// top, rack controllers below, one bus per VLAN with its subnets, machines at
|
||||
// the bottom wired to every bus they have an address on. Plain SVG; colours
|
||||
// come from the page's CSS variables so light and dark both work.
|
||||
|
||||
// ---------------------------------------------------------------- addresses
|
||||
function parseIP(s) {
|
||||
s = String(s || '').split('/')[0].trim();
|
||||
if (!s) return null;
|
||||
if (!s.includes(':')) {
|
||||
const p = s.split('.').map(Number);
|
||||
if (p.length !== 4 || p.some(x => !(x >= 0 && x <= 255))) return null;
|
||||
return { v: 4, n: BigInt(((p[0] << 24) >>> 0) + (p[1] << 16) + (p[2] << 8) + p[3]) };
|
||||
}
|
||||
const [head, tail] = s.split('::');
|
||||
const h = head ? head.split(':') : [], t = tail !== undefined ? (tail ? tail.split(':') : []) : null;
|
||||
const groups = t === null ? h : [...h, ...Array(8 - h.length - t.length).fill('0'), ...t];
|
||||
if (groups.length !== 8) return null;
|
||||
let n = 0n;
|
||||
for (const g of groups) n = (n << 16n) + BigInt(parseInt(g || '0', 16));
|
||||
return { v: 6, n };
|
||||
}
|
||||
function inCidr(ip, cidr) {
|
||||
const [a, b] = String(cidr).split('/');
|
||||
const x = parseIP(ip), y = parseIP(a);
|
||||
if (!x || !y || x.v !== y.v) return false;
|
||||
const shift = BigInt((x.v === 4 ? 32 : 128) - Number(b));
|
||||
return (x.n >> shift) === (y.n >> shift);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- model
|
||||
// topoModel groups subnets into segments (one per VLAN), attaches racks and machines.
|
||||
function topoModel() {
|
||||
const segs = [], byVlan = {};
|
||||
for (const s of S.subnets) {
|
||||
const key = s.vlan_id ? 'v' + s.vlan_id : 'c' + s.cidr;
|
||||
if (!byVlan[key]) { byVlan[key] = { key, fabric: s.fabric, vid: s.vid, mtu: s.mtu, subnets: [], racks: new Set(), dhcp: false, purpose: s.purpose }; segs.push(byVlan[key]); }
|
||||
const g = byVlan[key];
|
||||
g.subnets.push(s);
|
||||
if (s.dhcp) { g.dhcp = true; if (s.primary_rack) g.racks.add(s.primary_rack); if (s.secondary_rack) g.racks.add(s.secondary_rack); }
|
||||
if (s.purpose !== 'provisioning' && g.purpose === 'provisioning' && !s.dhcp) { /* keep the first */ }
|
||||
}
|
||||
segs.sort((a, b) => (b.dhcp - a.dhcp) || String(a.fabric).localeCompare(String(b.fabric)) || a.vid - b.vid);
|
||||
const segOf = ip => segs.findIndex(g => g.subnets.some(s => inCidr(ip, s.cidr)));
|
||||
// rack controllers: the segments they serve DHCP on, plus those their interfaces sit on
|
||||
const racks = S.controllers.map(c => {
|
||||
const at = new Set();
|
||||
segs.forEach((g, i) => { if (g.racks.has(c.name)) at.add(i); });
|
||||
for (const itf of (c.interfaces || [])) for (const a of (itf.addrs || [])) { const i = segOf(a); if (i >= 0) at.add(i); }
|
||||
return { c, at: [...at].sort((a, b) => a - b) };
|
||||
});
|
||||
// machines: every address Rollcall knows for them
|
||||
const ipsBy = {};
|
||||
for (const x of S.ips) if (x.machine_id) (ipsBy[x.machine_id] = ipsBy[x.machine_id] || []).push(x.ip);
|
||||
const ms = S.machines.map(m => {
|
||||
const at = new Set();
|
||||
for (const ip of [...(ipsBy[m.id] || []), m.mgmt_ip, m.ib_ip]) { if (!ip) continue; const i = segOf(ip); if (i >= 0) at.add(i); }
|
||||
return { m, at: [...at].sort((a, b) => a - b) };
|
||||
});
|
||||
return { segs, racks, ms };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- drawing
|
||||
// "10 failed · 10 deploying · 10 ok": worst first, short names, cut to fit
|
||||
function groupLine(by, max) {
|
||||
const rank = { bad: 0, warn: 1, ok: 2, idle: 3 };
|
||||
const short = s => s.replace('deploy-failed', 'failed').replace('release-failed', 'failed').replace('recommission', 'commiss.');
|
||||
const parts = Object.entries(by).sort((a, b) => rank[stateClass(a[0])] - rank[stateClass(b[0])] || b[1] - a[1]).map(([s, n]) => `${n} ${short(s)}`);
|
||||
let out = '';
|
||||
for (const p of parts) { const next = out ? out + ' · ' + p : p; if (next.length > max) return out ? out + ' …' : p.slice(0, max); out = next; }
|
||||
return out;
|
||||
}
|
||||
const stateClass = st => /failed/.test(st) ? 'bad' : /deploying|releasing|recommission|new|commission/.test(st) ? 'warn' : /deployed|booted|ready|stateless/.test(st) ? 'ok' : 'idle';
|
||||
|
||||
// topoSVG({compact}) -> svg markup. compact: overview card (fewer labels, collapsed groups).
|
||||
function topoSVG(opt = {}) {
|
||||
const { segs, racks, ms } = topoModel();
|
||||
if (!segs.length && !racks.length) return '<div class="empty">No subnets or rack controllers yet.</div>';
|
||||
const compact = !!opt.compact;
|
||||
const BW = compact ? 112 : 156, BH = compact ? 40 : 50, GAP = compact ? 12 : 16, M = 24;
|
||||
const collapseAt = compact ? 4 : 8;
|
||||
|
||||
// machine items: one box per machine, or one per group of machines with the same wiring
|
||||
const groups = {};
|
||||
for (const x of ms) { const k = (x.m.rack_controller || '') + '|' + x.at.join(','); (groups[k] = groups[k] || []).push(x); }
|
||||
const rackIdx = name => { const i = racks.findIndex(r => r.c.name === name); return i < 0 ? 999 : i; };
|
||||
let items = [];
|
||||
for (const [k, xs] of Object.entries(groups)) {
|
||||
const rack = k.split('|')[0];
|
||||
if (xs.length >= collapseAt) items.push({ group: xs, at: xs[0].at, rack });
|
||||
else for (const x of xs) items.push({ m: x.m, at: x.at, rack });
|
||||
}
|
||||
items.sort((a, b) => rackIdx(a.rack) - rackIdx(b.rack) || (a.at[0] ?? 99) - (b.at[0] ?? 99) || String(a.m ? nameOf(a.m) : '').localeCompare(String(b.m ? nameOf(b.m) : '')));
|
||||
|
||||
const minW = compact ? 640 : 980;
|
||||
let W = Math.max(minW, M * 2 + items.length * (BW + GAP) - GAP + 260);
|
||||
// x positions of machine items, then racks centred over their machines
|
||||
const itemX = items.map((_, i) => M + 260 + i * (BW + GAP));
|
||||
const RW = compact ? 120 : 150;
|
||||
const rackX = racks.map((r, ri) => {
|
||||
const mine = items.map((it, i) => it.rack === r.c.name ? itemX[i] + BW / 2 : null).filter(v => v !== null);
|
||||
return mine.length ? mine.reduce((a, b) => a + b, 0) / mine.length : null;
|
||||
});
|
||||
let free = M + 260;
|
||||
const order = racks.map((_, i) => i).sort((a, b) => (rackX[a] ?? 1e9) - (rackX[b] ?? 1e9));
|
||||
for (const i of order) { let x = rackX[i] ?? free + RW / 2; x = Math.max(x, free + RW / 2); rackX[i] = x; free = x + RW / 2 + 20; }
|
||||
W = Math.max(W, free + M);
|
||||
|
||||
const yRegion = 14, hTop = compact ? 40 : 48;
|
||||
const yRack = yRegion + hTop + (compact ? 34 : 44);
|
||||
const ySeg0 = yRack + hTop + (compact ? 40 : 52), segGap = compact ? 30 : 44;
|
||||
const segY = segs.map((_, i) => ySeg0 + i * segGap);
|
||||
const yM = (segs.length ? segY[segs.length - 1] : ySeg0) + (compact ? 36 : 48);
|
||||
const H = yM + BH + (items.length ? 16 : 0) + 10;
|
||||
const regionX = racks.length ? racks.map((_, i) => rackX[i]).reduce((a, b) => a + b, 0) / racks.length : W / 2;
|
||||
const regW = compact ? 150 : 190, iw = compact ? 150 : 180;
|
||||
const hasIPAM = !!(S.ipam && S.ipam.provider), ix = regionX + regW / 2 + 50 + iw / 2;
|
||||
if (hasIPAM) W = Math.max(W, ix + iw / 2 + M);
|
||||
|
||||
const o = [];
|
||||
o.push(`<svg class="topo${compact ? ' compact' : ''}" viewBox="0 0 ${W} ${H}" width="${W}" height="${H}" style="max-width:100%;height:auto" role="img" aria-label="network topology">`);
|
||||
// segments (buses) with their labels in the left gutter
|
||||
segs.forEach((g, i) => {
|
||||
const y = segY[i];
|
||||
const v4 = g.subnets.filter(s => s.family !== 6), v6 = g.subnets.filter(s => s.family === 6);
|
||||
const used = g.subnets.reduce((a, s) => a + (s.used || 0), 0);
|
||||
const title = g.subnets.map(s => `${s.cidr}${s.name ? ' ' + s.name : ''} · ${s.purpose} · ${s.used}/${s.size || '∞'} used${s.gateway ? ' · gw ' + s.gateway : ''}`).join('\n');
|
||||
o.push(`<g class="seg ${g.dhcp ? 'dhcp' : ''} p-${esc(g.purpose)}" data-ips="${esc(g.subnets[0].cidr)}"><title>${esc(title)}</title>`);
|
||||
o.push(`<line x1="${M + 250}" y1="${y}" x2="${W - M}" y2="${y}"/>`);
|
||||
o.push(`<text class="t1" x="${M}" y="${y - 4}">${esc((g.fabric || 'no fabric') + ' · ' + (g.vid ? 'vlan ' + g.vid : 'untagged'))}</text>`);
|
||||
if (!compact) o.push(`<text class="t2" x="${M}" y="${y + 11}">${esc([...v4, ...v6].map(s => s.cidr).join(' '))}</text>`);
|
||||
o.push(`<text class="t3" x="${W - M}" y="${y - 5}" text-anchor="end">${esc(g.purpose)}${g.dhcp ? ' · DHCP/PXE' : ''}${g.mtu && g.mtu !== 1500 ? ' · mtu ' + g.mtu : ''} · ${used} addr</text>`);
|
||||
o.push('</g>');
|
||||
});
|
||||
// region + external IPAM
|
||||
o.push(`<g class="node region"><title>region: API, database, CA</title><rect x="${regionX - regW / 2}" y="${yRegion}" width="${regW}" height="${hTop}" rx="7"/>
|
||||
<text class="n1" x="${regionX}" y="${yRegion + (compact ? 17 : 20)}" text-anchor="middle">region</text>
|
||||
<text class="n2" x="${regionX}" y="${yRegion + (compact ? 31 : 37)}" text-anchor="middle">${esc(location.host)}</text></g>`);
|
||||
if (hasIPAM) {
|
||||
const last = S.ipam.last_sync || {};
|
||||
const ok = !last.error;
|
||||
o.push(`<line class="wire ext" x1="${regionX + regW / 2}" y1="${yRegion + hTop / 2}" x2="${ix - iw / 2}" y2="${yRegion + hTop / 2}"/>`);
|
||||
o.push(`<g class="node ipam ${ok ? '' : 'bad'}"><title>${esc(`${S.ipam.provider} ${S.ipam.url}\nzone ${S.ipam.dns_zone || '-'}\nlast sync ${last.at ? new Date(last.at).toLocaleString() : 'never'}${last.error ? '\n' + last.error : ''}`)}</title>
|
||||
<rect x="${ix - iw / 2}" y="${yRegion}" width="${iw}" height="${hTop}" rx="7"/>
|
||||
<text class="n1" x="${ix}" y="${yRegion + (compact ? 17 : 20)}" text-anchor="middle">${esc(S.ipam.provider)} IPAM${ok ? '' : ' ⚠'}</text>
|
||||
<text class="n2" x="${ix}" y="${yRegion + (compact ? 31 : 37)}" text-anchor="middle">${esc(S.ipam.dns_zone || S.ipam.url)}</text></g>`);
|
||||
}
|
||||
// rack controllers, wired to the region (mTLS) and to their buses
|
||||
racks.forEach((r, i) => {
|
||||
const x = rackX[i], c = r.c, wx = x - RW / 2 + 22; // bus drop on the box's left, clear of machine drops
|
||||
o.push(`<path class="wire ctl ${c.online ? '' : 'down'}" d="M${regionX},${yRegion + hTop} V${(yRegion + hTop + yRack) / 2} H${x} V${yRack}"/>`);
|
||||
if (r.at.length) {
|
||||
o.push(`<line class="wire rackdrop" x1="${wx}" y1="${yRack + hTop}" x2="${wx}" y2="${segY[r.at[r.at.length - 1]]}"/>`);
|
||||
for (const si of r.at) o.push(`<rect class="tap rack" x="${wx - 4}" y="${segY[si] - 4}" width="8" height="8"/>`);
|
||||
}
|
||||
const addrs = (c.interfaces || []).flatMap(itf => itf.addrs || []).join(', ');
|
||||
o.push(`<g class="node rack ${c.online ? 'ok' : 'bad'}"><title>${esc(`${c.name} (${c.kind}) ${c.online ? 'online' : 'OFFLINE'}\n${c.url}\n${addrs}\n${c.machines} machines`)}</title>
|
||||
<rect x="${x - RW / 2}" y="${yRack}" width="${RW}" height="${hTop}" rx="7"/>
|
||||
<circle class="led" cx="${x - RW / 2 + 12}" cy="${yRack + 13}" r="4"/>
|
||||
<text class="n1" x="${x}" y="${yRack + (compact ? 17 : 20)}" text-anchor="middle">${esc(c.name)}</text>
|
||||
<text class="n2" x="${x}" y="${yRack + (compact ? 31 : 37)}" text-anchor="middle">${compact ? esc(c.kind) : `${esc(c.kind)} · ${c.machines} machines`}</text></g>`);
|
||||
});
|
||||
// machines
|
||||
items.forEach((it, i) => {
|
||||
const x = itemX[i], cx = x + BW / 2;
|
||||
if (it.at.length) {
|
||||
o.push(`<line class="wire" x1="${cx}" y1="${segY[it.at[0]]}" x2="${cx}" y2="${yM}"/>`);
|
||||
for (const si of it.at) o.push(`<circle class="tap" cx="${cx}" cy="${segY[si]}" r="3"/>`);
|
||||
}
|
||||
if (it.group) {
|
||||
const by = {}; for (const x of it.group) by[x.m.state] = (by[x.m.state] || 0) + 1;
|
||||
const worst = Object.keys(by).map(stateClass).sort((a, b) => ['bad', 'warn', 'ok', 'idle'].indexOf(a) - ['bad', 'warn', 'ok', 'idle'].indexOf(b))[0];
|
||||
o.push(`<g class="node mach group ${worst}" data-go="machines"><title>${esc(Object.entries(by).map(([s, n]) => `${n} ${s}`).join('\n'))}</title>
|
||||
<rect x="${x}" y="${yM}" width="${BW}" height="${BH}" rx="6"/><rect class="stack" x="${x + 4}" y="${yM + 4}" width="${BW}" height="${BH}" rx="6"/>
|
||||
<text class="n1" x="${cx}" y="${yM + (compact ? 17 : 21)}" text-anchor="middle">${it.group.length} machines</text>
|
||||
<text class="n2" x="${cx}" y="${yM + (compact ? 31 : 38)}" text-anchor="middle">${esc(groupLine(by, compact ? 18 : 26))}</text></g>`);
|
||||
} else {
|
||||
const m = it.m;
|
||||
o.push(`<g class="node mach ${stateClass(m.state)}" data-mid="${esc(m.id)}"><title>${esc(`${nameOf(m)} · ${m.state}\n${m.mgmt_ip || ''}\n${m.os || m.profile || ''}\n${locStr(m.location)}`)}</title>
|
||||
<rect x="${x}" y="${yM}" width="${BW}" height="${BH}" rx="6"/><rect class="stripe" x="${x}" y="${yM}" width="4" height="${BH}" rx="2"/>
|
||||
<text class="n1" x="${cx + 2}" y="${yM + (compact ? 17 : 21)}" text-anchor="middle">${esc(nameOf(m).slice(0, 18))}</text>
|
||||
<text class="n2" x="${cx + 2}" y="${yM + (compact ? 31 : 38)}" text-anchor="middle">${esc(compact ? m.state : (m.mgmt_ip || m.state))}</text></g>`);
|
||||
}
|
||||
});
|
||||
const loose = ms.filter(x => !x.at.length).length;
|
||||
o.push('</svg>');
|
||||
return o.join('') + (loose && !compact ? `<p class="muted" style="margin:6px 0 0">${loose} machine(s) have no address on a modelled subnet yet (new or commissioning); they are drawn without a link.</p>` : '');
|
||||
}
|
||||
|
||||
function wireTopo(root = document) {
|
||||
$$('.topo [data-mid]', root).forEach(g => g.onclick = () => { location.hash = '#/machine/' + g.dataset.mid; });
|
||||
$$('.topo [data-go]', root).forEach(g => g.onclick = () => nav(g.dataset.go));
|
||||
}
|
||||
Executable
+311
@@ -0,0 +1,311 @@
|
||||
#!/bin/bash
|
||||
# Rollcall region on a fresh Debian 13 host. Run as root from a checkout
|
||||
# (deploy/) or an unpacked release (rollcall + deploy/ + agent/).
|
||||
#
|
||||
# Day 0 - nothing about the network is decided yet:
|
||||
# ./bootstrap.sh [--image-url URL | --build-image] [--name NAME]
|
||||
# installs everything and starts the region on this host's management
|
||||
# address: web UI and API work, no DHCP, no subnets. Then look around:
|
||||
# rollcall net discover interfaces, switch ports (LLDP), DHCP servers, neighbours
|
||||
#
|
||||
# Later - once the provisioning network is known (run again, nothing is lost):
|
||||
# ./bootstrap.sh --iface IFACE [--gateway IP | --nat UPLINK] [--domain D] [--no-dhcp]
|
||||
# IFACE must have its address on that network; this host's address there
|
||||
# becomes the server URL machines use, the subnet is modelled with ranges
|
||||
# and (unless --no-dhcp) this host serves DHCP/PXE on it.
|
||||
# Or do the same from the web UI / CLI (rollcall subnet add, vlan add -dhcp),
|
||||
# or import the prefixes from NetBox (rollcall ipam import).
|
||||
#
|
||||
# --name NAME rack controller name of this host (default: short hostname)
|
||||
# --image-url directory with vmlinuz, initrd.img, filesystem.squashfs, SHA256SUMS - a
|
||||
# region's http://HOST/rollcall, or a release's download URL, where the files are
|
||||
# commissioning-* (deploy/publish-image.sh); IMAGE_TOKEN=... for a private repository
|
||||
# --build-image build the commissioning image here with live-build (~20 min)
|
||||
# --gateway IP default router for machines on the provisioning network
|
||||
# --nat UPLINK this host routes for them: forward + masquerade out of UPLINK
|
||||
# (default with --iface and no --gateway: the default route's interface)
|
||||
# --domain D DNS domain handed to machines (default rollcall.internal)
|
||||
# --no-dhcp model the network but leave DHCP to the existing server
|
||||
# --ssh-key K SSH public key (or a file of them) for root on deployed machines; repeatable.
|
||||
# Default: the keys of the user running sudo, then root's.
|
||||
# --os NAME import one OS image now (e.g. debian-13); a nightly timer syncs the catalog
|
||||
# --yes do not ask
|
||||
set -euo pipefail
|
||||
HERE=$(cd "$(dirname "$0")" && pwd)
|
||||
ROOT=$(cd "$HERE/.." && pwd)
|
||||
IFACE="" RACK=$(hostname -s) DOMAIN=rollcall.internal GATEWAY="" NAT="" IMAGE_URL="" BUILD=0 OS="" DHCP=1 YES=0 SSHKEYS=()
|
||||
while [ $# -gt 0 ]; do
|
||||
case "$1" in
|
||||
--iface) IFACE=$2; shift ;;
|
||||
--name) RACK=$2; shift ;;
|
||||
--domain) DOMAIN=$2; shift ;;
|
||||
--gateway) GATEWAY=$2; shift ;;
|
||||
--nat) NAT=$2; shift ;;
|
||||
--image-url) IMAGE_URL=${2%/}; shift ;;
|
||||
--build-image) BUILD=1 ;;
|
||||
--os) OS=$2; shift ;;
|
||||
--ssh-key) SSHKEYS+=("$2"); shift ;;
|
||||
--no-dhcp) DHCP=0 ;;
|
||||
--yes|-y) YES=1 ;;
|
||||
-h|--help) sed -n '2,31p' "$0"; exit 0 ;;
|
||||
*) echo "unknown option $1 (--help)" >&2; exit 2 ;;
|
||||
esac
|
||||
shift
|
||||
done
|
||||
|
||||
say() { printf '\n\033[1m== %s\033[0m\n' "$*"; }
|
||||
ok() { printf ' \033[32mOK\033[0m %s\n' "$*"; }
|
||||
die() { printf ' \033[31mERROR\033[0m %s\n' "$*" >&2; exit 1; }
|
||||
[ "$(id -u)" = 0 ] || die "run as root"
|
||||
OSNAME=$(. /etc/os-release; echo "$ID $VERSION_ID")
|
||||
case "$OSNAME" in debian*) ;; *) echo "note: tested on Debian 13; this is $OSNAME" >&2 ;; esac
|
||||
CONF=/etc/rollcall/config.json
|
||||
UPLINK=$(ip -4 route show default | awk '{print $5; exit}')
|
||||
addr_of() { ip -4 -o addr show dev "$1" scope global | awk '{print $4; exit}'; }
|
||||
|
||||
# ------------------------------------------------------------------ 0. where
|
||||
say "0/9 this host"
|
||||
if [ -n "$IFACE" ]; then
|
||||
CIDR_ADDR=$(addr_of "$IFACE")
|
||||
[ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; give it its address on the provisioning network first"
|
||||
read -r ADDR NET PREFIX FIRST <<<"$(python3 -c 'import ipaddress,sys;i=ipaddress.ip_interface(sys.argv[1]);print(i.ip,i.network,i.network.prefixlen,i.network.network_address)' "$CIDR_ADDR")"
|
||||
[ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; model it by hand (rollcall subnet add / range add)"
|
||||
if [ -z "$GATEWAY" ]; then GATEWAY=$ADDR; NAT=${NAT:-$UPLINK}; fi
|
||||
[ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not $IFACE"
|
||||
echo " provisioning $IFACE $CIDR_ADDR (subnet $NET), machines' gateway $GATEWAY${NAT:+ via NAT out of $NAT}, DHCP $([ $DHCP = 1 ] && echo on || echo off)"
|
||||
else
|
||||
[ -n "$UPLINK" ] || die "no default route: give this host network access first"
|
||||
ADDR=$(addr_of "$UPLINK" | cut -d/ -f1)
|
||||
if [ -f "$CONF" ]; then
|
||||
echo " existing installation: server address and network model are kept"
|
||||
else
|
||||
echo " no provisioning network yet: the region starts on $UPLINK ($ADDR) without DHCP or subnets"
|
||||
fi
|
||||
fi
|
||||
SERVER=http://$ADDR
|
||||
echo " region $SERVER, rack controller \"$RACK\""
|
||||
if [ $YES = 0 ] && [ -t 0 ]; then
|
||||
read -r -p " continue? [y/N] " a; case "$a" in y|Y|yes) ;; *) exit 1 ;; esac
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ 1. packages
|
||||
say "1/9 packages"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
PKGS="postgresql dnsmasq pxelinux syslinux-common syslinux-efi qemu-utils ipmitool jq zstd curl ca-certificates python3 nftables lldpd"
|
||||
[ $BUILD = 1 ] && PKGS="$PKGS live-build debootstrap squashfs-tools xorriso"
|
||||
apt-get update -q >/dev/null
|
||||
apt-get install -yq $PKGS >/dev/null
|
||||
install -d /etc/dnsmasq.d
|
||||
# until rollcall serves a VLAN, dnsmasq must not answer DNS/DHCP everywhere
|
||||
# (rollcall's generated configuration adds the interfaces of the VLANs it serves)
|
||||
printf '# rollcall bootstrap: answer only on loopback until rollcall serves a VLAN\n# (systemd-resolved keeps 127.0.0.53)\nbind-dynamic\ninterface=lo\n' > /etc/dnsmasq.d/00-rollcall-base.conf
|
||||
systemctl restart dnsmasq || true
|
||||
ok "installed (lldpd shows which switch port and VLAN each interface is on)"
|
||||
|
||||
# ------------------------------------------------------------------ 2. database
|
||||
say "2/9 PostgreSQL"
|
||||
systemctl enable --now postgresql >/dev/null
|
||||
PGPASS=""
|
||||
if [ -f "$CONF" ]; then
|
||||
ok "configured before, kept"
|
||||
else
|
||||
PGPASS=$(head -c 32 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28)
|
||||
if [ "$(runuser -u postgres -- psql -tAc "select 1 from pg_roles where rolname='rollcall'")" = 1 ]; then
|
||||
runuser -u postgres -- psql -q -c "ALTER ROLE rollcall PASSWORD '$PGPASS'"
|
||||
else
|
||||
runuser -u postgres -- psql -q -c "CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'"
|
||||
fi
|
||||
[ "$(runuser -u postgres -- psql -tAc "select 1 from pg_database where datname='rollcall'")" = 1 ] || runuser -u postgres -- psql -q -c "CREATE DATABASE rollcall OWNER rollcall"
|
||||
ok "role and database rollcall"
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ 3. TFTP
|
||||
say "3/9 TFTP boot files"
|
||||
T=/srv/tftp
|
||||
install -d $T $T/efi64 $T/pxelinux.cfg
|
||||
cp -u /usr/lib/PXELINUX/pxelinux.0 $T/
|
||||
for m in ldlinux.c32 libcom32.c32 libutil.c32 menu.c32; do cp -u /usr/lib/syslinux/modules/bios/$m $T/; done
|
||||
cp -u /usr/lib/SYSLINUX.EFI/efi64/syslinux.efi $T/efi64/
|
||||
for m in ldlinux.e64 libcom32.c32 libutil.c32 menu.c32; do cp -u /usr/lib/syslinux/modules/efi64/$m $T/efi64/; done
|
||||
ln -sfn ../pxelinux.cfg $T/efi64/pxelinux.cfg
|
||||
ok "BIOS (pxelinux.0) and UEFI (efi64/syslinux.efi)"
|
||||
|
||||
# ------------------------------------------------------------------ 4. binary
|
||||
say "4/9 rollcall binary"
|
||||
BIN=""
|
||||
for b in "$ROOT/rollcall" "$HERE/rollcall"; do [ -x "$b" ] && BIN=$b && break; done
|
||||
if [ -z "$BIN" ] && [ -d "$ROOT/cmd/rollcall" ] && command -v go >/dev/null; then
|
||||
(cd "$ROOT/cmd/rollcall" && CGO_ENABLED=0 go build -trimpath -ldflags="-s -w -X main.version=$(git describe --tags --always --dirty 2>/dev/null || echo dev)" -o "$HERE/rollcall" .)
|
||||
BIN=$HERE/rollcall
|
||||
fi
|
||||
[ -n "$BIN" ] || die "no rollcall binary next to this script (unpack a release, or build: cd cmd/rollcall && go build)"
|
||||
[ "$BIN" = "$HERE/rollcall" ] || cp "$BIN" "$HERE/rollcall"
|
||||
ok "$("$HERE/rollcall" version)"
|
||||
|
||||
# ------------------------------------------------------------------ 5. config
|
||||
say "5/9 configuration"
|
||||
getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall
|
||||
install -d -m 0750 -o root -g rollcall /etc/rollcall
|
||||
FQDN=$(hostname -f 2>/dev/null || hostname)
|
||||
CHANGED=0
|
||||
if [ ! -f "$CONF" ]; then
|
||||
AGENT=$("$HERE/rollcall" keygen) ADMIN=$("$HERE/rollcall" keygen)
|
||||
python3 - "$HERE/config.example.json" "$CONF" <<P
|
||||
import json, sys
|
||||
c = json.load(open(sys.argv[1]))
|
||||
c.update({
|
||||
"listen": ":80", "server_url": "$SERVER",
|
||||
"database_url": "postgres://rollcall:$PGPASS@127.0.0.1:5432/rollcall?sslmode=disable",
|
||||
"admin_token": "$ADMIN", "agent_token": "$AGENT",
|
||||
"commission_append": "", # derived from server_url, follows it when it changes
|
||||
"rack_name": "$RACK", "tls_listen": ":443", "tls_names": sorted({"$FQDN", "$ADDR"}), "admin_http": "loopback",
|
||||
"node_pools": {}, "bmc_hints": [],
|
||||
})
|
||||
c["cluster_vars"].update({"domain": "$DOMAIN", "cluster_name": "$RACK", "slurm_controller": "$RACK", "slurm_controller_ip": "$ADDR", "ssh_keys": []})
|
||||
c["ipam"]["provider"] = ""
|
||||
json.dump(c, open(sys.argv[2], "w"), indent=2)
|
||||
P
|
||||
chown root:rollcall "$CONF"; chmod 0640 "$CONF"
|
||||
ok "$CONF written (tokens generated; the web UI is HTTPS-only: admin_http loopback)"
|
||||
else
|
||||
# an existing installation: the provisioning address may have arrived since
|
||||
# (without --iface the server address stays what it is)
|
||||
if [ -n "$IFACE" ] && python3 - "$CONF" "$SERVER" "$ADDR" "$FQDN" <<'P'
|
||||
import json, sys
|
||||
p, srv, addr, fqdn = sys.argv[1:]
|
||||
c = json.load(open(p))
|
||||
old = c.get("server_url", "")
|
||||
names = set(c.get("tls_names") or []) | {addr, fqdn}
|
||||
changed = old != srv or set(c.get("tls_names") or []) != names
|
||||
gen = c.get("commission_append", "")
|
||||
if old and old in gen and "rollcall.token=" in gen: # the old generated form: let it follow server_url
|
||||
c["commission_append"] = ""
|
||||
changed = True
|
||||
c["server_url"], c["tls_names"] = srv, sorted(names)
|
||||
if changed:
|
||||
json.dump(c, open(p, "w"), indent=2)
|
||||
sys.exit(0 if changed else 1)
|
||||
P
|
||||
then CHANGED=1; ok "server_url now $SERVER (was different), tls_names updated"
|
||||
else SERVER=$(jq -r .server_url "$CONF"); ok "$CONF kept (server $SERVER)"
|
||||
fi
|
||||
fi
|
||||
|
||||
# SSH keys for deployed machines: given ones, else the operator's
|
||||
keyfiles=()
|
||||
if [ ${#SSHKEYS[@]} -eq 0 ] && grep -q '"ssh_keys": \[\]' "$CONF" 2>/dev/null; then
|
||||
for f in "/home/${SUDO_USER:-none}/.ssh/authorized_keys" /root/.ssh/authorized_keys; do [ -r "$f" ] && keyfiles+=("$f"); done
|
||||
fi
|
||||
if [ ${#SSHKEYS[@]} -gt 0 ] || [ ${#keyfiles[@]} -gt 0 ]; then
|
||||
n=$(python3 - "$CONF" "${SSHKEYS[@]}" -- "${keyfiles[@]}" <<'P2'
|
||||
import json, os, re, sys
|
||||
conf = sys.argv[1]; rest = sys.argv[2:]; sep = rest.index("--")
|
||||
given, files = rest[:sep], rest[sep + 1:]
|
||||
keyre = re.compile(r'((?:ssh-(?:ed25519|rsa|dss)|ecdsa-sha2-\S+|sk-\S+)\s+[A-Za-z0-9+/=]+(?:\s+\S+)?)')
|
||||
def keys_of(text):
|
||||
return [m.group(1).strip() for line in text.splitlines() for m in [keyre.search(line)] if m]
|
||||
found = []
|
||||
for g in given:
|
||||
found += keys_of(open(g).read()) if os.path.isfile(g) else keys_of(g)
|
||||
for f in files:
|
||||
found += keys_of(open(f).read())
|
||||
c = json.load(open(conf)); cv = c.setdefault("cluster_vars", {})
|
||||
have = cv.get("ssh_keys") or []
|
||||
new = [k for k in dict.fromkeys(found) if k not in have]
|
||||
if new:
|
||||
cv["ssh_keys"] = have + new
|
||||
json.dump(c, open(conf, "w"), indent=2)
|
||||
print(len(new))
|
||||
P2
|
||||
)
|
||||
[ "$n" -gt 0 ] && { ok "$n SSH key(s) for root on deployed machines"; CHANGED=1; } || true
|
||||
fi
|
||||
[ "$(jq '.cluster_vars.ssh_keys | length' "$CONF")" -gt 0 ] || echo " note: no SSH keys for deployed machines yet (--ssh-key KEY)" >&2
|
||||
|
||||
# ------------------------------------------------------------------ 6. services
|
||||
say "6/9 services"
|
||||
install -d /srv/rollcall/image
|
||||
(cd "$HERE" && sh ./install.sh >/dev/null)
|
||||
for _ in $(seq 1 30); do curl -sf -o /dev/null http://127.0.0.1/ca.pem && break; sleep 1; done
|
||||
curl -sf -o /dev/null http://127.0.0.1/ca.pem || die "rollcall did not come up: journalctl -u rollcall"
|
||||
ok "rollcall running, CA $(rollcall ca show | awk '/fingerprint/{print $2}')"
|
||||
|
||||
# ------------------------------------------------------------------ 7. commissioning image
|
||||
say "7/9 commissioning image"
|
||||
IMG=/srv/rollcall/image
|
||||
if [ -s $IMG/vmlinuz ] && [ -s $IMG/initrd.img ] && [ -s $IMG/filesystem.squashfs ]; then
|
||||
ok "present, kept"
|
||||
elif [ -n "$IMAGE_URL" ]; then
|
||||
tmp=$(mktemp -d -p /srv/rollcall)
|
||||
auth=(); [ -n "${IMAGE_TOKEN:-}" ] && auth=(-H "Authorization: token $IMAGE_TOKEN")
|
||||
pre=""
|
||||
if ! curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS"; then
|
||||
pre=commissioning- # a release: the image files carry a prefix
|
||||
curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/${pre}SHA256SUMS" || die "no SHA256SUMS or ${pre}SHA256SUMS at $IMAGE_URL"
|
||||
fi
|
||||
for f in vmlinuz initrd.img filesystem.squashfs; do curl -fSL --progress-bar "${auth[@]}" -o "$tmp/$f" "$IMAGE_URL/$pre$f"; done
|
||||
(cd "$tmp" && sha256sum -c --quiet SHA256SUMS) || die "checksum mismatch"
|
||||
mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rm -rf "$tmp"
|
||||
chmod 0644 $IMG/*
|
||||
ok "downloaded and verified from $IMAGE_URL"
|
||||
elif [ $BUILD = 1 ]; then
|
||||
[ -x "$ROOT/agent/build-image.sh" ] || die "agent/build-image.sh not found next to deploy/"
|
||||
OUT=$IMG "$ROOT/agent/build-image.sh"
|
||||
(cd $IMG && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS)
|
||||
ok "built"
|
||||
else
|
||||
echo " not yet: rerun with --image-url URL or --build-image before machines commission" >&2
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ 8. provisioning network
|
||||
say "8/9 provisioning network"
|
||||
if [ -z "$IFACE" ]; then
|
||||
echo " not decided yet - nothing modelled, no DHCP served. When it is:"
|
||||
echo " rollcall net discover what each interface sees (LLDP, DHCP servers, neighbours)"
|
||||
echo " $0 --iface IFACE ... or the web UI: Network -> New subnet"
|
||||
else
|
||||
[ $CHANGED = 1 ] && systemctl restart rollcall && sleep 2
|
||||
off() { python3 -c "import ipaddress,sys;print(ipaddress.ip_address('$FIRST')+int(sys.argv[1]))" "$1"; }
|
||||
FAB=provisioning
|
||||
rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null
|
||||
if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then
|
||||
rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null
|
||||
rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null
|
||||
rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null
|
||||
rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null
|
||||
ok "subnet $NET: reserved $(off 1)-$(off 20), static $(off 50)-$(off 99), dynamic $(off 100)-$(off 200)"
|
||||
else
|
||||
ok "subnet $NET exists, kept"
|
||||
fi
|
||||
if [ $DHCP = 1 ]; then
|
||||
rollcall vlan add "$FAB" 0 -dhcp -rack "$RACK" >/dev/null
|
||||
ok "DHCP/PXE on $IFACE"
|
||||
fi
|
||||
if [ -n "$NAT" ]; then
|
||||
echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf
|
||||
sysctl -q -p /etc/sysctl.d/60-rollcall-forward.conf
|
||||
install -d /etc/nftables.d
|
||||
printf 'table ip rollcall_nat {\n chain postrouting {\n type nat hook postrouting priority 100;\n oifname "%s" ip saddr %s masquerade\n }\n}\n' "$NAT" "$NET" > /etc/nftables.d/rollcall-nat.nft
|
||||
grep -q 'include "/etc/nftables.d/\*.nft"' /etc/nftables.conf 2>/dev/null || echo 'include "/etc/nftables.d/*.nft"' >> /etc/nftables.conf
|
||||
nft -c -f /etc/nftables.conf && systemctl enable nftables >/dev/null 2>&1 && systemctl restart nftables
|
||||
ok "forwarding + masquerade $NET out of $NAT"
|
||||
fi
|
||||
fi
|
||||
|
||||
# ------------------------------------------------------------------ 9. OS images
|
||||
say "9/9 OS images"
|
||||
if [ -n "$OS" ]; then
|
||||
runuser -u rollcall -- rollcall os sync "$OS" || echo " os sync $OS failed; the nightly timer retries" >&2
|
||||
else
|
||||
echo " none imported now; rollcall-os-sync.timer syncs the catalog nightly (now: runuser -u rollcall -- rollcall os sync debian-13)"
|
||||
fi
|
||||
|
||||
cat <<EOF
|
||||
|
||||
Done: rollcall region "$RACK" on $SERVER
|
||||
web UI https://$ADDR (trust the CA from /ca.pem: $(rollcall ca show | awk '/fingerprint/{print $2}'))
|
||||
admin from your laptop: rollcall admin-cert request NAME (token once: jq -r .admin_token $CONF)
|
||||
check rollcall doctor
|
||||
EOF
|
||||
[ -z "$IFACE" ] && echo " next rollcall net discover (then: $0 --iface IFACE)"
|
||||
exit 0
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"listen": "10.213.77.1:80",
|
||||
"listen": ":80",
|
||||
"image_dir": "/srv/rollcall/image",
|
||||
"tftp_root": "/srv/tftp",
|
||||
"database_url": "postgres://rollcall:<PGPASS>@127.0.0.1:5432/rollcall?sslmode=disable",
|
||||
@@ -12,6 +12,11 @@
|
||||
"images_dir": "/srv/rollcall/images",
|
||||
"overlays_dir": "/etc/rollcall/overlays",
|
||||
"dnsmasq_dir": "/var/lib/rollcall/dnsmasq",
|
||||
"rack_name": "pxe-server",
|
||||
"tls_listen": ":443",
|
||||
"tls_names": ["rollcall.example.com"],
|
||||
"admin_http": "loopback",
|
||||
"ipam": {"provider": "", "url": "https://netbox.int.example.com", "token_file": "/etc/rollcall/netbox.token", "ca_file": "/etc/rollcall/netbox-ca.pem", "dns_zone": "int.example.com", "sync_interval": "5m"},
|
||||
"os_dir": "/srv/rollcall/os",
|
||||
"os_sources": "/etc/rollcall/os-sources.json",
|
||||
"node_pools": {"mgmt": "10.213.77.50-10.213.77.99"},
|
||||
|
||||
Executable
+65
@@ -0,0 +1,65 @@
|
||||
#!/bin/sh
|
||||
# Install a rollcall rack controller on this host (Debian/Ubuntu). Run as root
|
||||
# from this directory with the rollcall binary next to it, after enrolling:
|
||||
#
|
||||
# region$ rollcall controller join-code -name hall-b -addr REGION:443
|
||||
# rack$ ./rollcall rack join <code> -url http://ADDR-ON-THE-SEGMENT
|
||||
#
|
||||
# which writes /etc/rollcall/rack.json and the rack's certificate; from then on
|
||||
# rack and region talk over mutual TLS (rack.example.json shows the fields).
|
||||
#
|
||||
# The rack controller serves DHCP/TFTP (dnsmasq) on the VLANs the region
|
||||
# assigns to it, keeps boot and OS images in a local cache, runs BMC commands
|
||||
# and passes the node API through to the region.
|
||||
set -eu
|
||||
cd "$(dirname "$0")"
|
||||
[ -r /etc/rollcall/rack.json ] || { echo "enroll first: ./rollcall rack join CODE -url http://ADDR (code from: rollcall controller join-code on the region)" >&2; exit 1; }
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get install -y -q dnsmasq ipmitool pxelinux syslinux-common syslinux-efi jq >/dev/null
|
||||
|
||||
getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall
|
||||
[ -x ./rollcall ] && install -m 0755 rollcall /usr/local/bin/rollcall
|
||||
chgrp rollcall /etc/rollcall/rack.json && chmod 0640 /etc/rollcall/rack.json
|
||||
# certificate from rollcall rack join: readable by the service's group, the key by nobody else
|
||||
if [ -d /etc/rollcall/rack ]; then chgrp -R rollcall /etc/rollcall/rack && chmod 0750 /etc/rollcall/rack && chmod 0640 /etc/rollcall/rack/*-key.pem; fi
|
||||
|
||||
TFTP=$(jq -r '(.tftp_root // "") | if . == "" then "/srv/tftp" else . end' /etc/rollcall/rack.json)
|
||||
CACHE=$(jq -r '(.cache_dir // "") | if . == "" then "/var/cache/rollcall" else . end' /etc/rollcall/rack.json)
|
||||
DNS=$(jq -r '(.dnsmasq_dir // "") | if . == "" then "/var/lib/rollcall/dnsmasq" else . end' /etc/rollcall/rack.json)
|
||||
|
||||
# image cache: kernels/initrds are prefetched (TFTP cannot fetch on demand), the rest fills on first use
|
||||
install -d -o rollcall -g rollcall "$CACHE" "$CACHE/rollcall" "$CACHE/images" "$CACHE/os"
|
||||
|
||||
# TFTP root: syslinux for BIOS and UEFI, per-MAC files written by the rack controller
|
||||
install -d "$TFTP" "$TFTP/efi64"
|
||||
install -d -o rollcall -g rollcall -m 2775 "$TFTP/pxelinux.cfg"
|
||||
cp /usr/lib/PXELINUX/pxelinux.0 "$TFTP/"
|
||||
for f in ldlinux.c32 menu.c32 libutil.c32 libcom32.c32 poweroff.c32; do cp /usr/lib/syslinux/modules/bios/$f "$TFTP/"; done
|
||||
cp /usr/lib/SYSLINUX.EFI/efi64/syslinux.efi "$TFTP/efi64/"
|
||||
for f in ldlinux.e64 menu.c32 libutil.c32 libcom32.c32; do cp /usr/lib/syslinux/modules/efi64/$f "$TFTP/efi64/"; done
|
||||
ln -sfn ../pxelinux.cfg "$TFTP/efi64/pxelinux.cfg"
|
||||
ln -sfn "$CACHE/rollcall" "$TFTP/rollcall"
|
||||
ln -sfn "$CACHE/images" "$TFTP/images"
|
||||
ln -sfn ../rollcall "$TFTP/efi64/rollcall"
|
||||
ln -sfn ../images "$TFTP/efi64/images"
|
||||
|
||||
# dnsmasq reads what the rack controller generates; until then it only answers local DNS
|
||||
install -d -o rollcall -g rollcall "$DNS" "$DNS/conf.d"
|
||||
for f in hosts hosts.dhcp; do [ -e "$DNS/$f" ] || install -m 0644 -o rollcall -g rollcall /dev/null "$DNS/$f"; done
|
||||
# bind-dynamic binds address by address, so systemd-resolved's 127.0.0.53 stub does not get in the way;
|
||||
# until the region assigns a VLAN only loopback is served
|
||||
printf '# rollcall rack controller: the configuration is generated from the region'"'"'s network model\nbind-dynamic\ninterface=lo\nconf-dir=%s/conf.d/,*.conf\n' "$DNS" > /etc/dnsmasq.d/rollcall.conf
|
||||
install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh
|
||||
install -m 0755 rollcall-rack-update.sh /usr/local/bin/rollcall-rack-update.sh
|
||||
install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/update
|
||||
install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service rollcall-rack-update.path rollcall-rack-update.service /etc/systemd/system/
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable dnsmasq.service rollcall-dnsmasq.path rollcall-rack-update.path >/dev/null 2>&1
|
||||
systemctl start rollcall-rack-update.path
|
||||
systemctl restart dnsmasq.service || { journalctl -u dnsmasq -n 5 --no-pager >&2; exit 1; }
|
||||
systemctl start rollcall-dnsmasq.path
|
||||
systemctl enable --now rollcall-rack.service
|
||||
systemctl restart rollcall-rack.service
|
||||
sleep 2
|
||||
systemctl --no-pager status rollcall-rack.service | head -5
|
||||
Regular → Executable
+15
-2
@@ -16,7 +16,7 @@ chown rollcall:rollcall /etc/rollcall
|
||||
# rollcall writes per-MAC pxelinux configs
|
||||
chgrp rollcall /srv/tftp/pxelinux.cfg && chmod 2775 /srv/tftp/pxelinux.cfg
|
||||
install -m 0644 /usr/lib/syslinux/modules/bios/poweroff.c32 /srv/tftp/poweroff.c32
|
||||
sed "s/@AGENT_TOKEN@/$(jq -r .agent_token /etc/rollcall/config.json)/g" pxelinux-default > /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default
|
||||
sed -e "s/@AGENT_TOKEN@/$(jq -r .agent_token /etc/rollcall/config.json)/g" -e "s#@SERVER@#$(jq -r .server_url /etc/rollcall/config.json)#g" pxelinux-default > /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default
|
||||
[ -e /srv/tftp/rollcall ] || ln -s /srv/rollcall/image /srv/tftp/rollcall
|
||||
[ -d /srv/tftp/efi64 ] && ln -sfn ../rollcall /srv/tftp/efi64/rollcall
|
||||
# stateless images: <images_dir>/<name>/..., reachable by pxelinux as images/<name>/
|
||||
@@ -34,8 +34,21 @@ install -m 0644 rollcall-os-sync.service rollcall-os-sync.timer /etc/systemd/sys
|
||||
command -v qemu-img >/dev/null || echo "WARNING: qemu-img missing (apt install qemu-utils) - rollcall os sync/import will fail" >&2
|
||||
# dnsmasq host export + reload trigger
|
||||
install -d -m 0755 /var/lib/rollcall
|
||||
install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/dnsmasq
|
||||
install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/dnsmasq /var/lib/rollcall/dnsmasq/conf.d
|
||||
for f in hosts hosts.dhcp; do [ -e /var/lib/rollcall/dnsmasq/$f ] || install -m 0644 -o rollcall -g rollcall /dev/null /var/lib/rollcall/dnsmasq/$f; done
|
||||
install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh
|
||||
install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service /etc/systemd/system/
|
||||
# the embedded rack controller writes conf.d/rollcall.conf once a VLAN with dhcp is assigned to it;
|
||||
# a hand-written dnsmasq configuration for the same interface has to go then (see README, Network)
|
||||
if [ -d /etc/dnsmasq.d ] && [ ! -e /etc/dnsmasq.d/rollcall.conf ]; then
|
||||
printf '# rollcall: generated from the network model (rollcall subnet / vlan)\nconf-dir=/var/lib/rollcall/dnsmasq/conf.d/,*.conf\n' > /etc/dnsmasq.d/rollcall.conf
|
||||
fi
|
||||
legacy=$(grep -lE '^(dhcp-range|bind-interfaces|enable-tftp)' /etc/dnsmasq.conf /etc/dnsmasq.d/*.conf 2>/dev/null | grep -v '/rollcall.conf$' || true)
|
||||
if [ -n "$legacy" ]; then
|
||||
echo "NOTE: hand-written dnsmasq DHCP/TFTP configuration in: $legacy" >&2
|
||||
echo " Once a VLAN with dhcp is assigned to this host (rollcall vlan add … -dhcp), rollcall generates" >&2
|
||||
echo " that configuration; move these files away then, or the generated one is rejected." >&2
|
||||
fi
|
||||
# optional: this host is also the Slurm controller -> keep its slurm.conf rendered from the profile
|
||||
install -m 0644 rollcall-controller-sync.service rollcall-controller-sync.timer /etc/systemd/system/
|
||||
install -m 0755 rollcall-controller-sync.sh /usr/local/bin/rollcall-controller-sync.sh
|
||||
|
||||
@@ -43,8 +43,8 @@
|
||||
"name": "centos-stream-9",
|
||||
"description": "CentOS Stream 9 GenericCloud",
|
||||
"family": "rhel",
|
||||
"url": "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-x86_64-9-latest.x86_64.qcow2",
|
||||
"checksum_url": "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-x86_64-9-latest.x86_64.qcow2.SHA256SUM",
|
||||
"url": "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2",
|
||||
"checksum_url": "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-latest.x86_64.qcow2.SHA256SUM",
|
||||
"checksum": "sha256",
|
||||
"track": "latest",
|
||||
"keep": 2
|
||||
|
||||
Executable
+22
@@ -0,0 +1,22 @@
|
||||
#!/bin/sh
|
||||
# Publish this region's commissioning image as assets of a Gitea release, so new
|
||||
# installations download it (bootstrap.sh --image-url .../releases/download/TAG).
|
||||
# GITEA_TOKEN=... ./publish-image.sh v0.2.0 [https://gitea.example/api/v1/repos/OWNER/REPO]
|
||||
# Files are uploaded as commissioning-{vmlinuz,initrd.img,filesystem.squashfs,SHA256SUMS}.
|
||||
set -eu
|
||||
TAG=${1:?usage: publish-image.sh TAG [API]}
|
||||
API=${2:-https://gitea.mynodes.xyz/api/v1/repos/ilker/rollcall}
|
||||
IMG=${IMG:-/srv/rollcall/image}
|
||||
: "${GITEA_TOKEN:?set GITEA_TOKEN (write:repository)}"
|
||||
cd "$IMG"
|
||||
sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS
|
||||
id=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" "$API/releases/tags/$TAG" | python3 -c 'import json,sys;print(json.load(sys.stdin)["id"])')
|
||||
existing=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" "$API/releases/$id/assets" | python3 -c 'import json,sys;[print(a["id"],a["name"]) for a in json.load(sys.stdin)]')
|
||||
for f in vmlinuz initrd.img filesystem.squashfs SHA256SUMS; do
|
||||
name=commissioning-$f
|
||||
old=$(echo "$existing" | awk -v n="$name" '$2==n{print $1}')
|
||||
[ -n "$old" ] && curl -fsS -X DELETE -H "Authorization: token $GITEA_TOKEN" "$API/releases/$id/assets/$old" >/dev/null
|
||||
echo "uploading $name ($(du -h "$f" | cut -f1))"
|
||||
curl -fsS -H "Authorization: token $GITEA_TOKEN" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" >/dev/null
|
||||
done
|
||||
echo "done: bootstrap.sh --image-url ${API%/api/v1/repos/*}/$(echo "$API" | sed 's#.*/repos/##')/releases/download/$TAG"
|
||||
@@ -3,20 +3,20 @@
|
||||
DEFAULT menu.c32
|
||||
PROMPT 0
|
||||
TIMEOUT 80
|
||||
MENU TITLE pxe-server (10.213.77.1) - unknown machine: commissioning in 8s
|
||||
MENU TITLE rollcall (@SERVER@) - unknown machine: commissioning in 8s
|
||||
|
||||
LABEL commission
|
||||
MENU LABEL Commission this machine (inventory + BMC account)
|
||||
MENU DEFAULT
|
||||
KERNEL rollcall/vmlinuz
|
||||
INITRD rollcall/initrd.img
|
||||
APPEND boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 noautologin rollcall.token=@AGENT_TOKEN@ console=tty0 console=ttyS0,115200n8
|
||||
APPEND boot=live components fetch=@SERVER@/rollcall/filesystem.squashfs rollcall.server=@SERVER@ noautologin rollcall.token=@AGENT_TOKEN@ console=tty0 console=ttyS0,115200n8
|
||||
|
||||
LABEL commission-debug
|
||||
MENU LABEL Commission, but stay up afterwards (ssh root@<ip>)
|
||||
KERNEL rollcall/vmlinuz
|
||||
INITRD rollcall/initrd.img
|
||||
APPEND boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 noautologin rollcall.token=@AGENT_TOKEN@ rollcall.nopoweroff console=tty0 console=ttyS0,115200n8
|
||||
APPEND boot=live components fetch=@SERVER@/rollcall/filesystem.squashfs rollcall.server=@SERVER@ noautologin rollcall.token=@AGENT_TOKEN@ rollcall.nopoweroff console=tty0 console=ttyS0,115200n8
|
||||
|
||||
LABEL netbootxyz
|
||||
MENU LABEL netboot.xyz (needs internet via NAT)
|
||||
|
||||
@@ -0,0 +1,15 @@
|
||||
{
|
||||
"_comment": "written by: rollcall rack join <code from: rollcall controller join-code> -url http://10.213.78.1",
|
||||
"name": "hall-b",
|
||||
"region": "https://10.0.0.234:443",
|
||||
"url": "http://10.213.78.1",
|
||||
"listen": "10.213.78.1:80",
|
||||
"tftp_root": "/srv/tftp",
|
||||
"dnsmasq_dir": "/var/lib/rollcall/dnsmasq",
|
||||
"cache_dir": "/var/cache/rollcall",
|
||||
"ipmitool": "ipmitool",
|
||||
"dhcp_interfaces": [],
|
||||
"ca_file": "/etc/rollcall/rack/ca.pem",
|
||||
"cert_file": "/etc/rollcall/rack/rack.pem",
|
||||
"key_file": "/etc/rollcall/rack/rack-key.pem"
|
||||
}
|
||||
@@ -0,0 +1,53 @@
|
||||
#!/bin/sh
|
||||
# Called by rollcall-dnsmasq.path when rollcall rewrites dnsmasq's files.
|
||||
#
|
||||
# Host files (leases, names) only need a HUP. When the generated configuration
|
||||
# changed, dnsmasq is restarted. If it does not come back (a conflicting
|
||||
# hand-written file in /etc/dnsmasq.d, for instance - `dnsmasq --test` misses
|
||||
# some of those), the last configuration that worked is put back and dnsmasq
|
||||
# is started with it, so DHCP stays up. The rejected content is kept next to it
|
||||
# and not tried again until it changes.
|
||||
set -u
|
||||
D=/var/lib/rollcall/dnsmasq
|
||||
CONF=$D/conf.d/rollcall.conf
|
||||
GOOD=$D/rollcall.conf.accepted
|
||||
BAD=$D/rollcall.conf.rejected
|
||||
STAMP=/run/rollcall-dnsmasq.sum
|
||||
|
||||
sum() { cat "$1" 2>/dev/null | sha256sum | cut -d' ' -f1; }
|
||||
restore() { # put the last working configuration back (or none)
|
||||
if [ -s "$GOOD" ]; then cp "$GOOD" "$CONF.tmp" && mv -f "$CONF.tmp" "$CONF"; else rm -f "$CONF"; fi
|
||||
sum "$CONF" > "$STAMP"
|
||||
}
|
||||
|
||||
new=$(sum "$CONF")
|
||||
if [ "$new" = "$(cat "$STAMP" 2>/dev/null)" ]; then
|
||||
systemctl kill -s HUP dnsmasq.service
|
||||
exit 0
|
||||
fi
|
||||
if [ -e "$CONF" ] && [ "$new" = "$(sum "$BAD")" ]; then
|
||||
restore
|
||||
echo "generated dnsmasq configuration was rejected before and has not changed; keeping the last working one" >&2
|
||||
exit 1
|
||||
fi
|
||||
if out=$(dnsmasq --test 2>&1) && systemctl restart dnsmasq.service; then
|
||||
echo "$new" > "$STAMP"
|
||||
if [ -e "$CONF" ]; then cp "$CONF" "$GOOD"; else rm -f "$GOOD"; fi
|
||||
rm -f "$BAD"
|
||||
echo "dnsmasq restarted: generated configuration changed"
|
||||
exit 0
|
||||
fi
|
||||
why=$(journalctl -u dnsmasq.service -n 5 --no-pager -o cat 2>/dev/null | grep -v '^$' | tail -3 | tr '\n' ' ')
|
||||
[ -e "$CONF" ] && cp "$CONF" "$BAD"
|
||||
restore
|
||||
if systemctl restart dnsmasq.service; then
|
||||
echo "generated dnsmasq configuration rejected ($out $why); dnsmasq runs with the last working one, the rejected file is $BAD" >&2
|
||||
exit 1
|
||||
fi
|
||||
# not even that starts: something outside rollcall's file conflicts. Run dnsmasq
|
||||
# without rollcall's file so whatever else is configured keeps working.
|
||||
rm -f "$CONF"
|
||||
sum "$CONF" > "$STAMP"
|
||||
systemctl restart dnsmasq.service || true
|
||||
echo "dnsmasq does not start with rollcall's configuration ($why); started without it - check /etc/dnsmasq.conf and /etc/dnsmasq.d for conflicting options" >&2
|
||||
exit 1
|
||||
@@ -1,9 +1,10 @@
|
||||
[Unit]
|
||||
Description=rollcall: reload dnsmasq when node host files change
|
||||
Description=rollcall: reload dnsmasq when its generated files change
|
||||
|
||||
[Path]
|
||||
PathChanged=/var/lib/rollcall/dnsmasq/hosts.dhcp
|
||||
PathChanged=/var/lib/rollcall/dnsmasq/hosts
|
||||
PathChanged=/var/lib/rollcall/dnsmasq/conf.d/rollcall.conf
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,6 +1,6 @@
|
||||
[Unit]
|
||||
Description=rollcall: HUP dnsmasq to re-read dhcp-hostsfile / addn-hosts
|
||||
Description=rollcall: reload dnsmasq (HUP for host files, checked restart for configuration)
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/bin/systemctl kill -s HUP dnsmasq.service
|
||||
ExecStart=/usr/local/bin/rollcall-dnsmasq-reload.sh
|
||||
@@ -0,0 +1,8 @@
|
||||
[Unit]
|
||||
Description=rollcall: install a rollcall binary the rack controller staged
|
||||
|
||||
[Path]
|
||||
PathExists=/var/lib/rollcall/update/ready
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,6 @@
|
||||
[Unit]
|
||||
Description=rollcall: install the staged rollcall binary, restart the rack controller, roll back on failure
|
||||
|
||||
[Service]
|
||||
Type=oneshot
|
||||
ExecStart=/usr/local/bin/rollcall-rack-update.sh
|
||||
Executable
+32
@@ -0,0 +1,32 @@
|
||||
#!/bin/sh
|
||||
# Installs a rollcall binary the rack controller staged (rollcall-rack-update.path).
|
||||
# The rack checked the region's checksum and the binary's version; this checks
|
||||
# them again, keeps the running binary as .prev, restarts the rack and puts the
|
||||
# previous binary back if the rack does not register with the region in 60 s.
|
||||
set -u
|
||||
D=/var/lib/rollcall/update
|
||||
BIN=/usr/local/bin/rollcall
|
||||
[ -f "$D/ready" ] || exit 0
|
||||
want=$(cat "$D/ready"); rm -f "$D/ready"
|
||||
cd "$D" || exit 1
|
||||
result() { echo "$(date -Is) $*" > "$D/last-result"; echo "$*"; }
|
||||
sha256sum -c rollcall.new.sha256 >/dev/null 2>&1 || { result "$want: checksum mismatch, not installed"; exit 1; }
|
||||
got=$(./rollcall.new version 2>/dev/null | awk '{print $2}')
|
||||
[ "$got" = "$want" ] || { result "$want: binary reports '$got', not installed"; exit 1; }
|
||||
cur=$("$BIN" version 2>/dev/null | awk '{print $2}')
|
||||
cp -p "$BIN" "$BIN.prev"
|
||||
install -m 0755 rollcall.new "$BIN.tmp" && mv -f "$BIN.tmp" "$BIN"
|
||||
since=$(date +%s)
|
||||
systemctl restart rollcall-rack.service
|
||||
for _ in $(seq 1 30); do
|
||||
sleep 2
|
||||
if journalctl -u rollcall-rack.service --since "@$since" -o cat 2>/dev/null | grep -q "^.*registered with"; then
|
||||
rm -f rollcall.new rollcall.new.sha256
|
||||
result "updated $cur -> $want"
|
||||
exit 0
|
||||
fi
|
||||
done
|
||||
cp -p "$BIN.prev" "$BIN.tmp" && mv -f "$BIN.tmp" "$BIN"
|
||||
systemctl restart rollcall-rack.service
|
||||
result "$want did not register within 60s; rolled back to $cur"
|
||||
exit 1
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=rollcall rack controller (DHCP/TFTP state, BMC commands, image cache, node API proxy)
|
||||
After=network-online.target dnsmasq.service
|
||||
Wants=network-online.target
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/rollcall rack -config /etc/rollcall/rack.json
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
User=rollcall
|
||||
Group=rollcall
|
||||
AmbientCapabilities=CAP_NET_BIND_SERVICE
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/srv/tftp/pxelinux.cfg /var/lib/rollcall /var/cache/rollcall
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,25 @@
|
||||
# Tag fragments
|
||||
|
||||
Each file here is a cloud-init fragment that a tag can carry. When a machine
|
||||
with that tag is deployed, the fragment is added to the generated user-data as
|
||||
a separate MIME part (cloud-config parts merge with `list(append)`, so
|
||||
`packages`, `runcmd` and `write_files` accumulate; a part starting with `#!`
|
||||
runs as a script).
|
||||
|
||||
| File | Tag | Use |
|
||||
|---|---|---|
|
||||
| `gpu-nvidia.yaml` | `gpu` (auto: `{{ gt .Node.GPUs 0 }}`) | NVIDIA driver from the distro's preferred source, reboot when done |
|
||||
|
||||
Register: `rollcall tags add gpu -match '{{ gt .Node.GPUs 0 }}' -cloud-config deploy/tags/gpu-nvidia.yaml -comment "NVIDIA GPUs"`.
|
||||
|
||||
Match templates see `.Node`: `GPUs`, `IBGUIDs`, `CPUs`, `Sockets`, `MemMB`,
|
||||
`Disks`, `NVMe`, `PCIVendors` (e.g. `10de` NVIDIA, `15b3` Mellanox), `Manufacturer`,
|
||||
`Product`, `MACs`, `Tags`, `OS`. Helpers: `has`, `contains`, `hasPrefix`, `matches`,
|
||||
`gte`, plus Go's `eq/gt/lt/and/or/not`. Examples:
|
||||
|
||||
```
|
||||
{{ gt .Node.GPUs 0 }}
|
||||
{{ has .Node.PCIVendors "15b3" }}
|
||||
{{ and (gte .Node.MemMB 500000) (gt .Node.Sockets 1) }}
|
||||
{{ matches "(?i)PowerEdge R7" .Node.Product }}
|
||||
```
|
||||
@@ -0,0 +1,47 @@
|
||||
#cloud-config
|
||||
# Rollcall tag fragment: a TLS certificate for the node from the site's ACME CA
|
||||
# (an ACME server under a Rollcall intermediate, see README "Delegation"), renewed automatically.
|
||||
# Register with: rollcall tags add acme-cert -cloud-config deploy/tags/acme-cert.yaml
|
||||
# Edit the two lines in /etc/rollcall-acme.conf for your site. Deployments already trust the
|
||||
# Rollcall CA (cloud-init ca_certs), so certbot can talk to the ACME server over TLS.
|
||||
# Result: /etc/letsencrypt/live/<fqdn>/{fullchain,privkey}.pem; log /var/log/rollcall-acme.log.
|
||||
write_files:
|
||||
- path: /etc/rollcall-acme.conf
|
||||
permissions: "0644"
|
||||
content: |
|
||||
ACME_DIRECTORY=https://acme.int.mynodes.xyz/acme/rollcall/directory
|
||||
DOMAIN=int.mynodes.xyz
|
||||
- path: /usr/local/sbin/rollcall-acme-cert
|
||||
permissions: "0755"
|
||||
content: |
|
||||
#!/bin/sh
|
||||
# Gets <host>.<DOMAIN> from the ACME server (http-01 on port 80) and sets up renewal.
|
||||
set -eu
|
||||
exec >>/var/log/rollcall-acme.log 2>&1
|
||||
. /etc/rollcall-acme.conf
|
||||
. /etc/os-release
|
||||
fqdn="$(hostname -s).$DOMAIN"
|
||||
echo "== $(date -Is) certificate for $fqdn from $ACME_DIRECTORY"
|
||||
case "$ID" in
|
||||
debian|ubuntu) bundle=/etc/ssl/certs/ca-certificates.crt
|
||||
command -v certbot >/dev/null || { apt-get update; DEBIAN_FRONTEND=noninteractive apt-get install -y certbot; } ;;
|
||||
rocky|almalinux|centos|rhel) bundle=/etc/pki/tls/certs/ca-bundle.crt
|
||||
command -v certbot >/dev/null || { dnf -y install epel-release || true; dnf -y install certbot; } ;;
|
||||
*) echo "unsupported distro $ID"; exit 1 ;;
|
||||
esac
|
||||
export REQUESTS_CA_BUNDLE="$bundle"
|
||||
# renewals run from certbot's own timer; it needs the same trust and server
|
||||
for unit in certbot.service certbot-renew.service; do
|
||||
mkdir -p "/etc/systemd/system/$unit.d"
|
||||
printf '[Service]\nEnvironment=REQUESTS_CA_BUNDLE=%s\n' "$bundle" > "/etc/systemd/system/$unit.d/rollcall-acme.conf"
|
||||
done
|
||||
systemctl daemon-reload
|
||||
certbot certonly --standalone --non-interactive --agree-tos --register-unsafely-without-email \
|
||||
--server "$ACME_DIRECTORY" --cert-name "$fqdn" -d "$fqdn" --keep-until-expiring
|
||||
# short-lived certificates: renew when a third of the lifetime is left, not 30 days before
|
||||
sed -i 's/^#\? *renew_before_expiry.*/renew_before_expiry = 2 days/' "/etc/letsencrypt/renewal/$fqdn.conf"
|
||||
grep -q '^renew_before_expiry' "/etc/letsencrypt/renewal/$fqdn.conf" || echo 'renew_before_expiry = 2 days' >> "/etc/letsencrypt/renewal/$fqdn.conf"
|
||||
systemctl enable --now certbot.timer 2>/dev/null || systemctl enable --now certbot-renew.timer
|
||||
openssl x509 -in "/etc/letsencrypt/live/$fqdn/cert.pem" -noout -subject -issuer -enddate
|
||||
runcmd:
|
||||
- [/usr/local/sbin/rollcall-acme-cert]
|
||||
@@ -0,0 +1,48 @@
|
||||
#cloud-config
|
||||
# Rollcall tag fragment: NVIDIA driver + container toolkit on GPU nodes.
|
||||
# Register with: rollcall tags add gpu -match '{{ gt .Node.GPUs 0 }}' -cloud-config deploy/tags/gpu-nvidia.yaml
|
||||
# Merged into the deployment user-data of every machine tagged "gpu"; distro is detected at runtime.
|
||||
packages:
|
||||
- pciutils
|
||||
- curl
|
||||
- gnupg
|
||||
write_files:
|
||||
- path: /usr/local/sbin/rollcall-nvidia-install.sh
|
||||
permissions: "0755"
|
||||
content: |
|
||||
#!/bin/sh
|
||||
# Installs the NVIDIA driver from the distribution's preferred source. Logs to /var/log/rollcall-nvidia.log
|
||||
set -eu
|
||||
exec >>/var/log/rollcall-nvidia.log 2>&1
|
||||
. /etc/os-release
|
||||
echo "== $(date -Is) nvidia install on $PRETTY_NAME"
|
||||
lspci -nn | grep -i 10de || { echo "no NVIDIA device"; exit 0; }
|
||||
case "$ID" in
|
||||
debian)
|
||||
# contrib + non-free for nvidia-driver (kernel headers for DKMS)
|
||||
sed -i 's/^\(deb .* main\)$/\1 contrib non-free non-free-firmware/' /etc/apt/sources.list 2>/dev/null || true
|
||||
if [ -f /etc/apt/sources.list.d/debian.sources ]; then sed -i 's/^Components: main$/Components: main contrib non-free non-free-firmware/' /etc/apt/sources.list.d/debian.sources; fi
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y "linux-headers-$(uname -r)" nvidia-driver nvidia-smi firmware-misc-nonfree
|
||||
;;
|
||||
ubuntu)
|
||||
apt-get update
|
||||
DEBIAN_FRONTEND=noninteractive apt-get install -y ubuntu-drivers-common
|
||||
ubuntu-drivers install --gpgpu || ubuntu-drivers autoinstall
|
||||
;;
|
||||
rocky|almalinux|centos|rhel)
|
||||
rel=$(echo "$VERSION_ID" | cut -d. -f1)
|
||||
dnf -y install epel-release || dnf -y install "https://dl.fedoraproject.org/pub/epel/epel-release-latest-$rel.noarch.rpm"
|
||||
dnf config-manager --add-repo "https://developer.download.nvidia.com/compute/cuda/repos/rhel$rel/x86_64/cuda-rhel$rel.repo"
|
||||
dnf -y install kernel-devel-"$(uname -r)" kernel-headers-"$(uname -r)" dkms
|
||||
dnf -y module install nvidia-driver:latest-dkms || dnf -y install nvidia-driver nvidia-driver-cuda
|
||||
;;
|
||||
*) echo "unsupported distro $ID"; exit 1 ;;
|
||||
esac
|
||||
echo "== done; nvidia-smi after reboot"
|
||||
runcmd:
|
||||
- [ /usr/local/sbin/rollcall-nvidia-install.sh ]
|
||||
power_state:
|
||||
mode: reboot
|
||||
condition: test -e /var/log/rollcall-nvidia.log && grep -q "== done" /var/log/rollcall-nvidia.log
|
||||
message: "rollcall: rebooting to load the NVIDIA driver"
|
||||
@@ -5,8 +5,9 @@ set -eu
|
||||
cd "$(dirname "$0")"
|
||||
getent passwd pve-vbmc >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group pve-vbmc
|
||||
install -m 0755 pve_vbmc.py /usr/local/bin/pve_vbmc.py
|
||||
install -m 0755 pve_redfish.py /usr/local/bin/pve_redfish.py
|
||||
install -m 0755 pve-vbmc-addrs.sh /usr/local/bin/pve-vbmc-addrs.sh
|
||||
install -m 0644 pve-vbmc@.service pve-vbmc-addrs.service /etc/systemd/system/
|
||||
install -m 0644 pve-vbmc@.service pve-redfish@.service pve-vbmc-addrs.service /etc/systemd/system/
|
||||
chgrp pve-vbmc /etc/pve-vbmc/config.json && chmod 0640 /etc/pve-vbmc/config.json
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now pve-vbmc-addrs.service
|
||||
@@ -16,4 +17,9 @@ systemctl enable --now pve-vbmc-addrs.service
|
||||
for v in $(python3 -c 'import json;print(" ".join(str(b["vmid"]) for b in json.load(open("/etc/pve-vbmc/config.json"))["bmcs"]))'); do
|
||||
systemctl enable --now "pve-vbmc@$v.service"
|
||||
done
|
||||
systemctl --no-pager --type=service | grep pve-vbmc
|
||||
# BMC entries with "redfish": "idrac" | "ilo" | "generic" also get a Redfish service on port 443
|
||||
for v in $(python3 -c 'import json;print(" ".join(str(b["vmid"]) for b in json.load(open("/etc/pve-vbmc/config.json"))["bmcs"] if b.get("redfish")))'); do
|
||||
systemctl enable --now "pve-redfish@$v.service"
|
||||
systemctl restart "pve-redfish@$v.service"
|
||||
done
|
||||
systemctl --no-pager --type=service | grep -E "pve-vbmc|pve-redfish"
|
||||
@@ -0,0 +1,21 @@
|
||||
[Unit]
|
||||
Description=pve-redfish Redfish service for Proxmox VM %i (lab BMC driver tests)
|
||||
After=network-online.target pve-vbmc-addrs.service
|
||||
Wants=network-online.target
|
||||
Requires=pve-vbmc-addrs.service
|
||||
|
||||
[Service]
|
||||
Type=simple
|
||||
ExecStart=/usr/local/bin/pve_redfish.py --config /etc/pve-vbmc/config.json --vmid %i
|
||||
Restart=always
|
||||
RestartSec=3
|
||||
AmbientCapabilities=CAP_NET_BIND_SERVICE
|
||||
User=pve-vbmc
|
||||
Group=pve-vbmc
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -1,13 +1,14 @@
|
||||
#!/bin/sh
|
||||
# Adds/removes the BMC listen addresses from /etc/pve-vbmc/config.json
|
||||
# as secondary IPs on the interface that carries the PXE subnet.
|
||||
# Adds/removes the BMC listen addresses from /etc/pve-vbmc/config.json as
|
||||
# secondary IPs on the interface whose connected network contains each one.
|
||||
set -eu
|
||||
CONF=/etc/pve-vbmc/config.json
|
||||
PXE_IF=$(ip -o -4 addr show | awk '/10\.213\.77\.1\//{print $2; exit}')
|
||||
[ -n "$PXE_IF" ] || { echo "pxenet interface not found" >&2; exit 1; }
|
||||
for a in $(python3 -c 'import json;print(" ".join(b["address"] for b in json.load(open("'"$CONF"'"))["bmcs"]))'); do
|
||||
dev=$(ip -o -4 route get "$a" 2>/dev/null | awk '{for (i = 1; i < NF; i++) if ($i == "dev") { print $(i+1); exit }}')
|
||||
[ -n "$dev" ] || { echo "no interface reaches $a" >&2; exit 1; }
|
||||
plen=$(ip -o -4 addr show dev "$dev" | awk '{split($4, p, "/"); print p[2]; exit}')
|
||||
case "$1" in
|
||||
add) ip -4 addr replace "$a/24" dev "$PXE_IF" label "$PXE_IF:bmc" ;;
|
||||
del) ip -4 addr del "$a/24" dev "$PXE_IF" 2>/dev/null || true ;;
|
||||
add) ip -4 addr replace "$a/${plen:-24}" dev "$dev" label "$dev:bmc" ;;
|
||||
del) ip -4 addr del "$a/${plen:-24}" dev "$dev" 2>/dev/null || true ;;
|
||||
esac
|
||||
done
|
||||
@@ -0,0 +1,239 @@
|
||||
#!/usr/bin/env python3
|
||||
"""pve-redfish: a small Redfish service for one Proxmox VM, for testing BMC drivers.
|
||||
|
||||
It answers on the same address as the VM's pve-vbmc (port 443 instead of UDP
|
||||
623) and shapes itself like a vendor so rollcall's Redfish probe and the
|
||||
idrac / ilo / generic drivers see what real hardware shows:
|
||||
|
||||
idrac Vendor "Dell", system /redfish/v1/Systems/System.Embedded.1, PowerCycle
|
||||
ilo Vendor "HPE", system /redfish/v1/Systems/1, no PowerCycle, PushPowerButton
|
||||
generic no vendor, system /redfish/v1/Systems/1, AllowableValues via @Redfish.ActionInfo
|
||||
|
||||
Config: the pve-vbmc config.json; a bmcs entry may carry "redfish": "idrac"|"ilo"|"generic".
|
||||
Basic auth with the same user/password as the IPMI side. Self-signed TLS.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import socket
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import time
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
|
||||
import requests
|
||||
import urllib3
|
||||
|
||||
log = logging.getLogger("pve-redfish")
|
||||
|
||||
PROFILES = {
|
||||
"idrac": {"vendor": "Dell", "manufacturer": "Dell Inc.", "model": "PowerEdge R760 (pve)", "id": "System.Embedded.1",
|
||||
"resets": ["On", "ForceOff", "ForceRestart", "GracefulRestart", "GracefulShutdown", "PushPowerButton", "Nmi", "PowerCycle"],
|
||||
"actioninfo": False, "etag": True},
|
||||
"ilo": {"vendor": "HPE", "manufacturer": "HPE", "model": "ProLiant DL380 Gen11 (pve)", "id": "1",
|
||||
"resets": ["On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"], "actioninfo": False, "etag": False},
|
||||
"generic": {"vendor": "", "manufacturer": "QEMU", "model": "Standard PC (pve)", "id": "1",
|
||||
"resets": ["On", "ForceOff", "GracefulShutdown", "ForceRestart"], "actioninfo": True, "etag": False},
|
||||
}
|
||||
|
||||
|
||||
class Pve:
|
||||
def __init__(self, cfg, vmid):
|
||||
self.api, self.node, self.vmid = cfg["api"].rstrip("/"), cfg["node"], int(vmid)
|
||||
self.http = requests.Session()
|
||||
self.http.headers["Authorization"] = "PVEAPIToken=" + cfg["token"]
|
||||
self.http.verify = cfg.get("verify_tls", False)
|
||||
if not self.http.verify:
|
||||
urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
|
||||
|
||||
def _url(self, path):
|
||||
return f"{self.api}/api2/json/nodes/{self.node}/qemu/{self.vmid}/{path}"
|
||||
|
||||
def get(self, path):
|
||||
r = self.http.get(self._url(path), timeout=10)
|
||||
r.raise_for_status()
|
||||
return r.json()["data"]
|
||||
|
||||
def post(self, path, **data):
|
||||
r = self.http.post(self._url(path), data=data, timeout=30)
|
||||
r.raise_for_status()
|
||||
return r.json()["data"]
|
||||
|
||||
def put(self, path, **data):
|
||||
r = self.http.put(self._url(path), data=data, timeout=30)
|
||||
r.raise_for_status()
|
||||
|
||||
def running(self):
|
||||
return self.get("status/current")["status"] == "running"
|
||||
|
||||
def boot(self, target):
|
||||
cfg = self.get("config")
|
||||
disks = [k for k in ("scsi0", "virtio0", "sata0", "ide0") if k in cfg]
|
||||
cds = sorted(k for k, v in cfg.items() if "media=cdrom" in str(v))
|
||||
nets = sorted(k for k in cfg if k.startswith("net") and k[3:].isdigit())
|
||||
order = {"Pxe": nets + disks + cds, "Cd": cds + disks + nets}.get(target, disks + cds + nets)
|
||||
self.put("config", boot="order=" + ";".join(order))
|
||||
return order
|
||||
|
||||
|
||||
def make_handler(pve, prof, user, password):
|
||||
sysid = prof["id"]
|
||||
syspath = f"/redfish/v1/Systems/{sysid}"
|
||||
state = {"override": "None", "enabled": "Disabled", "etag": 1}
|
||||
want = "Basic " + base64.b64encode(f"{user}:{password}".encode()).decode()
|
||||
|
||||
class H(BaseHTTPRequestHandler):
|
||||
server_version = "pve-redfish"
|
||||
|
||||
def log_message(self, fmt, *args):
|
||||
log.info("%s %s", self.address_string(), fmt % args)
|
||||
|
||||
def send(self, code, body=None, headers=None):
|
||||
data = json.dumps(body).encode() if body is not None else b""
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("OData-Version", "4.0")
|
||||
for k, v in (headers or {}).items():
|
||||
self.send_header(k, v)
|
||||
self.send_header("Content-Length", str(len(data)))
|
||||
self.end_headers()
|
||||
self.wfile.write(data)
|
||||
|
||||
def error(self, code, msg):
|
||||
self.send(code, {"error": {"code": "Base.1.0.GeneralError", "message": msg,
|
||||
"@Message.ExtendedInfo": [{"Message": msg}]}})
|
||||
|
||||
def authed(self):
|
||||
if self.path.rstrip("/") == "/redfish/v1" or self.path.rstrip("/") == "/redfish":
|
||||
return True
|
||||
if self.headers.get("Authorization") != want:
|
||||
self.error(401, "authentication required")
|
||||
return False
|
||||
return True
|
||||
|
||||
def body(self):
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
return json.loads(self.rfile.read(n) or b"{}")
|
||||
|
||||
def system(self):
|
||||
reset = {"target": syspath + "/Actions/ComputerSystem.Reset"}
|
||||
if prof["actioninfo"]:
|
||||
reset["@Redfish.ActionInfo"] = syspath + "/ResetActionInfo"
|
||||
else:
|
||||
reset["ResetType@Redfish.AllowableValues"] = prof["resets"]
|
||||
return {"@odata.id": syspath, "Id": sysid, "Name": "System", "Manufacturer": prof["manufacturer"], "Model": prof["model"],
|
||||
"PowerState": "On" if pve.running() else "Off",
|
||||
"Boot": {"BootSourceOverrideTarget": state["override"], "BootSourceOverrideEnabled": state["enabled"],
|
||||
"BootSourceOverrideTarget@Redfish.AllowableValues": ["None", "Pxe", "Hdd", "Cd", "BiosSetup"]},
|
||||
"Actions": {"#ComputerSystem.Reset": reset}}
|
||||
|
||||
def do_GET(self):
|
||||
if not self.authed():
|
||||
return
|
||||
p = self.path.split("?")[0].rstrip("/")
|
||||
if p in ("/redfish/v1", "/redfish"):
|
||||
root = {"@odata.id": "/redfish/v1/", "RedfishVersion": "1.15.0", "Systems": {"@odata.id": "/redfish/v1/Systems"}}
|
||||
if prof["vendor"]:
|
||||
root["Vendor"] = prof["vendor"]
|
||||
root["Oem"] = {prof["vendor"]: {}}
|
||||
return self.send(200, root)
|
||||
if p == "/redfish/v1/Systems":
|
||||
return self.send(200, {"Members": [{"@odata.id": syspath}], "Members@odata.count": 1})
|
||||
if p == syspath:
|
||||
hdr = {"ETag": f'W/"{state["etag"]}"'} if prof["etag"] else None
|
||||
return self.send(200, self.system(), hdr)
|
||||
if p == syspath + "/ResetActionInfo":
|
||||
return self.send(200, {"Parameters": [{"Name": "ResetType", "Required": True, "AllowableValues": prof["resets"]}]})
|
||||
self.error(404, "not found")
|
||||
|
||||
def do_PATCH(self):
|
||||
if not self.authed():
|
||||
return
|
||||
if self.path.rstrip("/") != syspath:
|
||||
return self.error(404, "not found")
|
||||
if prof["etag"] and self.headers.get("If-Match") not in (f'W/"{state["etag"]}"', "*"):
|
||||
return self.error(412, "If-Match does not match the current ETag")
|
||||
boot = self.body().get("Boot", {})
|
||||
if "BootSourceOverrideEnabled" in boot:
|
||||
state["enabled"] = boot["BootSourceOverrideEnabled"]
|
||||
if "BootSourceOverrideTarget" in boot:
|
||||
state["override"] = boot["BootSourceOverrideTarget"]
|
||||
if state["enabled"] != "Disabled":
|
||||
order = pve.boot(state["override"])
|
||||
log.info("boot override %s -> order %s", state["override"], order)
|
||||
state["etag"] += 1
|
||||
self.send(204)
|
||||
|
||||
def do_POST(self):
|
||||
if not self.authed():
|
||||
return
|
||||
if self.path.rstrip("/") != syspath + "/Actions/ComputerSystem.Reset":
|
||||
return self.error(404, "not found")
|
||||
t = self.body().get("ResetType", "")
|
||||
if t not in prof["resets"]:
|
||||
return self.error(400, f"ResetType {t} is not supported")
|
||||
on = pve.running()
|
||||
log.info("reset %s (was %s)", t, "on" if on else "off")
|
||||
if t == "On" or (t == "PushPowerButton" and not on):
|
||||
if not on:
|
||||
pve.post("status/start")
|
||||
elif t == "ForceOff":
|
||||
if on:
|
||||
pve.post("status/stop")
|
||||
elif t in ("GracefulShutdown",) or (t == "PushPowerButton" and on):
|
||||
pve.post("status/shutdown", forceStop=1, timeout=60)
|
||||
elif t in ("ForceRestart", "GracefulRestart", "PowerCycle"):
|
||||
pve.post("status/reset" if on else "status/start")
|
||||
self.send(204)
|
||||
|
||||
return H
|
||||
|
||||
|
||||
def self_signed(addr):
|
||||
d = tempfile.mkdtemp()
|
||||
crt, key = os.path.join(d, "c.pem"), os.path.join(d, "k.pem")
|
||||
subprocess.run(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", "-days", "3650",
|
||||
"-subj", "/CN=pve-redfish", "-addext", f"subjectAltName=IP:{addr}", "-keyout", key, "-out", crt],
|
||||
check=True, capture_output=True)
|
||||
return crt, key
|
||||
|
||||
|
||||
class Server6(ThreadingHTTPServer):
|
||||
address_family = socket.AF_INET6
|
||||
|
||||
|
||||
def main():
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--config", default="/etc/pve-vbmc/config.json")
|
||||
ap.add_argument("--vmid", type=int, required=True)
|
||||
args = ap.parse_args()
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s")
|
||||
cfg = json.load(open(args.config))
|
||||
entry = next((b for b in cfg["bmcs"] if int(b["vmid"]) == args.vmid), None)
|
||||
if entry is None:
|
||||
log.error("vmid %s not in config", args.vmid)
|
||||
sys.exit(2)
|
||||
prof = PROFILES[entry.get("redfish", "generic")]
|
||||
addr = entry["address"]
|
||||
user = entry.get("user", cfg.get("user", "admin"))
|
||||
password = entry.get("password", cfg.get("password", "password"))
|
||||
handler = make_handler(Pve(cfg, args.vmid), prof, user, password)
|
||||
ip = ipaddress.ip_address(addr)
|
||||
cls = Server6 if ip.version == 6 else ThreadingHTTPServer
|
||||
srv = cls((addr, int(entry.get("redfish_port", 443))), handler)
|
||||
crt, key = self_signed(addr)
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
ctx.load_cert_chain(crt, key)
|
||||
srv.socket = ctx.wrap_socket(srv.socket, server_side=True)
|
||||
log.info("Redfish (%s profile) for VM %s on https://%s:%s", entry.get("redfish", "generic"), args.vmid, addr, entry.get("redfish_port", 443))
|
||||
srv.serve_forever()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in new issue
Block a user