Compare commits
No files matched your search
@@ -56,8 +56,10 @@ jobs:
|
||||
run: |
|
||||
set -eu
|
||||
ssh "$USER@$HOST" 'sudo mkdir -p /opt/rollcall/deploy /opt/rollcall/agent && sudo chown -R "$USER" /opt/rollcall'
|
||||
scp -q deploy/rollcall deploy/install.sh deploy/rollcall.service deploy/pxelinux-default deploy/config.example.json "$USER@$HOST:/opt/rollcall/deploy/"
|
||||
scp -q agent/rollcall-agent.py agent/rollcall-agent.service agent/build-image.sh "$USER@$HOST:/opt/rollcall/agent/"
|
||||
# all of deploy/ and agent/: install.sh installs scripts and units from there,
|
||||
# and a file left out would be installed from a stale copy
|
||||
scp -q -r deploy/* "$USER@$HOST:/opt/rollcall/deploy/"
|
||||
scp -q -r agent/* "$USER@$HOST:/opt/rollcall/agent/"
|
||||
ssh "$USER@$HOST" 'cd /opt/rollcall/deploy && chmod +x install.sh rollcall ../agent/*.sh ../agent/*.py && sudo ./install.sh && sudo rollcall list >/dev/null && echo DEPLOY_OK'
|
||||
|
||||
image:
|
||||
@@ -78,7 +80,8 @@ jobs:
|
||||
env:
|
||||
HOST: ${{ vars.PXE_HOST }}
|
||||
USER: ${{ vars.PXE_USER }}
|
||||
run: ssh "$USER@$HOST" 'sudo /opt/rollcall/agent/build-image.sh 2>&1 | tail -20'
|
||||
# the lab image lets the deploy user's keys in as root (debugging); release images do not
|
||||
run: ssh "$USER@$HOST" 'sudo env AUTHKEYS=$HOME/.ssh/authorized_keys /opt/rollcall/agent/build-image.sh 2>&1 | tail -20'
|
||||
|
||||
site:
|
||||
needs: server
|
||||
|
||||
@@ -69,3 +69,56 @@ jobs:
|
||||
-F "attachment=@$f" "$API/releases/$id/assets?name=$f" >/dev/null
|
||||
echo "uploaded $f"
|
||||
done
|
||||
|
||||
# The commissioning image of this tag: built with live-build on the PXE server
|
||||
# (the runner cannot), in a directory of its own (the lab's live image and
|
||||
# build stay as they are), without anyone's SSH keys, and uploaded to the
|
||||
# release as commissioning-* for bootstrap.sh --image-url.
|
||||
image:
|
||||
needs: release
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- name: SSH setup
|
||||
env:
|
||||
KEY: ${{ secrets.PXE_SSH_KEY }}
|
||||
run: |
|
||||
mkdir -p ~/.ssh && chmod 700 ~/.ssh
|
||||
printf '%s\n' "$KEY" > ~/.ssh/id_ed25519 && chmod 600 ~/.ssh/id_ed25519
|
||||
printf 'Host *\n StrictHostKeyChecking accept-new\n ServerAliveInterval 30\n' > ~/.ssh/config
|
||||
- name: Build the image on the PXE server
|
||||
env:
|
||||
HOST: ${{ vars.PXE_HOST }}
|
||||
USER: ${{ vars.PXE_USER }}
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: |
|
||||
set -eu
|
||||
W=/var/tmp/rollcall-release-$TAG
|
||||
ssh "$USER@$HOST" "sudo rm -rf $W && mkdir -p $W/src"
|
||||
scp -q -r agent deploy/publish-image.sh "$USER@$HOST:$W/src/"
|
||||
ssh "$USER@$HOST" "sudo env AUTHKEYS= BUILD=$W/build OUT=$W/image bash $W/src/agent/build-image.sh > $W/build.log 2>&1 || { tail -40 $W/build.log; exit 1; }; tail -5 $W/build.log"
|
||||
# refuse an image that lets someone in (a listing that fails refuses too)
|
||||
ssh "$USER@$HOST" "sudo unsquashfs -l $W/image/filesystem.squashfs > $W/files" ||
|
||||
{ echo "::error::cannot list the image" >&2; exit 1; }
|
||||
if ssh "$USER@$HOST" "grep -E '/\.ssh/authorized_keys' $W/files"; then
|
||||
echo "::error::the image carries SSH authorized_keys" >&2; exit 1
|
||||
fi
|
||||
- name: Upload to the release
|
||||
env:
|
||||
HOST: ${{ vars.PXE_HOST }}
|
||||
USER: ${{ vars.PXE_USER }}
|
||||
TAG: ${{ github.ref_name }}
|
||||
API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
|
||||
TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
set -eu
|
||||
W=/var/tmp/rollcall-release-$TAG
|
||||
# the token over stdin, not on a command line
|
||||
printf '%s\n' "$TOKEN" | ssh "$USER@$HOST" "read -r GITEA_TOKEN && export GITEA_TOKEN && IMG=$W/image sh $W/src/publish-image.sh $TAG $API"
|
||||
- name: Clean up
|
||||
if: always()
|
||||
env:
|
||||
HOST: ${{ vars.PXE_HOST }}
|
||||
USER: ${{ vars.PXE_USER }}
|
||||
TAG: ${{ github.ref_name }}
|
||||
run: ssh "$USER@$HOST" "sudo rm -rf /var/tmp/rollcall-release-$TAG" || true
|
||||
+155
@@ -6,6 +6,161 @@ publish a tag that has no section here.
|
||||
|
||||
## [Unreleased]
|
||||
|
||||
## [0.3.0] - 2026-10-30
|
||||
|
||||
Third release: working with a site's existing network services (DNS provider, fixed leases on
|
||||
Kea and OpenWrt, NetBox device sync, Infoblox), Redfish across vendors with firmware updates,
|
||||
UEFI Secure Boot, agents over HTTPS, and a security review of the whole code base. Upgrading:
|
||||
rerun `deploy/install.sh` on the region and `deploy/install-rack.sh` on rack hosts.
|
||||
|
||||
### Added
|
||||
- Proxy DHCP: a VLAN in `proxy` mode (`rollcall vlan add ... -proxy-dhcp`, web UI, `bootstrap.sh
|
||||
--proxy-dhcp`) leaves addresses to the site's DHCP server and only answers PXE; machines keep DHCP
|
||||
unless a static range is set aside; doctor and discover know the mode.
|
||||
- DNS provider: `dns` in `config.json` writes machines' A/AAAA records into the site's DNS, through
|
||||
the PowerDNS API (ownership by RRset comment) or RFC 2136 dynamic update with TSIG (ownership by a
|
||||
`_rollcall.<name>` TXT marker and update prerequisites, so foreign records are never overwritten;
|
||||
drift is detected by querying). `rollcall dns`, `rollcall dns sync [-dry]`, a doctor check.
|
||||
- NetBox device sync (`ipam.device_sync`): machines become dcim devices with serial, device type,
|
||||
role, status, interfaces with MAC objects, assigned addresses and primary IP. Site, location and
|
||||
rack are matched by name from the machine's location and never created; ownership by tag and
|
||||
`rollcall_machine` as for addresses, foreign devices of the same name are conflicts.
|
||||
- Infoblox WAPI driver for the IPAM authority (`ipam.provider: infoblox`): one host record per
|
||||
address (DNS-published in `dns_view`, or IPAM-only), ownership by the `rollcall_machine` extensible
|
||||
attribute (kept with `extattrs+`), leases, fixed addresses and ranges count as taken, delegated
|
||||
networks by the `rollcall` attribute. Not tested against a real grid yet.
|
||||
- The release workflow builds the tag's commissioning image on the PXE server and publishes it as
|
||||
`commissioning-*` assets.
|
||||
- Lab NetBox (`lab/netbox/install.sh`): a container on the lab segment for the NetBox drivers.
|
||||
- Fixed leases on the site's DHCP server for proxy-mode VLANs (`dhcp_reservations`), for Kea with
|
||||
`host_cmds` (Kea 3.0+) and OpenWrt (rpcd). Rollcall's own leases are reconciled; foreign ones are
|
||||
reported. `rollcall reservations`.
|
||||
- Firmware updates over Redfish. A package repository on the region (`rollcall firmware import`
|
||||
from a file or URL, served to BMCs through rack controllers) and each machine's firmware inventory.
|
||||
- Update jobs: hand-over, task watch through BMC restarts, optional power cycle, version check
|
||||
against the inventory. They resume after a region restart.
|
||||
- Vendor plans from documentation or vendor tools: Dell, HPE (`.fwpkg` unpacked), Lenovo,
|
||||
Supermicro, GIGABYTE/AMI/ASRock Rack, OpenBMC, xFusion, KAYTUS, Fujitsu iRMC. Cisco CIMC is
|
||||
refused (SCP only).
|
||||
- `fw_method`, `fw_params` and `firmware_base` on the BMC record.
|
||||
- CLI `rollcall firmware`, a Firmware tab and repository in the web UI.
|
||||
- Redfish vendor profiles: one standard driver plus per-vendor behaviour from public documentation,
|
||||
for Dell iDRAC, HPE iLO, Lenovo XCC, Supermicro, xFusion/Huawei iBMC, KAYTUS/Inspur, Fujitsu iRMC,
|
||||
Cisco CIMC, GIGABYTE, ASRock Rack, OpenBMC and AMI MegaRAC.
|
||||
- Boot mode kept, omitted or tried per vendor.
|
||||
- If-Match with 412/428 handling; future-state (`/SD`) and `@Redfish.Settings` targets.
|
||||
- Busy retries and task following.
|
||||
- Chassis reset fallback and an emulated power cycle.
|
||||
- Session login (`auth=session`).
|
||||
- Probe results (reset types, boot targets, boot mode, profile) are kept on the BMC and shown in
|
||||
the UI, and the UI's driver list comes from the server.
|
||||
|
||||
### Security
|
||||
A review of the whole code base; the fixes below. Upgrading: rerun `deploy/install.sh` on the
|
||||
region and `deploy/install-rack.sh` on rack hosts (new dnsmasq layout, signed rack updates).
|
||||
- **Root SSH keys in published images.** `agent/build-image.sh` took
|
||||
the build user's `authorized_keys` unless told otherwise, so the images published with v0.2.0 and
|
||||
v0.2.1 let three lab keys (the maintainer's laptop, the CI deploy key, the lab hypervisor) log in as
|
||||
root to machines while they commission. Keys are now opt-in (`AUTHKEYS=`, the lab deploy sets it),
|
||||
and the release workflow builds the image of the tag itself and refuses one with `authorized_keys`.
|
||||
- **Region file read (critical).** A rack controller could read any file on the region (CA key,
|
||||
config) through `/api/v1/rack/binary?arch=../..`. `arch` is now a known architecture.
|
||||
- **Path traversal.** In `/api/v1/render/...` (admin) as well.
|
||||
- **Machine-reported MACs.** A MAC could be a path, letting a PXE-booting host make the rack write
|
||||
outside the TFTP directory. Only real hardware addresses are accepted. Rack controllers write only
|
||||
`01-xx-..` file names.
|
||||
- **Delegated intermediates.** An ACME intermediate could mint admin or rack certificates (name
|
||||
constraints do not cover the CN), and IP addresses when only a domain was permitted.
|
||||
- Admin and rack identities must now be signed by the root itself.
|
||||
- Intermediates carry a ServerAuth-only EKU, and the unconstrained name type is closed.
|
||||
- **Rack certificates.** These were ServerAuth certificates for any address the rack reported, so
|
||||
one rack could pose as the region.
|
||||
- They are now client-only with no names.
|
||||
- Racks accept only the region's own certificate.
|
||||
- `rack join` checks the returned CA against the fingerprint in the code.
|
||||
- An unbound join code cannot take over an existing rack's name.
|
||||
- Job results are accepted only from the rack that took the job.
|
||||
- **Rack self-update.** The root unit ran a binary the unprivileged rack user staged.
|
||||
- The region now signs each binary with its CA key.
|
||||
- The root unit verifies the signature against the CA from `rack join`, in a root-owned directory,
|
||||
without following links.
|
||||
- **dnsmasq as root.** dnsmasq (root) read configuration the rollcall user writes, and the reload
|
||||
script copied files through links in rollcall's directory.
|
||||
- It now reads a checked copy in `/var/lib/rollcall-dnsmasq`.
|
||||
- The copy may hold only the options Rollcall generates: nothing that runs a program or includes
|
||||
a file.
|
||||
- **Machine record takeover.** Any host on the provisioning network could overwrite a deployed
|
||||
machine's record (and BMC) by reporting its UUID.
|
||||
- Known machines are re-commissioned only when `ready`, `recommission` or failed.
|
||||
- A MAC another machine owns keeps its boot file.
|
||||
- The rack controller of a ready machine is not changed by whoever reports it.
|
||||
- **Injection into dnsmasq, cloud-init, PXE menus, udev rules and DNS updates.** Subnet domain and
|
||||
gateway, host names, NIC names, reservation names and machine-reported identity strings are now
|
||||
validated or reduced to one line. Machine IDs are letters and digits only.
|
||||
- **Web UI.**
|
||||
- Escaping is fixed for machine-reported inventory fields (stored XSS).
|
||||
- A Content-Security-Policy, `frame-ancestors 'none'`, nosniff and no-referrer are set.
|
||||
- Cross-origin state-changing requests are refused (`http.CrossOriginProtection`), so a page
|
||||
elsewhere cannot use a browser's admin certificate.
|
||||
- `?token=` is no longer logged.
|
||||
- **Deploy agent.**
|
||||
- The image URL no longer goes into a shell command; the checksum is taken over the bytes written.
|
||||
- Seed files cannot leave the target root.
|
||||
- The BMC password is not kept in the fallback inventory file.
|
||||
- **Redfish.** Requests stay on the BMC and under `/redfish/`: paths the BMC returns cannot send the
|
||||
credentials elsewhere, and `scheme://` addresses are reduced to scheme and host.
|
||||
- **Machine tokens.** Job and node tokens are readable from the TFTP pxelinux files. They are now
|
||||
accepted only from the machine's own address (its static lease); through a rack, the rack's
|
||||
`X-Forwarded-For` entry counts. A machine without a fixed address binds a job to its first user.
|
||||
`agent_source_check: false` turns this off.
|
||||
- **Agents over HTTPS.** Rack controllers get a server certificate for their own host and serve
|
||||
the agents' endpoints and images on 443. Agents accept the CA by the fingerprint on the PXE
|
||||
command line and do not fall back to HTTP. Old agents keep working over HTTP. The commissioning
|
||||
image has to be rebuilt to get the new agents.
|
||||
- The stateless overlay client works the same way (rebuild stateless images).
|
||||
- **UEFI and Secure Boot.** UEFI machines PXE-boot through Debian's signed shim and GRUB, so machines
|
||||
with Secure Boot on can boot at all; before, the firmware refused the unsigned `syslinux.efi`.
|
||||
- GRUB menus are made from the pxelinux ones.
|
||||
- The local disk is booted by chainloading the installed OS's signed loader.
|
||||
- Secure Boot verifies the kernel; the initrd and squashfs are not covered.
|
||||
- **BMC certificates.** HTTPS BMCs (Redfish, Proxmox) are pinned on first use. The certificate
|
||||
is read in a handshake that carries no credentials, and a changed certificate gets none
|
||||
(`tls_fingerprint`; cleared on an address change or a BMC firmware update by Rollcall).
|
||||
- **BMC hints.** `bmc-hint` no longer returns the password. The region attaches the configured
|
||||
credentials itself and checks the BMC from the rack controller.
|
||||
- **Rack certificate renewal.** Renewal could not overwrite the root-owned files from `rack join`.
|
||||
Renewed certificates now go atomically to `/var/lib/rollcall/rack-cert`, the previous serial is
|
||||
accepted for 7 days after a renewal, and `rollcall rack renew` renews on demand.
|
||||
- **Defaults and limits.**
|
||||
- `admin_http` defaults to `loopback`.
|
||||
- Request bodies are capped (commissioning 4 MB, all 16 MB), with read and idle timeouts.
|
||||
- `bootstrap.sh` and `publish-image.sh` keep the database password and API tokens off command
|
||||
lines.
|
||||
|
||||
### Fixed
|
||||
- IPAM: a reconcile running during an allocation could delete the record just reserved in NetBox.
|
||||
- IPAM and DNS sync: one failing record no longer stops the rest of the pass, deletes included.
|
||||
Reservations are stored in one spelling.
|
||||
- The job watchdog could fail a job that had just started.
|
||||
- Release without `-disk` wiped the smallest disk, not the one the OS was deployed to. Automatic
|
||||
disk choice skips SAN LUNs (iSCSI, FC, SRP).
|
||||
- RFC 2136 lookups follow the request context. The marker is not removed when the database cannot
|
||||
say whether other records need it.
|
||||
- Smaller fixes:
|
||||
- `seq` in templates is capped.
|
||||
- Layout selector `ib:-1` no longer panics.
|
||||
- Redfish detection no longer runs on a machine that cannot be read.
|
||||
- dnsmasq busy loop in proxy mode (a dnsmasq bug with tagged proxy ranges, found with gdb): proxy
|
||||
ranges are untagged, PXE services limited by interface; `dhcp-boot`/`dhcp-match` only for served
|
||||
subnets, no `dhcp-host` lines for proxied ones.
|
||||
|
||||
## [0.2.1] - 2026-10-28
|
||||
|
||||
### Fixed
|
||||
- `bootstrap.sh --image-url` with a release URL took the release's `SHA256SUMS` (the binaries') instead
|
||||
of `commissioning-SHA256SUMS`; it now prefers the image's and says when a private repository
|
||||
needs `IMAGE_TOKEN`.
|
||||
|
||||
## [0.2.0] - 2026-10-28
|
||||
|
||||
Second release: install from scratch, network model and region/rack split, external IPAM
|
||||
|
||||
+10
-4
@@ -17,7 +17,7 @@ exists.
|
||||
| Network access | a default route; the internet or a mirror for packages and cloud images |
|
||||
| Later: provisioning network | an interface of this host (or of a rack controller) on the VLAN where machines PXE-boot |
|
||||
| Later: BMC network | the rack controller must reach the machines' BMCs (IPMI, Redfish) |
|
||||
| Ports | DHCP 67/udp and TFTP 69/udp on the provisioning network; HTTP 80 for machines; HTTPS 443 for people and rack controllers |
|
||||
| Ports | DHCP 67/udp and TFTP 69/udp on the provisioning network; HTTP 80 and HTTPS 443 for machines (agents use 443); HTTPS 443 for people and rack controllers |
|
||||
|
||||
The release bundle `rollcall-<version>-linux-amd64.tar.gz` holds the binary,
|
||||
`deploy/` and `agent/`. From a checkout, `deploy/bootstrap.sh` builds the binary
|
||||
@@ -26,8 +26,8 @@ itself when Go is installed.
|
||||
## Day 0: install
|
||||
|
||||
```sh
|
||||
tar xzf rollcall-v0.2.0-linux-amd64.tar.gz && cd rollcall-v0.2.0-linux-amd64
|
||||
sudo ./deploy/bootstrap.sh --image-url https://gitea.example/OWNER/rollcall/releases/download/v0.2.0
|
||||
tar xzf rollcall-v0.2.1-linux-amd64.tar.gz && cd rollcall-v0.2.1-linux-amd64
|
||||
sudo ./deploy/bootstrap.sh --image-url https://gitea.example/OWNER/rollcall/releases/download/v0.2.1
|
||||
```
|
||||
|
||||
- **Packages, database, TFTP.** PostgreSQL with its own role and a random password,
|
||||
@@ -74,7 +74,7 @@ competes with Rollcall for the machines' PXE boot.
|
||||
|---|---|
|
||||
| An interface with an address and no DHCP | the provisioning network: next step |
|
||||
| Link up, no address, LLDP shows the VLAN | give the interface its address on that VLAN first |
|
||||
| Another DHCP server on the provisioning VLAN | the network team turns it off for that VLAN, or keeps it and points it at this host (next-server, or a DHCP relay), then `--no-dhcp` |
|
||||
| Another DHCP server on the provisioning VLAN | keep it: `--proxy-dhcp` makes this host answer only PXE, the site's server keeps the addresses (or the network team turns it off for that VLAN) |
|
||||
| No link | cable it |
|
||||
|
||||
## Add the provisioning network
|
||||
@@ -149,6 +149,12 @@ region's version.
|
||||
## Integrations
|
||||
|
||||
- **NetBox (or Infoblox) as the address authority.** `ipam` in `config.json`: see README, External IPAM.
|
||||
With NetBox, `ipam.device_sync` also keeps the machines as devices (README, Devices in NetBox);
|
||||
`lab/netbox/install.sh` sets up a throwaway NetBox for trying it.
|
||||
- **The site's DHCP server keeps the addresses (proxy DHCP).** `dhcp_reservations` writes machines'
|
||||
planned addresses there as fixed leases (Kea 3.0+, OpenWrt): see README, Fixed leases.
|
||||
- **DNS names without an IPAM authority.** `dns` in `config.json` (PowerDNS API or RFC 2136
|
||||
with TSIG): see README, DNS provider.
|
||||
- **Certificates for machines.** Delegate a name-constrained intermediate to an ACME
|
||||
server: see README, Certificate authority.
|
||||
|
||||
|
||||
@@ -0,0 +1,674 @@
|
||||
GNU GENERAL PUBLIC LICENSE
|
||||
Version 3, 29 June 2007
|
||||
|
||||
Copyright (C) 2007 Free Software Foundation, Inc. <https://fsf.org/>
|
||||
Everyone is permitted to copy and distribute verbatim copies
|
||||
of this license document, but changing it is not allowed.
|
||||
|
||||
Preamble
|
||||
|
||||
The GNU General Public License is a free, copyleft license for
|
||||
software and other kinds of works.
|
||||
|
||||
The licenses for most software and other practical works are designed
|
||||
to take away your freedom to share and change the works. By contrast,
|
||||
the GNU General Public License is intended to guarantee your freedom to
|
||||
share and change all versions of a program--to make sure it remains free
|
||||
software for all its users. We, the Free Software Foundation, use the
|
||||
GNU General Public License for most of our software; it applies also to
|
||||
any other work released this way by its authors. You can apply it to
|
||||
your programs, too.
|
||||
|
||||
When we speak of free software, we are referring to freedom, not
|
||||
price. Our General Public Licenses are designed to make sure that you
|
||||
have the freedom to distribute copies of free software (and charge for
|
||||
them if you wish), that you receive source code or can get it if you
|
||||
want it, that you can change the software or use pieces of it in new
|
||||
free programs, and that you know you can do these things.
|
||||
|
||||
To protect your rights, we need to prevent others from denying you
|
||||
these rights or asking you to surrender the rights. Therefore, you have
|
||||
certain responsibilities if you distribute copies of the software, or if
|
||||
you modify it: responsibilities to respect the freedom of others.
|
||||
|
||||
For example, if you distribute copies of such a program, whether
|
||||
gratis or for a fee, you must pass on to the recipients the same
|
||||
freedoms that you received. You must make sure that they, too, receive
|
||||
or can get the source code. And you must show them these terms so they
|
||||
know their rights.
|
||||
|
||||
Developers that use the GNU GPL protect your rights with two steps:
|
||||
(1) assert copyright on the software, and (2) offer you this License
|
||||
giving you legal permission to copy, distribute and/or modify it.
|
||||
|
||||
For the developers' and authors' protection, the GPL clearly explains
|
||||
that there is no warranty for this free software. For both users' and
|
||||
authors' sake, the GPL requires that modified versions be marked as
|
||||
changed, so that their problems will not be attributed erroneously to
|
||||
authors of previous versions.
|
||||
|
||||
Some devices are designed to deny users access to install or run
|
||||
modified versions of the software inside them, although the manufacturer
|
||||
can do so. This is fundamentally incompatible with the aim of
|
||||
protecting users' freedom to change the software. The systematic
|
||||
pattern of such abuse occurs in the area of products for individuals to
|
||||
use, which is precisely where it is most unacceptable. Therefore, we
|
||||
have designed this version of the GPL to prohibit the practice for those
|
||||
products. If such problems arise substantially in other domains, we
|
||||
stand ready to extend this provision to those domains in future versions
|
||||
of the GPL, as needed to protect the freedom of users.
|
||||
|
||||
Finally, every program is threatened constantly by software patents.
|
||||
States should not allow patents to restrict development and use of
|
||||
software on general-purpose computers, but in those that do, we wish to
|
||||
avoid the special danger that patents applied to a free program could
|
||||
make it effectively proprietary. To prevent this, the GPL assures that
|
||||
patents cannot be used to render the program non-free.
|
||||
|
||||
The precise terms and conditions for copying, distribution and
|
||||
modification follow.
|
||||
|
||||
TERMS AND CONDITIONS
|
||||
|
||||
0. Definitions.
|
||||
|
||||
"This License" refers to version 3 of the GNU General Public License.
|
||||
|
||||
"Copyright" also means copyright-like laws that apply to other kinds of
|
||||
works, such as semiconductor masks.
|
||||
|
||||
"The Program" refers to any copyrightable work licensed under this
|
||||
License. Each licensee is addressed as "you". "Licensees" and
|
||||
"recipients" may be individuals or organizations.
|
||||
|
||||
To "modify" a work means to copy from or adapt all or part of the work
|
||||
in a fashion requiring copyright permission, other than the making of an
|
||||
exact copy. The resulting work is called a "modified version" of the
|
||||
earlier work or a work "based on" the earlier work.
|
||||
|
||||
A "covered work" means either the unmodified Program or a work based
|
||||
on the Program.
|
||||
|
||||
To "propagate" a work means to do anything with it that, without
|
||||
permission, would make you directly or secondarily liable for
|
||||
infringement under applicable copyright law, except executing it on a
|
||||
computer or modifying a private copy. Propagation includes copying,
|
||||
distribution (with or without modification), making available to the
|
||||
public, and in some countries other activities as well.
|
||||
|
||||
To "convey" a work means any kind of propagation that enables other
|
||||
parties to make or receive copies. Mere interaction with a user through
|
||||
a computer network, with no transfer of a copy, is not conveying.
|
||||
|
||||
An interactive user interface displays "Appropriate Legal Notices"
|
||||
to the extent that it includes a convenient and prominently visible
|
||||
feature that (1) displays an appropriate copyright notice, and (2)
|
||||
tells the user that there is no warranty for the work (except to the
|
||||
extent that warranties are provided), that licensees may convey the
|
||||
work under this License, and how to view a copy of this License. If
|
||||
the interface presents a list of user commands or options, such as a
|
||||
menu, a prominent item in the list meets this criterion.
|
||||
|
||||
1. Source Code.
|
||||
|
||||
The "source code" for a work means the preferred form of the work
|
||||
for making modifications to it. "Object code" means any non-source
|
||||
form of a work.
|
||||
|
||||
A "Standard Interface" means an interface that either is an official
|
||||
standard defined by a recognized standards body, or, in the case of
|
||||
interfaces specified for a particular programming language, one that
|
||||
is widely used among developers working in that language.
|
||||
|
||||
The "System Libraries" of an executable work include anything, other
|
||||
than the work as a whole, that (a) is included in the normal form of
|
||||
packaging a Major Component, but which is not part of that Major
|
||||
Component, and (b) serves only to enable use of the work with that
|
||||
Major Component, or to implement a Standard Interface for which an
|
||||
implementation is available to the public in source code form. A
|
||||
"Major Component", in this context, means a major essential component
|
||||
(kernel, window system, and so on) of the specific operating system
|
||||
(if any) on which the executable work runs, or a compiler used to
|
||||
produce the work, or an object code interpreter used to run it.
|
||||
|
||||
The "Corresponding Source" for a work in object code form means all
|
||||
the source code needed to generate, install, and (for an executable
|
||||
work) run the object code and to modify the work, including scripts to
|
||||
control those activities. However, it does not include the work's
|
||||
System Libraries, or general-purpose tools or generally available free
|
||||
programs which are used unmodified in performing those activities but
|
||||
which are not part of the work. For example, Corresponding Source
|
||||
includes interface definition files associated with source files for
|
||||
the work, and the source code for shared libraries and dynamically
|
||||
linked subprograms that the work is specifically designed to require,
|
||||
such as by intimate data communication or control flow between those
|
||||
subprograms and other parts of the work.
|
||||
|
||||
The Corresponding Source need not include anything that users
|
||||
can regenerate automatically from other parts of the Corresponding
|
||||
Source.
|
||||
|
||||
The Corresponding Source for a work in source code form is that
|
||||
same work.
|
||||
|
||||
2. Basic Permissions.
|
||||
|
||||
All rights granted under this License are granted for the term of
|
||||
copyright on the Program, and are irrevocable provided the stated
|
||||
conditions are met. This License explicitly affirms your unlimited
|
||||
permission to run the unmodified Program. The output from running a
|
||||
covered work is covered by this License only if the output, given its
|
||||
content, constitutes a covered work. This License acknowledges your
|
||||
rights of fair use or other equivalent, as provided by copyright law.
|
||||
|
||||
You may make, run and propagate covered works that you do not
|
||||
convey, without conditions so long as your license otherwise remains
|
||||
in force. You may convey covered works to others for the sole purpose
|
||||
of having them make modifications exclusively for you, or provide you
|
||||
with facilities for running those works, provided that you comply with
|
||||
the terms of this License in conveying all material for which you do
|
||||
not control copyright. Those thus making or running the covered works
|
||||
for you must do so exclusively on your behalf, under your direction
|
||||
and control, on terms that prohibit them from making any copies of
|
||||
your copyrighted material outside their relationship with you.
|
||||
|
||||
Conveying under any other circumstances is permitted solely under
|
||||
the conditions stated below. Sublicensing is not allowed; section 10
|
||||
makes it unnecessary.
|
||||
|
||||
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
|
||||
|
||||
No covered work shall be deemed part of an effective technological
|
||||
measure under any applicable law fulfilling obligations under article
|
||||
11 of the WIPO copyright treaty adopted on 20 December 1996, or
|
||||
similar laws prohibiting or restricting circumvention of such
|
||||
measures.
|
||||
|
||||
When you convey a covered work, you waive any legal power to forbid
|
||||
circumvention of technological measures to the extent such circumvention
|
||||
is effected by exercising rights under this License with respect to
|
||||
the covered work, and you disclaim any intention to limit operation or
|
||||
modification of the work as a means of enforcing, against the work's
|
||||
users, your or third parties' legal rights to forbid circumvention of
|
||||
technological measures.
|
||||
|
||||
4. Conveying Verbatim Copies.
|
||||
|
||||
You may convey verbatim copies of the Program's source code as you
|
||||
receive it, in any medium, provided that you conspicuously and
|
||||
appropriately publish on each copy an appropriate copyright notice;
|
||||
keep intact all notices stating that this License and any
|
||||
non-permissive terms added in accord with section 7 apply to the code;
|
||||
keep intact all notices of the absence of any warranty; and give all
|
||||
recipients a copy of this License along with the Program.
|
||||
|
||||
You may charge any price or no price for each copy that you convey,
|
||||
and you may offer support or warranty protection for a fee.
|
||||
|
||||
5. Conveying Modified Source Versions.
|
||||
|
||||
You may convey a work based on the Program, or the modifications to
|
||||
produce it from the Program, in the form of source code under the
|
||||
terms of section 4, provided that you also meet all of these conditions:
|
||||
|
||||
a) The work must carry prominent notices stating that you modified
|
||||
it, and giving a relevant date.
|
||||
|
||||
b) The work must carry prominent notices stating that it is
|
||||
released under this License and any conditions added under section
|
||||
7. This requirement modifies the requirement in section 4 to
|
||||
"keep intact all notices".
|
||||
|
||||
c) You must license the entire work, as a whole, under this
|
||||
License to anyone who comes into possession of a copy. This
|
||||
License will therefore apply, along with any applicable section 7
|
||||
additional terms, to the whole of the work, and all its parts,
|
||||
regardless of how they are packaged. This License gives no
|
||||
permission to license the work in any other way, but it does not
|
||||
invalidate such permission if you have separately received it.
|
||||
|
||||
d) If the work has interactive user interfaces, each must display
|
||||
Appropriate Legal Notices; however, if the Program has interactive
|
||||
interfaces that do not display Appropriate Legal Notices, your
|
||||
work need not make them do so.
|
||||
|
||||
A compilation of a covered work with other separate and independent
|
||||
works, which are not by their nature extensions of the covered work,
|
||||
and which are not combined with it such as to form a larger program,
|
||||
in or on a volume of a storage or distribution medium, is called an
|
||||
"aggregate" if the compilation and its resulting copyright are not
|
||||
used to limit the access or legal rights of the compilation's users
|
||||
beyond what the individual works permit. Inclusion of a covered work
|
||||
in an aggregate does not cause this License to apply to the other
|
||||
parts of the aggregate.
|
||||
|
||||
6. Conveying Non-Source Forms.
|
||||
|
||||
You may convey a covered work in object code form under the terms
|
||||
of sections 4 and 5, provided that you also convey the
|
||||
machine-readable Corresponding Source under the terms of this License,
|
||||
in one of these ways:
|
||||
|
||||
a) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by the
|
||||
Corresponding Source fixed on a durable physical medium
|
||||
customarily used for software interchange.
|
||||
|
||||
b) Convey the object code in, or embodied in, a physical product
|
||||
(including a physical distribution medium), accompanied by a
|
||||
written offer, valid for at least three years and valid for as
|
||||
long as you offer spare parts or customer support for that product
|
||||
model, to give anyone who possesses the object code either (1) a
|
||||
copy of the Corresponding Source for all the software in the
|
||||
product that is covered by this License, on a durable physical
|
||||
medium customarily used for software interchange, for a price no
|
||||
more than your reasonable cost of physically performing this
|
||||
conveying of source, or (2) access to copy the
|
||||
Corresponding Source from a network server at no charge.
|
||||
|
||||
c) Convey individual copies of the object code with a copy of the
|
||||
written offer to provide the Corresponding Source. This
|
||||
alternative is allowed only occasionally and noncommercially, and
|
||||
only if you received the object code with such an offer, in accord
|
||||
with subsection 6b.
|
||||
|
||||
d) Convey the object code by offering access from a designated
|
||||
place (gratis or for a charge), and offer equivalent access to the
|
||||
Corresponding Source in the same way through the same place at no
|
||||
further charge. You need not require recipients to copy the
|
||||
Corresponding Source along with the object code. If the place to
|
||||
copy the object code is a network server, the Corresponding Source
|
||||
may be on a different server (operated by you or a third party)
|
||||
that supports equivalent copying facilities, provided you maintain
|
||||
clear directions next to the object code saying where to find the
|
||||
Corresponding Source. Regardless of what server hosts the
|
||||
Corresponding Source, you remain obligated to ensure that it is
|
||||
available for as long as needed to satisfy these requirements.
|
||||
|
||||
e) Convey the object code using peer-to-peer transmission, provided
|
||||
you inform other peers where the object code and Corresponding
|
||||
Source of the work are being offered to the general public at no
|
||||
charge under subsection 6d.
|
||||
|
||||
A separable portion of the object code, whose source code is excluded
|
||||
from the Corresponding Source as a System Library, need not be
|
||||
included in conveying the object code work.
|
||||
|
||||
A "User Product" is either (1) a "consumer product", which means any
|
||||
tangible personal property which is normally used for personal, family,
|
||||
or household purposes, or (2) anything designed or sold for incorporation
|
||||
into a dwelling. In determining whether a product is a consumer product,
|
||||
doubtful cases shall be resolved in favor of coverage. For a particular
|
||||
product received by a particular user, "normally used" refers to a
|
||||
typical or common use of that class of product, regardless of the status
|
||||
of the particular user or of the way in which the particular user
|
||||
actually uses, or expects or is expected to use, the product. A product
|
||||
is a consumer product regardless of whether the product has substantial
|
||||
commercial, industrial or non-consumer uses, unless such uses represent
|
||||
the only significant mode of use of the product.
|
||||
|
||||
"Installation Information" for a User Product means any methods,
|
||||
procedures, authorization keys, or other information required to install
|
||||
and execute modified versions of a covered work in that User Product from
|
||||
a modified version of its Corresponding Source. The information must
|
||||
suffice to ensure that the continued functioning of the modified object
|
||||
code is in no case prevented or interfered with solely because
|
||||
modification has been made.
|
||||
|
||||
If you convey an object code work under this section in, or with, or
|
||||
specifically for use in, a User Product, and the conveying occurs as
|
||||
part of a transaction in which the right of possession and use of the
|
||||
User Product is transferred to the recipient in perpetuity or for a
|
||||
fixed term (regardless of how the transaction is characterized), the
|
||||
Corresponding Source conveyed under this section must be accompanied
|
||||
by the Installation Information. But this requirement does not apply
|
||||
if neither you nor any third party retains the ability to install
|
||||
modified object code on the User Product (for example, the work has
|
||||
been installed in ROM).
|
||||
|
||||
The requirement to provide Installation Information does not include a
|
||||
requirement to continue to provide support service, warranty, or updates
|
||||
for a work that has been modified or installed by the recipient, or for
|
||||
the User Product in which it has been modified or installed. Access to a
|
||||
network may be denied when the modification itself materially and
|
||||
adversely affects the operation of the network or violates the rules and
|
||||
protocols for communication across the network.
|
||||
|
||||
Corresponding Source conveyed, and Installation Information provided,
|
||||
in accord with this section must be in a format that is publicly
|
||||
documented (and with an implementation available to the public in
|
||||
source code form), and must require no special password or key for
|
||||
unpacking, reading or copying.
|
||||
|
||||
7. Additional Terms.
|
||||
|
||||
"Additional permissions" are terms that supplement the terms of this
|
||||
License by making exceptions from one or more of its conditions.
|
||||
Additional permissions that are applicable to the entire Program shall
|
||||
be treated as though they were included in this License, to the extent
|
||||
that they are valid under applicable law. If additional permissions
|
||||
apply only to part of the Program, that part may be used separately
|
||||
under those permissions, but the entire Program remains governed by
|
||||
this License without regard to the additional permissions.
|
||||
|
||||
When you convey a copy of a covered work, you may at your option
|
||||
remove any additional permissions from that copy, or from any part of
|
||||
it. (Additional permissions may be written to require their own
|
||||
removal in certain cases when you modify the work.) You may place
|
||||
additional permissions on material, added by you to a covered work,
|
||||
for which you have or can give appropriate copyright permission.
|
||||
|
||||
Notwithstanding any other provision of this License, for material you
|
||||
add to a covered work, you may (if authorized by the copyright holders of
|
||||
that material) supplement the terms of this License with terms:
|
||||
|
||||
a) Disclaiming warranty or limiting liability differently from the
|
||||
terms of sections 15 and 16 of this License; or
|
||||
|
||||
b) Requiring preservation of specified reasonable legal notices or
|
||||
author attributions in that material or in the Appropriate Legal
|
||||
Notices displayed by works containing it; or
|
||||
|
||||
c) Prohibiting misrepresentation of the origin of that material, or
|
||||
requiring that modified versions of such material be marked in
|
||||
reasonable ways as different from the original version; or
|
||||
|
||||
d) Limiting the use for publicity purposes of names of licensors or
|
||||
authors of the material; or
|
||||
|
||||
e) Declining to grant rights under trademark law for use of some
|
||||
trade names, trademarks, or service marks; or
|
||||
|
||||
f) Requiring indemnification of licensors and authors of that
|
||||
material by anyone who conveys the material (or modified versions of
|
||||
it) with contractual assumptions of liability to the recipient, for
|
||||
any liability that these contractual assumptions directly impose on
|
||||
those licensors and authors.
|
||||
|
||||
All other non-permissive additional terms are considered "further
|
||||
restrictions" within the meaning of section 10. If the Program as you
|
||||
received it, or any part of it, contains a notice stating that it is
|
||||
governed by this License along with a term that is a further
|
||||
restriction, you may remove that term. If a license document contains
|
||||
a further restriction but permits relicensing or conveying under this
|
||||
License, you may add to a covered work material governed by the terms
|
||||
of that license document, provided that the further restriction does
|
||||
not survive such relicensing or conveying.
|
||||
|
||||
If you add terms to a covered work in accord with this section, you
|
||||
must place, in the relevant source files, a statement of the
|
||||
additional terms that apply to those files, or a notice indicating
|
||||
where to find the applicable terms.
|
||||
|
||||
Additional terms, permissive or non-permissive, may be stated in the
|
||||
form of a separately written license, or stated as exceptions;
|
||||
the above requirements apply either way.
|
||||
|
||||
8. Termination.
|
||||
|
||||
You may not propagate or modify a covered work except as expressly
|
||||
provided under this License. Any attempt otherwise to propagate or
|
||||
modify it is void, and will automatically terminate your rights under
|
||||
this License (including any patent licenses granted under the third
|
||||
paragraph of section 11).
|
||||
|
||||
However, if you cease all violation of this License, then your
|
||||
license from a particular copyright holder is reinstated (a)
|
||||
provisionally, unless and until the copyright holder explicitly and
|
||||
finally terminates your license, and (b) permanently, if the copyright
|
||||
holder fails to notify you of the violation by some reasonable means
|
||||
prior to 60 days after the cessation.
|
||||
|
||||
Moreover, your license from a particular copyright holder is
|
||||
reinstated permanently if the copyright holder notifies you of the
|
||||
violation by some reasonable means, this is the first time you have
|
||||
received notice of violation of this License (for any work) from that
|
||||
copyright holder, and you cure the violation prior to 30 days after
|
||||
your receipt of the notice.
|
||||
|
||||
Termination of your rights under this section does not terminate the
|
||||
licenses of parties who have received copies or rights from you under
|
||||
this License. If your rights have been terminated and not permanently
|
||||
reinstated, you do not qualify to receive new licenses for the same
|
||||
material under section 10.
|
||||
|
||||
9. Acceptance Not Required for Having Copies.
|
||||
|
||||
You are not required to accept this License in order to receive or
|
||||
run a copy of the Program. Ancillary propagation of a covered work
|
||||
occurring solely as a consequence of using peer-to-peer transmission
|
||||
to receive a copy likewise does not require acceptance. However,
|
||||
nothing other than this License grants you permission to propagate or
|
||||
modify any covered work. These actions infringe copyright if you do
|
||||
not accept this License. Therefore, by modifying or propagating a
|
||||
covered work, you indicate your acceptance of this License to do so.
|
||||
|
||||
10. Automatic Licensing of Downstream Recipients.
|
||||
|
||||
Each time you convey a covered work, the recipient automatically
|
||||
receives a license from the original licensors, to run, modify and
|
||||
propagate that work, subject to this License. You are not responsible
|
||||
for enforcing compliance by third parties with this License.
|
||||
|
||||
An "entity transaction" is a transaction transferring control of an
|
||||
organization, or substantially all assets of one, or subdividing an
|
||||
organization, or merging organizations. If propagation of a covered
|
||||
work results from an entity transaction, each party to that
|
||||
transaction who receives a copy of the work also receives whatever
|
||||
licenses to the work the party's predecessor in interest had or could
|
||||
give under the previous paragraph, plus a right to possession of the
|
||||
Corresponding Source of the work from the predecessor in interest, if
|
||||
the predecessor has it or can get it with reasonable efforts.
|
||||
|
||||
You may not impose any further restrictions on the exercise of the
|
||||
rights granted or affirmed under this License. For example, you may
|
||||
not impose a license fee, royalty, or other charge for exercise of
|
||||
rights granted under this License, and you may not initiate litigation
|
||||
(including a cross-claim or counterclaim in a lawsuit) alleging that
|
||||
any patent claim is infringed by making, using, selling, offering for
|
||||
sale, or importing the Program or any portion of it.
|
||||
|
||||
11. Patents.
|
||||
|
||||
A "contributor" is a copyright holder who authorizes use under this
|
||||
License of the Program or a work on which the Program is based. The
|
||||
work thus licensed is called the contributor's "contributor version".
|
||||
|
||||
A contributor's "essential patent claims" are all patent claims
|
||||
owned or controlled by the contributor, whether already acquired or
|
||||
hereafter acquired, that would be infringed by some manner, permitted
|
||||
by this License, of making, using, or selling its contributor version,
|
||||
but do not include claims that would be infringed only as a
|
||||
consequence of further modification of the contributor version. For
|
||||
purposes of this definition, "control" includes the right to grant
|
||||
patent sublicenses in a manner consistent with the requirements of
|
||||
this License.
|
||||
|
||||
Each contributor grants you a non-exclusive, worldwide, royalty-free
|
||||
patent license under the contributor's essential patent claims, to
|
||||
make, use, sell, offer for sale, import and otherwise run, modify and
|
||||
propagate the contents of its contributor version.
|
||||
|
||||
In the following three paragraphs, a "patent license" is any express
|
||||
agreement or commitment, however denominated, not to enforce a patent
|
||||
(such as an express permission to practice a patent or covenant not to
|
||||
sue for patent infringement). To "grant" such a patent license to a
|
||||
party means to make such an agreement or commitment not to enforce a
|
||||
patent against the party.
|
||||
|
||||
If you convey a covered work, knowingly relying on a patent license,
|
||||
and the Corresponding Source of the work is not available for anyone
|
||||
to copy, free of charge and under the terms of this License, through a
|
||||
publicly available network server or other readily accessible means,
|
||||
then you must either (1) cause the Corresponding Source to be so
|
||||
available, or (2) arrange to deprive yourself of the benefit of the
|
||||
patent license for this particular work, or (3) arrange, in a manner
|
||||
consistent with the requirements of this License, to extend the patent
|
||||
license to downstream recipients. "Knowingly relying" means you have
|
||||
actual knowledge that, but for the patent license, your conveying the
|
||||
covered work in a country, or your recipient's use of the covered work
|
||||
in a country, would infringe one or more identifiable patents in that
|
||||
country that you have reason to believe are valid.
|
||||
|
||||
If, pursuant to or in connection with a single transaction or
|
||||
arrangement, you convey, or propagate by procuring conveyance of, a
|
||||
covered work, and grant a patent license to some of the parties
|
||||
receiving the covered work authorizing them to use, propagate, modify
|
||||
or convey a specific copy of the covered work, then the patent license
|
||||
you grant is automatically extended to all recipients of the covered
|
||||
work and works based on it.
|
||||
|
||||
A patent license is "discriminatory" if it does not include within
|
||||
the scope of its coverage, prohibits the exercise of, or is
|
||||
conditioned on the non-exercise of one or more of the rights that are
|
||||
specifically granted under this License. You may not convey a covered
|
||||
work if you are a party to an arrangement with a third party that is
|
||||
in the business of distributing software, under which you make payment
|
||||
to the third party based on the extent of your activity of conveying
|
||||
the work, and under which the third party grants, to any of the
|
||||
parties who would receive the covered work from you, a discriminatory
|
||||
patent license (a) in connection with copies of the covered work
|
||||
conveyed by you (or copies made from those copies), or (b) primarily
|
||||
for and in connection with specific products or compilations that
|
||||
contain the covered work, unless you entered into that arrangement,
|
||||
or that patent license was granted, prior to 28 March 2007.
|
||||
|
||||
Nothing in this License shall be construed as excluding or limiting
|
||||
any implied license or other defenses to infringement that may
|
||||
otherwise be available to you under applicable patent law.
|
||||
|
||||
12. No Surrender of Others' Freedom.
|
||||
|
||||
If conditions are imposed on you (whether by court order, agreement or
|
||||
otherwise) that contradict the conditions of this License, they do not
|
||||
excuse you from the conditions of this License. If you cannot convey a
|
||||
covered work so as to satisfy simultaneously your obligations under this
|
||||
License and any other pertinent obligations, then as a consequence you may
|
||||
not convey it at all. For example, if you agree to terms that obligate you
|
||||
to collect a royalty for further conveying from those to whom you convey
|
||||
the Program, the only way you could satisfy both those terms and this
|
||||
License would be to refrain entirely from conveying the Program.
|
||||
|
||||
13. Use with the GNU Affero General Public License.
|
||||
|
||||
Notwithstanding any other provision of this License, you have
|
||||
permission to link or combine any covered work with a work licensed
|
||||
under version 3 of the GNU Affero General Public License into a single
|
||||
combined work, and to convey the resulting work. The terms of this
|
||||
License will continue to apply to the part which is the covered work,
|
||||
but the special requirements of the GNU Affero General Public License,
|
||||
section 13, concerning interaction through a network will apply to the
|
||||
combination as such.
|
||||
|
||||
14. Revised Versions of this License.
|
||||
|
||||
The Free Software Foundation may publish revised and/or new versions of
|
||||
the GNU General Public License from time to time. Such new versions will
|
||||
be similar in spirit to the present version, but may differ in detail to
|
||||
address new problems or concerns.
|
||||
|
||||
Each version is given a distinguishing version number. If the
|
||||
Program specifies that a certain numbered version of the GNU General
|
||||
Public License "or any later version" applies to it, you have the
|
||||
option of following the terms and conditions either of that numbered
|
||||
version or of any later version published by the Free Software
|
||||
Foundation. If the Program does not specify a version number of the
|
||||
GNU General Public License, you may choose any version ever published
|
||||
by the Free Software Foundation.
|
||||
|
||||
If the Program specifies that a proxy can decide which future
|
||||
versions of the GNU General Public License can be used, that proxy's
|
||||
public statement of acceptance of a version permanently authorizes you
|
||||
to choose that version for the Program.
|
||||
|
||||
Later license versions may give you additional or different
|
||||
permissions. However, no additional obligations are imposed on any
|
||||
author or copyright holder as a result of your choosing to follow a
|
||||
later version.
|
||||
|
||||
15. Disclaimer of Warranty.
|
||||
|
||||
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
|
||||
APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
|
||||
HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
|
||||
OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
|
||||
THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
||||
PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
|
||||
IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
|
||||
ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
|
||||
|
||||
16. Limitation of Liability.
|
||||
|
||||
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
|
||||
WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
|
||||
THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
|
||||
GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
|
||||
USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
|
||||
DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
|
||||
PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
|
||||
EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
|
||||
SUCH DAMAGES.
|
||||
|
||||
17. Interpretation of Sections 15 and 16.
|
||||
|
||||
If the disclaimer of warranty and limitation of liability provided
|
||||
above cannot be given local legal effect according to their terms,
|
||||
reviewing courts shall apply local law that most closely approximates
|
||||
an absolute waiver of all civil liability in connection with the
|
||||
Program, unless a warranty or assumption of liability accompanies a
|
||||
copy of the Program in return for a fee.
|
||||
|
||||
END OF TERMS AND CONDITIONS
|
||||
|
||||
How to Apply These Terms to Your New Programs
|
||||
|
||||
If you develop a new program, and you want it to be of the greatest
|
||||
possible use to the public, the best way to achieve this is to make it
|
||||
free software which everyone can redistribute and change under these terms.
|
||||
|
||||
To do so, attach the following notices to the program. It is safest
|
||||
to attach them to the start of each source file to most effectively
|
||||
state the exclusion of warranty; and each file should have at least
|
||||
the "copyright" line and a pointer to where the full notice is found.
|
||||
|
||||
<one line to give the program's name and a brief idea of what it does.>
|
||||
Copyright (C) <year> <name of author>
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <https://www.gnu.org/licenses/>.
|
||||
|
||||
Also add information on how to contact you by electronic and paper mail.
|
||||
|
||||
If the program does terminal interaction, make it output a short
|
||||
notice like this when it starts in an interactive mode:
|
||||
|
||||
<program> Copyright (C) <year> <name of author>
|
||||
This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
|
||||
This is free software, and you are welcome to redistribute it
|
||||
under certain conditions; type `show c' for details.
|
||||
|
||||
The hypothetical commands `show w' and `show c' should show the appropriate
|
||||
parts of the General Public License. Of course, your program's commands
|
||||
might be different; for a GUI interface, you would use an "about box".
|
||||
|
||||
You should also get your employer (if you work as a programmer) or school,
|
||||
if any, to sign a "copyright disclaimer" for the program, if necessary.
|
||||
For more information on this, and how to apply and follow the GNU GPL, see
|
||||
<https://www.gnu.org/licenses/>.
|
||||
|
||||
The GNU General Public License does not permit incorporating your program
|
||||
into proprietary programs. If your program is a subroutine library, you
|
||||
may consider it more useful to permit linking proprietary applications with
|
||||
the library. If this is what you want to do, use the GNU Lesser General
|
||||
Public License instead of this License. But first, please read
|
||||
<https://www.gnu.org/licenses/why-not-lgpl.html>.
|
||||
@@ -401,6 +401,104 @@ The lab runs both: `pxe-server` (embedded, 10.213.77.0/24) and `pxe-rack2`
|
||||
pxe-rack2 and its virtual BMC on pxe-rack2. Rack ↔ region traffic is mutual
|
||||
TLS (next section), so the BMC passwords of queued commands are encrypted.
|
||||
|
||||
|
||||
### Agents over HTTPS
|
||||
|
||||
Agents talk to their rack controller over HTTPS: the commissioning agent,
|
||||
the deploy agent and the image download. The region gives each rack a
|
||||
server certificate for its own URL's host (`CN=rack-server:NAME`, server
|
||||
use only, never one of the region's addresses). The rack serves the
|
||||
agents' endpoints on `<host>:443` as well as on HTTP; `tls_listen` in
|
||||
`rack.json` changes the port, `off` turns it off. The embedded rack uses
|
||||
the region's own HTTPS listener.
|
||||
|
||||
The pxelinux command line carries `rollcall.tls=https://HOST` and
|
||||
`rollcall.ca=<SHA-256 of the CA>`. An agent fetches `/ca.pem`, accepts it
|
||||
only by that fingerprint, and from then on uses HTTPS with it. When TLS
|
||||
was asked for and fails, it does not fall back to HTTP.
|
||||
|
||||
What stays on HTTP: live-boot fetching the commissioning squashfs, and
|
||||
`/ca.pem` itself (checked by its fingerprint). This keeps BMC passwords
|
||||
and seed files from anyone who only listens, and from an HTTP server
|
||||
posing as the rack. TFTP and the kernel and initrd stay unauthenticated,
|
||||
so an attacker who can rewrite boot files on the wire is not stopped;
|
||||
that needs UEFI Secure Boot with signed images.
|
||||
### DHCP already on the network: proxy DHCP
|
||||
|
||||
Where the site's DHCP server keeps handing out addresses (a corporate DHCP,
|
||||
Infoblox, a router), a VLAN can be served in **proxy** mode: the rack
|
||||
controller does not hand out addresses, it only answers PXE clients with where
|
||||
to boot from (dnsmasq proxy DHCP), and TFTP and HTTP stay on the rack.
|
||||
|
||||
```
|
||||
rollcall vlan add FABRIC VID -proxy-dhcp -rack RACK # web UI: subnet, DHCP / PXE: proxy
|
||||
deploy/bootstrap.sh --iface eth1 --proxy-dhcp # the same at install time
|
||||
```
|
||||
|
||||
- The subnet needs no ranges. Without a static range, deployed machines keep
|
||||
getting their addresses from the site's server (`dhcp` links); set a static
|
||||
range aside, outside the site's pool, to give them fixed ones.
|
||||
- No `dhcp-host` lines are written for proxy subnets, and the router and DNS
|
||||
options come from the site's server.
|
||||
- `rollcall doctor` expects the foreign DHCP server here, and fails if none answers.
|
||||
- dnsmasq (2.91, still in 2.93) loops forever in its proxy code when a proxy
|
||||
range carries a `set:` tag: Rollcall's generated configuration leaves the tag
|
||||
out and limits the PXE services by interface instead.
|
||||
|
||||
Verified in the lab with a separate DHCP server on the provisioning network:
|
||||
commissioning and three deployments in a row through the proxy.
|
||||
|
||||
#### Fixed leases on the site's DHCP server (Kea, OpenWrt)
|
||||
|
||||
With a reservation driver, machines on proxy VLANs still get the addresses
|
||||
Rollcall plans: every such address becomes a fixed lease (the MAC of its
|
||||
interface, the address, the hostname) on the site's server. This is the
|
||||
counterpart of Infoblox's `fixedaddress`. Leases are reconciled on every
|
||||
address change and every `sync_interval`. Rollcall changes or deletes only
|
||||
its own leases; one held by someone else (same MAC or address) is reported
|
||||
as a conflict.
|
||||
|
||||
```json
|
||||
"dhcp_reservations": {
|
||||
"provider": "kea",
|
||||
"url": "http://kea.example:8000",
|
||||
"user": "rollcall",
|
||||
"password_file": "/etc/rollcall/kea.pw"
|
||||
}
|
||||
```
|
||||
|
||||
- **kea:** the control API with the `host_cmds` hook, open source since Kea
|
||||
3.0; Debian's Kea 2.6 lacks it and the driver says so.
|
||||
- Ownership goes in `user-context {"rollcall": machine}`.
|
||||
- With a hosts database the leases are kept there. Without one they go
|
||||
into the running configuration, and `config-write` saves them
|
||||
(`persist: false` turns that off).
|
||||
- **openwrt:** rpcd over HTTP (`url: http://router`). Fixed leases are
|
||||
`host` sections in `/etc/config/dhcp` with `option rollcall`, followed by
|
||||
`uci commit` and a dnsmasq reload. Give Rollcall its own rpcd login with
|
||||
only this:
|
||||
|
||||
```json
|
||||
/usr/share/rpcd/acl.d/rollcall.json
|
||||
{ "rollcall": { "description": "Rollcall fixed DHCP leases",
|
||||
"read": { "ubus": { "uci": ["get"] }, "uci": ["dhcp"] },
|
||||
"write": { "ubus": { "uci": ["add", "delete", "commit"], "rc": ["init"] }, "uci": ["dhcp"] } } }
|
||||
```
|
||||
|
||||
and a `config login` in `/etc/config/rpcd` (username, an MD5-crypt
|
||||
password, `list read/write 'rollcall'`).
|
||||
|
||||
```
|
||||
rollcall reservations # provider, last sync, conflicts
|
||||
rollcall reservations sync -dry
|
||||
```
|
||||
|
||||
IPv6 addresses are skipped: DHCPv6 reservations are keyed on DUIDs, not
|
||||
MACs. So are interfaces without a single MAC, such as bonds.
|
||||
|
||||
Verified in the lab against Kea 3.0.5 and OpenWrt 25.12.6: the machine got
|
||||
its planned address from the site's server and not from the pool.
|
||||
|
||||
## Certificate authority and TLS
|
||||
|
||||
The region is the installation's certificate authority, as the llmgw control
|
||||
@@ -473,13 +571,77 @@ that reaches the BMC network.
|
||||
| Driver | For |
|
||||
|---|---|
|
||||
| `ipmi` | IPMI 2.0 over LAN (ipmitool lanplus), any vendor; settings `cipher` (17 on newer BMCs), `privilege`, `interface` |
|
||||
| `redfish` | DMTF Redfish, any vendor: system discovered, ResetType chosen from what the BMC allows, boot override once (with If-Match where the BMC wants it) |
|
||||
| `idrac` | Dell iDRAC 8/9 (Redfish, `System.Embedded.1`) |
|
||||
| `ilo` | HPE iLO 4 (2.30+), 5, 6 (Redfish; no PowerCycle, so ForceRestart; PushPowerButton for soft off on older firmware) |
|
||||
| `supermicro`, `xcc`, `openbmc` | Supermicro, Lenovo XClarity, OpenBMC (Redfish) |
|
||||
| `redfish` and the vendor drivers below | DMTF Redfish (see below) |
|
||||
| `proxmox` | Proxmox VE VMs: API token (user = token id, password = secret), settings `node`, `vmid` |
|
||||
| `manual` | no power control; actions say so |
|
||||
|
||||
### Redfish across vendors
|
||||
|
||||
There is one Redfish implementation of the DMTF standard. These rules hold for
|
||||
every vendor:
|
||||
|
||||
- **Reset types and boot targets.** Read from what the BMC advertises
|
||||
(`@Redfish.AllowableValues` or the action's ActionInfo), never assumed.
|
||||
- **Boot override.** Target and enable go in one PATCH.
|
||||
- **If-Match.** Sent with the resource's ETag (header or `@odata.etag`). On
|
||||
412 it is retried with the strong form, then without; on 428 it is fetched
|
||||
first.
|
||||
- **Pending settings.** A BMC that keeps them elsewhere (`.../SD`,
|
||||
`@Redfish.Settings`) gets the same PATCH there.
|
||||
- **Busy answers.** Retried for up to a minute: the server is in POST, 409
|
||||
right after a power change, 503, "Retry in N seconds".
|
||||
- **202 answers.** Followed to their task monitor.
|
||||
- **Reset on the chassis.** Used when the system has no reset of its own.
|
||||
- **Missing power cycle.** ForceOff, wait for Off, then On.
|
||||
- **Paths from the BMC.** Only followed if they stay on the BMC and under
|
||||
`/redfish/`.
|
||||
|
||||
What a vendor documents differently is a profile in
|
||||
`cmd/rollcall/redfish.go`, with its sources. The profile decides the driver
|
||||
name, how the BMC is recognised, its usual system path, the boot mode rule, how
|
||||
to clear an override and the login method.
|
||||
|
||||
| Driver | Vendor | Boot mode in an override | Notes |
|
||||
|---|---|---|---|
|
||||
| `idrac` | Dell iDRAC 8/9/10 | not sent: a change starts a BIOS job | `System.Embedded.1`; iDRAC 10 takes boot settings at `.../Settings` |
|
||||
| `ilo` | HPE iLO 4 (2.30+), 5, 6 | current one, without it if refused | no PowerCycle (ForceRestart); no boot changes during POST (retried) |
|
||||
| `xcc` | Lenovo XCC, XCC2 | current one, without it if refused | override is Once only |
|
||||
| `supermicro` | Supermicro X11+, H12+ | always the current one | left out, some boards switch to Legacy |
|
||||
| `xfusion` | xFusion / Huawei iBMC | current one, without it if refused | session login; If-Match on every PATCH; ForcePowerCycle |
|
||||
| `kaytus` | KAYTUS / Inspur | current one, without it if refused | no public Redfish guide: reset may be on the chassis, a PATCH that worked can answer 400 |
|
||||
| `irmc` | Fujitsu / Fsas iRMC S5, S6 | not sent | `Systems/0`; ETag from the body; clearing is Target None (never Enabled Disabled); no GracefulShutdown (power button) |
|
||||
| `cimc` | Cisco UCS C-series CIMC | current one, without it if refused | `Systems/<serial>`, found through the collection |
|
||||
| `gigabyte` | GIGABYTE (MegaRAC SP-X) | always the current one | `Systems/Self`; If-Match required; short reset list |
|
||||
| `asrockrack` | ASRock Rack (MegaRAC) | always the current one | boot settings in `Systems/Self/SD` |
|
||||
| `openbmc` | OpenBMC (bmcweb) | current one, without it if refused | `Systems/system` |
|
||||
| `ami` | AMI MegaRAC SP-X boards | always the current one | `Systems/Self`; If-Match required |
|
||||
|
||||
Settings a machine can override:
|
||||
|
||||
| Setting | Effect |
|
||||
|---|---|
|
||||
| `system=` | the system path |
|
||||
| `boot_mode=UEFI\|Legacy` | always send this mode |
|
||||
| `auth=session\|basic` | login method |
|
||||
| `verify_tls=true` | check the BMC's certificate |
|
||||
|
||||
Every profile is tested against a fake service that behaves as its sources
|
||||
describe (`redfish_test.go`). The profiles have not yet run against the real
|
||||
machines.
|
||||
|
||||
**HTTPS BMCs (Redfish, Proxmox) are trusted by their certificate.** Before
|
||||
credentials first go to such a BMC, the rack controller reads its
|
||||
certificate with a TLS handshake that sends nothing. The region pins the
|
||||
SHA-256 on the BMC record as `extra tls_fingerprint`. A later connection
|
||||
showing a different certificate sends no credentials: the command fails
|
||||
with both fingerprints, and the machine gets the event
|
||||
`bmc certificate changed`.
|
||||
|
||||
To trust a replaced BMC, clear the pin with `rollcall bmc ID -set
|
||||
tls_fingerprint=`. Changing the address does the same, and so does a BMC
|
||||
firmware update run by Rollcall. To check against the system's CAs
|
||||
instead, set `verify_tls=true`.
|
||||
|
||||
After commissioning, a BMC still on IPMI is asked whether it speaks Redfish
|
||||
with the same account; if it does, the driver for its vendor takes over
|
||||
(event `bmc driver`). An operator's choice is never overridden.
|
||||
@@ -491,6 +653,74 @@ rollcall bmc m-1234 -driver redfish -set system=/redfish/v1/Systems/1 -set verif
|
||||
rollcall bmc m-1234 -probe # detect Redfish again
|
||||
```
|
||||
|
||||
### Firmware updates (Redfish)
|
||||
|
||||
Rollcall keeps firmware packages in a repository on the region, reads each
|
||||
machine's firmware inventory from its BMC, and runs updates as jobs.
|
||||
|
||||
```sh
|
||||
# on the region: a file (or from anywhere: an https URL, which the region downloads)
|
||||
sudo rollcall firmware import BIOS_R760_WN64_2.3.5.EXE -vendor idrac -component bios -version 2.3.5 -models R760 -sha256 ...
|
||||
rollcall firmware list
|
||||
rollcall firmware show m-1234 -refresh # what is installed, what the repository offers
|
||||
rollcall firmware update m-1234 idrac-bios-2.3.5-2c49752f -reboot -watch
|
||||
rollcall firmware update tag:r760 idrac-bmc-7.20.10.50-097a4a54
|
||||
```
|
||||
|
||||
**The package.** A package names the BMC driver it is for (`idrac`, `ilo`,
|
||||
`supermicro`, ...), a component (`bios`, `bmc`, `nic`, `raid`, `cpld`,
|
||||
`psu`, `drive`, `other`), the version the component reports once installed,
|
||||
and optionally the models it fits. An update refuses a package for another
|
||||
vendor or model unless `-force`. The BMC checks the package itself:
|
||||
signature, model and downgrade policy. An HPE `.fwpkg` is unpacked at
|
||||
import to the iLO `.bin` or UEFI `.signed.flash` inside, because iLO's
|
||||
SimpleUpdate accepts a `.fwpkg` and then flashes nothing.
|
||||
|
||||
**The job.** The region drives the update through the rack controller next
|
||||
to the BMC:
|
||||
|
||||
1. It hands the package over: an upload, or the BMC downloads it from the
|
||||
rack controller's `/firmware/`. Set `firmware_base=` on the BMC record
|
||||
when BMCs reach another address.
|
||||
2. It watches the BMC's task. A BMC that restarts after flashing itself is
|
||||
waited for, up to 25 minutes.
|
||||
3. It reads the inventory until the new version shows.
|
||||
|
||||
`-reboot` power-cycles the host when the update waits for a reset (BIOS on
|
||||
most vendors), then verifies. Without it, a staged update ends as
|
||||
"accepted, applies at the next reboot". A running update survives a region
|
||||
restart. It cannot be cancelled once the BMC has the package.
|
||||
|
||||
**How each vendor takes a package.** Each vendor profile in
|
||||
`cmd/rollcall/redfish_fwplans.go` carries its documented request, with
|
||||
sources. Without a profile, the standard methods are tried in this order:
|
||||
multipart push, SimpleUpdate pull, HttpPushUri.
|
||||
|
||||
| Driver | How | Packages | Notes |
|
||||
|---|---|---|---|
|
||||
| `idrac` | MultipartUpload (SimpleUpdate pull on older iDRAC 9) | DUP `.EXE`, `.d9` | OnReset by default; BIOS/RAID/NIC apply at the next reboot |
|
||||
| `ilo` | SimpleUpdate: iLO pulls over HTTP | iLO `.bin`, UEFI `.signed.flash` (from `.fwpkg`) | components only an OS agent flashes need SUM/iSUT |
|
||||
| `xcc` | `/mfwupdate`, after taking `HttpPushUriTargetsBusy` | `.uxz` | update XCC before UEFI |
|
||||
| `supermicro` | `/UpdateService/upload` with targets and preserve options | `.bin` | licence SFT-OOB-LIC/SFT-DCMS-SINGLE; host powered off for BIOS |
|
||||
| `gigabyte`, `ami`, `asrockrack` | `/UpdateService/upload` with `OemParameters` ImageType | `.ima`, `.ima_enc`, `.hpm`, `.bin` | the BMC restarts after a BMC flash; ASRock Rack unverified |
|
||||
| `openbmc` | update-multipart (BMC Immediate, others OnReset) | signed `.tar` | |
|
||||
| `xfusion` | upload to `/tmp/web`, then SimpleUpdate | `.hpm` | BMC, BIOS, CPLD |
|
||||
| `kaytus` | upload, then OEM SimpleUpdate (FlashItem) | `.hpm`, `.bin` | from KAYTUS's own SDK; BMC and BIOS |
|
||||
| `irmc` | OEM actions (FWUpdate, BiosUpdate) | iRMC `.bin`, BIOS `.UPC` | |
|
||||
| `cimc` | not supported | | Cisco documents only SCP pulls |
|
||||
|
||||
Per machine, on the BMC record:
|
||||
|
||||
| Setting | Effect |
|
||||
|---|---|
|
||||
| `fw_method=multipart\|simple\|push` | skip the vendor plan and use this standard method |
|
||||
| `fw_params={...}` | replace the plan's UpdateParameters (e.g. Supermicro Gen 13 BIOS preserve options) |
|
||||
| `firmware_base=http://ADDR` | the address BMCs download from |
|
||||
|
||||
The lab Redfish emulator has an UpdateService of the same shape: iDRAC
|
||||
profile multipart, iLO profile pull. The BMC version applies at once, the
|
||||
BIOS at the next reset.
|
||||
|
||||
The lab has `lab/pve-vbmc/pve_redfish.py`, a Redfish service per Proxmox VM
|
||||
with iDRAC, iLO and generic profiles, next to the IPMI emulator.
|
||||
|
||||
@@ -514,7 +744,7 @@ Verified in the lab: Debian 13 (with a bond and VLAN), Rocky 9 and AlmaLinux 9
|
||||
dual-stack on two segments, AAAA lookups through dnsmasq, the admin CLI over
|
||||
`https://[fdc3:...]`.
|
||||
|
||||
## External IPAM (NetBox; Infoblox-style)
|
||||
## External IPAM (NetBox, Infoblox)
|
||||
|
||||
Rollcall works on its own, but at a site with an established network plan the
|
||||
address authority is somewhere else. With `ipam` in `config.json` every
|
||||
@@ -558,9 +788,126 @@ rollcall ipam sync
|
||||
rollcall ipam prefixes | import
|
||||
```
|
||||
|
||||
### Devices in NetBox
|
||||
|
||||
With `device_sync` in the `ipam` block, the NetBox driver also keeps every
|
||||
commissioned machine as a `dcim` device:
|
||||
|
||||
```json
|
||||
"ipam": { "provider": "netbox", ..., "device_sync": { "site": "dc1", "role": "server" } }
|
||||
```
|
||||
|
||||
- **What is written.** Name (node name, else the machine ID), serial, status
|
||||
(deployed/booted/stateless are `active`, commissioned ones `staged`, failed
|
||||
jobs `failed`), the device type from the DMI manufacturer and product, the
|
||||
role (default `server`), interfaces with their MACs (NetBox 4.2+ MAC objects,
|
||||
set as primary MAC), Rollcall's addresses assigned to those interfaces, and
|
||||
the management address as primary IP. Machines still `new` are skipped.
|
||||
- **Location comes from NetBox.** The machine's building, room and rack are
|
||||
matched by name to a site, a location and a rack (in that site); the unit
|
||||
becomes the rack position, the asset tag the device's. Rollcall never creates
|
||||
sites, locations or racks; a machine whose building matches no site goes to
|
||||
`device_sync.site`, which must exist. Manufacturers, device types and the role
|
||||
are created when missing (a device needs them).
|
||||
- **Ownership as for addresses.** Devices carry the tag `rollcall` and the
|
||||
machine ID in `rollcall_machine` (enable the field for devices too). A device
|
||||
of the same name without them is reported as a conflict and left alone;
|
||||
Rollcall's devices whose machine is gone are deleted. Interfaces are removed
|
||||
only when they carry the tag.
|
||||
- **It runs after the address reconcile**, in the same loop; `rollcall ipam
|
||||
sync` reports both (`devices` in the JSON).
|
||||
|
||||
The NetBox user needs `view/add/change/delete` on `dcim.device`,
|
||||
`dcim.interface` and `dcim.macaddress`, `view/add` on `dcim.manufacturer`,
|
||||
`dcim.devicetype` and `dcim.devicerole`, and `view` on `dcim.location` and
|
||||
`dcim.rack`, on top of the IPAM permissions.
|
||||
|
||||
### Infoblox
|
||||
|
||||
The same contract over Infoblox WAPI (v2.12 by default; give a full
|
||||
`.../wapi/vX.Y` URL for another):
|
||||
|
||||
```json
|
||||
"ipam": {
|
||||
"provider": "infoblox",
|
||||
"url": "https://gm.example.com",
|
||||
"user": "rollcall",
|
||||
"password_file": "/etc/rollcall/infoblox.pw",
|
||||
"ca_file": "/etc/rollcall/infoblox-ca.pem",
|
||||
"dns_zone": "int.example.com",
|
||||
"network_view": "default",
|
||||
"dns_view": "default"
|
||||
}
|
||||
```
|
||||
|
||||
- **One host record per address.** Named by its DNS name (`rollcall-<address>`
|
||||
without one) and configured for DNS in `dns_view`, so Infoblox publishes the
|
||||
A/AAAA record itself; `"dns_view": "-"` keeps them IPAM-only. The zone must
|
||||
exist in that view.
|
||||
- **Ownership** is the extensible attribute `rollcall_machine` (define it in
|
||||
the grid as String). Rollcall changes it with `extattrs+`, so a host's other
|
||||
attributes stay; hosts without it are never changed or deleted.
|
||||
- **Taken** is anything else on the address: another host, a lease, a fixed
|
||||
address, a DHCP or reserved range, the network or broadcast address.
|
||||
- **Delegated networks** carry the extensible attribute `rollcall` (any value;
|
||||
define it as String); `Site` and `VLAN` attributes are shown when present.
|
||||
- The WAPI user needs read/write on host records and read on networks and
|
||||
IPv4/IPv6 addresses in the network view (and the DNS view when publishing).
|
||||
|
||||
Device sync (above) is NetBox-only.
|
||||
|
||||
The provider is an interface (reserve, owned records, update, delete,
|
||||
delegated prefixes); the NetBox driver is the first. An Infoblox WAPI driver
|
||||
maps them to `record:host`/`fixedaddress` with extensible attributes.
|
||||
delegated prefixes) with the NetBox and Infoblox drivers.
|
||||
|
||||
## DNS provider (PowerDNS, RFC 2136)
|
||||
|
||||
Without an IPAM authority whose DNS sync publishes names, Rollcall can write
|
||||
its machines' A/AAAA records into the site's DNS itself. Names follow the
|
||||
IPAM rule above (`<host>.<zone>`, `<host>-<iface>.<zone>`).
|
||||
|
||||
```json
|
||||
"dns": {
|
||||
"provider": "powerdns",
|
||||
"zone": "int.example.com",
|
||||
"url": "http://dns.int.example.com:8081",
|
||||
"api_key_file": "/etc/rollcall/pdns.key"
|
||||
}
|
||||
```
|
||||
|
||||
```json
|
||||
"dns": {
|
||||
"provider": "rfc2136",
|
||||
"zone": "int.example.com",
|
||||
"server": "ns1.int.example.com:53",
|
||||
"tsig_name": "rollcall-key",
|
||||
"tsig_algorithm": "hmac-sha256",
|
||||
"tsig_secret_file": "/etc/rollcall/tsig.secret"
|
||||
}
|
||||
```
|
||||
|
||||
- **powerdns** uses the HTTP API. Rollcall's RRsets carry a comment with
|
||||
account `rollcall`; RRsets without it are never changed or deleted.
|
||||
- **rfc2136** is a TSIG-signed dynamic update, for BIND, Knot, PowerDNS,
|
||||
Windows DNS and Infoblox. Next to each name Rollcall writes a marker
|
||||
`_rollcall.<name> TXT "heritage=rollcall"`. A change to an existing record
|
||||
needs the marker as a prerequisite; a new record needs the name to be
|
||||
free. The server checks both atomically, so a record someone else owns is
|
||||
never overwritten. What Rollcall wrote is kept in the database, and each
|
||||
sync queries those records. A record changed behind Rollcall's back is
|
||||
repaired when the marker is still there. When someone else's record has
|
||||
taken the name, it is reported as a conflict.
|
||||
For BIND, grant the key the zone: `update-policy { grant rollcall-key zonesub ANY; };`.
|
||||
|
||||
A sync runs on every allocation and release and every `sync_interval`
|
||||
(default 5m), with `ttl` defaulting to 300.
|
||||
|
||||
```
|
||||
rollcall dns # provider, last sync, conflicts
|
||||
rollcall dns sync -dry # what would change
|
||||
rollcall dns sync
|
||||
```
|
||||
|
||||
Conflicts and errors appear there, in the log and in `rollcall doctor`.
|
||||
|
||||
## Lab
|
||||
|
||||
|
||||
@@ -6,7 +6,9 @@ SRC=$(cd "$(dirname "$0")" && pwd)
|
||||
BUILD=${BUILD:-/srv/rollcall/build}
|
||||
OUT=${OUT:-/srv/rollcall/image}
|
||||
MIRROR=${MIRROR:-http://deb.debian.org/debian}
|
||||
AUTHKEYS=${AUTHKEYS:-/home/ilker/.ssh/authorized_keys}
|
||||
# root's SSH keys in the commissioning environment: none unless asked for (a lab
|
||||
# debugging aid; a published image must not carry anyone's keys)
|
||||
AUTHKEYS=${AUTHKEYS:-}
|
||||
|
||||
mkdir -p "$BUILD" "$OUT"
|
||||
cd "$BUILD"
|
||||
@@ -98,7 +100,7 @@ install -m 0755 "$SRC/rollcall-agent.py" config/includes.chroot/usr/local/bin/ro
|
||||
install -m 0755 "$SRC/rollcall-deploy.py" config/includes.chroot/usr/local/bin/rollcall-deploy.py
|
||||
install -m 0644 "$SRC/rollcall-agent.service" config/includes.chroot/etc/systemd/system/rollcall-agent.service
|
||||
printf 'ipmi_msghandler\nipmi_devintf\nipmi_si\nipmi_ssif\n' > config/includes.chroot/etc/modules-load.d/ipmi.conf
|
||||
if [ -r "$AUTHKEYS" ]; then
|
||||
if [ -n "$AUTHKEYS" ] && [ -r "$AUTHKEYS" ]; then
|
||||
install -m 0600 "$AUTHKEYS" config/includes.chroot/root/.ssh/authorized_keys
|
||||
chmod 0700 config/includes.chroot/root/.ssh
|
||||
fi
|
||||
@@ -125,5 +127,7 @@ for f in vmlinuz initrd.img filesystem.squashfs; do
|
||||
[ -n "$src" ] || { echo "missing $f in binary/live" >&2; exit 1; }
|
||||
install -m 0644 "$src" "$OUT/$f"
|
||||
done
|
||||
# checked by bootstrap.sh --image-url when another region copies this one
|
||||
(cd "$OUT" && sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS)
|
||||
ls -la "$OUT"
|
||||
echo "=== done $(date)"
|
||||
+47
-6
@@ -13,11 +13,13 @@ Kernel cmdline options:
|
||||
rollcall.mode=deploy rollcall.job=.. run rollcall-deploy.py instead (OS deployment / release job)
|
||||
"""
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import socket
|
||||
import ssl
|
||||
import string
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -399,6 +401,36 @@ def bmc_inband(user_name):
|
||||
AGENT_TOKEN = ""
|
||||
|
||||
|
||||
def tls_setup(opts, server):
|
||||
"""rollcall.tls=URL rollcall.ca=SHA256: the CA certificate is fetched and
|
||||
accepted only by that fingerprint; from then on everything goes over HTTPS
|
||||
with it. When TLS was asked for and does not work, nothing falls back to HTTP."""
|
||||
url, want = opts.get("tls"), str(opts.get("ca") or "").replace(":", "").lower()
|
||||
if not isinstance(url, str) or not url.startswith("https://") or len(want) != 64:
|
||||
return server, None, None
|
||||
pem = None
|
||||
for i in range(10):
|
||||
try:
|
||||
with urllib.request.urlopen(f"{server}/ca.pem", timeout=10) as r:
|
||||
pem = r.read().decode()
|
||||
break
|
||||
except (urllib.error.URLError, OSError) as e:
|
||||
log(f"ca.pem: {e}, retry {i + 1}/10")
|
||||
time.sleep(3)
|
||||
if pem is None:
|
||||
raise RuntimeError("cannot fetch the CA certificate")
|
||||
if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != want:
|
||||
raise RuntimeError("the CA certificate does not match rollcall.ca: not talking to this server")
|
||||
path = "/run/rollcall-ca.pem"
|
||||
with open(path, "w") as f:
|
||||
f.write(pem)
|
||||
log(f"talking to {url} over HTTPS (CA {want[:16]}...)")
|
||||
return url.rstrip("/"), ssl.create_default_context(cafile=path), path
|
||||
|
||||
|
||||
TLS = None # ssl context once tls_setup ran
|
||||
|
||||
|
||||
def agent_headers():
|
||||
h = {"Content-Type": "application/json"}
|
||||
if AGENT_TOKEN:
|
||||
@@ -409,14 +441,19 @@ def agent_headers():
|
||||
def bmc_hint(server, macs):
|
||||
try:
|
||||
req = urllib.request.Request(f"{server}/api/v1/bmc-hint?macs={','.join(macs)}", headers=agent_headers())
|
||||
with urllib.request.urlopen(req, timeout=10) as r:
|
||||
with urllib.request.urlopen(req, timeout=10, context=TLS) as r:
|
||||
h = json.load(r)
|
||||
except (urllib.error.URLError, json.JSONDecodeError, OSError):
|
||||
return None
|
||||
if not h or not h.get("address"):
|
||||
return None
|
||||
res = {"source": "hint", "address": h["address"], "port": h.get("port", 623),
|
||||
"user": h.get("user"), "password": h.get("password"), "errors": []}
|
||||
"user": h.get("user"), "errors": []}
|
||||
if not h.get("password"):
|
||||
# the region keeps the credentials and checks the BMC itself (rollcall >= 0.3)
|
||||
res["configured"] = True
|
||||
return res
|
||||
res["password"] = h["password"]
|
||||
base = ["-I", "lanplus", "-H", h["address"], "-p", str(h.get("port", 623)), "-U", h.get("user", ""),
|
||||
"-P", h.get("password", "")]
|
||||
rc, mc = ipmi(base + ["mc", "info"], timeout=20)
|
||||
@@ -434,7 +471,7 @@ def post(server, payload, tries=30):
|
||||
req = urllib.request.Request(f"{server}/api/v1/commission", data=data, headers=agent_headers(), method="POST")
|
||||
for i in range(tries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
with urllib.request.urlopen(req, timeout=30, context=TLS) as r:
|
||||
return json.load(r)
|
||||
except (urllib.error.URLError, OSError) as e:
|
||||
log(f"post failed ({e}), retry {i + 1}/{tries}")
|
||||
@@ -452,7 +489,8 @@ def main():
|
||||
log("rollcall.server= missing on kernel cmdline; nothing to do")
|
||||
return 2
|
||||
server = server.rstrip("/")
|
||||
global AGENT_TOKEN
|
||||
global AGENT_TOKEN, TLS
|
||||
server, TLS, _ = tls_setup(opts, server)
|
||||
AGENT_TOKEN = opts.get("token") if isinstance(opts.get("token"), str) else ""
|
||||
user_name = opts.get("bmcuser") or "commissioner"
|
||||
started = time.time()
|
||||
@@ -499,8 +537,11 @@ def main():
|
||||
log(f"server accepted: machine {resp.get('id')} state={resp.get('state')}")
|
||||
else:
|
||||
log("could not deliver inventory to server")
|
||||
with open("/root/inventory.json", "w") as f:
|
||||
json.dump(inv, f, indent=1)
|
||||
keep = json.loads(json.dumps(inv))
|
||||
if (keep.get("bmc") or {}).get("password"):
|
||||
keep["bmc"]["password"] = "<not kept>"
|
||||
with open(os.open("/root/inventory.json", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600), "w") as f:
|
||||
json.dump(keep, f, indent=1)
|
||||
if opts.get("nopoweroff") or not resp:
|
||||
log("staying up (nopoweroff or delivery failure)")
|
||||
return 0 if resp else 1
|
||||
|
||||
+81
-27
@@ -12,9 +12,11 @@ Everything the agent needs (image URL, checksum, disk, seed files) comes from
|
||||
GET /api/v1/jobs/self; it never decides policy itself.
|
||||
"""
|
||||
import glob
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import ssl
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
@@ -61,6 +63,35 @@ def cmdline():
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- server
|
||||
TLS, CAFILE = None, None
|
||||
|
||||
def tls_setup(opts, server):
|
||||
"""rollcall.tls=URL rollcall.ca=SHA256: the CA certificate is fetched and
|
||||
accepted only by that fingerprint; from then on everything goes over HTTPS
|
||||
with it. When TLS was asked for and does not work, nothing falls back to HTTP."""
|
||||
url, want = opts.get("tls"), str(opts.get("ca") or "").replace(":", "").lower()
|
||||
if not isinstance(url, str) or not url.startswith("https://") or len(want) != 64:
|
||||
return server, None, None
|
||||
pem = None
|
||||
for i in range(10):
|
||||
try:
|
||||
with urllib.request.urlopen(f"{server}/ca.pem", timeout=10) as r:
|
||||
pem = r.read().decode()
|
||||
break
|
||||
except (urllib.error.URLError, OSError) as e:
|
||||
log(f"ca.pem: {e}, retry {i + 1}/10")
|
||||
time.sleep(3)
|
||||
if pem is None:
|
||||
raise RuntimeError("cannot fetch the CA certificate")
|
||||
if hashlib.sha256(ssl.PEM_cert_to_DER_cert(pem)).hexdigest() != want:
|
||||
raise RuntimeError("the CA certificate does not match rollcall.ca: not talking to this server")
|
||||
path = "/run/rollcall-ca.pem"
|
||||
with open(path, "w") as f:
|
||||
f.write(pem)
|
||||
log(f"talking to {url} over HTTPS (CA {want[:16]}...)")
|
||||
return url.rstrip("/"), ssl.create_default_context(cafile=path), path
|
||||
|
||||
|
||||
def api(method, path, body=None, tries=20):
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
req = urllib.request.Request(SERVER + path, data=data, method=method,
|
||||
@@ -68,7 +99,7 @@ def api(method, path, body=None, tries=20):
|
||||
last = None
|
||||
for i in range(tries):
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30) as r:
|
||||
with urllib.request.urlopen(req, timeout=30, context=TLS) as r:
|
||||
return json.load(r)
|
||||
except urllib.error.HTTPError as e:
|
||||
if 400 <= e.code < 500:
|
||||
@@ -156,7 +187,9 @@ def find_disk(spec):
|
||||
log(f"{want} not found; serial {serial} is {d['path']}")
|
||||
return d
|
||||
raise RuntimeError(f"target disk {want} not present")
|
||||
cands = [d for d in disks if not d.get("rm") and d.get("tran") != "usb" and int(d["size"]) >= need]
|
||||
# never a USB stick or a SAN LUN by itself (the region picks the same way)
|
||||
remote = {"usb", "iscsi", "fc", "fcoe", "srp", "iser"}
|
||||
cands = [d for d in disks if not d.get("rm") and d.get("tran") not in remote and int(d["size"]) >= need]
|
||||
if not cands:
|
||||
raise RuntimeError(f"no disk >= {need} bytes")
|
||||
cands.sort(key=lambda d: (int(d["size"]), d["name"]))
|
||||
@@ -182,31 +215,48 @@ def release_disk(path):
|
||||
# --------------------------------------------------------------------------- deploy
|
||||
def write_image(spec, disk, prog):
|
||||
url, want, size = spec["image_url"], spec["sha256"], int(spec.get("size") or 0)
|
||||
sumf, pvf = "/run/rollcall.sum", "/run/rollcall.pv"
|
||||
for f in (sumf, pvf):
|
||||
if os.path.exists(f):
|
||||
os.remove(f)
|
||||
pipeline = (f"set -o pipefail; curl -fsSL --retry 5 --retry-connrefused '{url}' "
|
||||
f"| tee >(sha256sum > {sumf}) "
|
||||
f"| pv -n -f -i 2 -s {size} 2>{pvf} "
|
||||
f"| zstd -dc -T0 "
|
||||
f"| dd of='{disk}' bs=4M iflag=fullblock oflag=direct conv=fsync status=none")
|
||||
if not re.match(r"^https?://", url):
|
||||
raise RuntimeError(f"image url {url!r} is not http(s)")
|
||||
pvf = "/run/rollcall.pv"
|
||||
if os.path.exists(pvf):
|
||||
os.remove(pvf)
|
||||
# values go in as arguments, never into the shell text; the checksum is taken
|
||||
# here, over exactly the bytes handed to the writer
|
||||
down = subprocess.Popen(
|
||||
["bash", "-c", 'set -o pipefail; pv -n -f -i 2 -s "$1" 2>"$2" | zstd -dc -T0 '
|
||||
'| dd of="$3" bs=4M iflag=fullblock oflag=direct conv=fsync status=none', "write", str(size), pvf, disk],
|
||||
stdin=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
cacert = ["--cacert", CAFILE] if CAFILE and url.startswith("https://") else []
|
||||
curl = subprocess.Popen(["curl", "-fsSL", "--retry", "5", "--retry-connrefused"] + cacert + ["--", url],
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
log(f"writing {url} -> {disk}")
|
||||
proc = subprocess.Popen(["bash", "-c", pipeline], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
||||
pct = 0
|
||||
while proc.poll() is None:
|
||||
time.sleep(2)
|
||||
try:
|
||||
with open(pvf) as f:
|
||||
lines = f.read().split()
|
||||
if lines:
|
||||
pct = int(lines[-1])
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
prog.set("writing", min(pct, 99), f"{pct}% of {size // (1 << 20)} MiB written to {disk}")
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"image write failed rc={proc.returncode}: {proc.stderr.read().strip()[-400:]}")
|
||||
got = readf(sumf).split()[0] if readf(sumf) else ""
|
||||
h, pct, shown = hashlib.sha256(), 0, 0.0
|
||||
try:
|
||||
while True:
|
||||
b = curl.stdout.read(1 << 20)
|
||||
if not b:
|
||||
break
|
||||
h.update(b)
|
||||
down.stdin.write(b)
|
||||
if time.time() - shown >= 2:
|
||||
shown = time.time()
|
||||
try:
|
||||
with open(pvf) as f:
|
||||
lines = f.read().split()
|
||||
if lines:
|
||||
pct = int(lines[-1])
|
||||
except (OSError, ValueError):
|
||||
pass
|
||||
prog.set("writing", min(pct, 99), f"{pct}% of {size // (1 << 20)} MiB written to {disk}")
|
||||
down.stdin.close()
|
||||
except BrokenPipeError:
|
||||
pass
|
||||
crc, drc = curl.wait(), down.wait()
|
||||
if crc != 0:
|
||||
raise RuntimeError(f"download failed rc={crc}: {curl.stderr.read().decode(errors='replace').strip()[-400:]}")
|
||||
if drc != 0:
|
||||
raise RuntimeError(f"image write failed rc={drc}: {down.stderr.read().decode(errors='replace').strip()[-400:]}")
|
||||
got = h.hexdigest()
|
||||
if got != want:
|
||||
raise RuntimeError(f"sha256 mismatch: got {got} want {want}")
|
||||
run(["sync"])
|
||||
@@ -244,7 +294,9 @@ def seed(root, files):
|
||||
if not os.path.isdir(f"{mnt}/etc"):
|
||||
raise RuntimeError(f"{root['path']} has no /etc; not a root filesystem")
|
||||
for f in files:
|
||||
dst = os.path.join(mnt, f["path"].lstrip("/"))
|
||||
dst = os.path.realpath(os.path.join(mnt, f["path"].lstrip("/")))
|
||||
if not dst.startswith(mnt + "/"):
|
||||
raise RuntimeError(f"seed file {f['path']!r} is outside the root filesystem")
|
||||
os.makedirs(os.path.dirname(dst), exist_ok=True)
|
||||
with open(dst, "w") as fh:
|
||||
fh.write(f["content"])
|
||||
@@ -349,6 +401,8 @@ def main():
|
||||
global SERVER, TOKEN
|
||||
opts = cmdline()
|
||||
SERVER = (opts.get("server") or "").rstrip("/")
|
||||
global TLS, CAFILE
|
||||
SERVER, TLS, CAFILE = tls_setup(opts, SERVER)
|
||||
TOKEN = opts.get("job") if isinstance(opts.get("job"), str) else ""
|
||||
if not SERVER or not TOKEN:
|
||||
log("rollcall.server= and rollcall.job= required")
|
||||
|
||||
@@ -117,12 +117,28 @@ func (a *App) dropAdminCertCache() {
|
||||
a.adminCC.mu.Unlock()
|
||||
}
|
||||
|
||||
// rootIssued is the client certificate when the Rollcall root signed it
|
||||
// itself. Admin and rack identities are never accepted through an
|
||||
// intermediate: a delegated (ACME) intermediate could otherwise mint
|
||||
// CN=admin:... - its name constraints cover SANs, not the CN.
|
||||
func rootIssued(r *http.Request) *x509.Certificate {
|
||||
if r.TLS == nil {
|
||||
return nil
|
||||
}
|
||||
for _, ch := range r.TLS.VerifiedChains {
|
||||
if len(ch) == 2 {
|
||||
return ch[0]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// peerAdmin is the admin name of a verified client certificate, "" otherwise.
|
||||
func (a *App) peerAdmin(r *http.Request) string {
|
||||
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 || len(r.TLS.PeerCertificates) == 0 {
|
||||
c := rootIssued(r)
|
||||
if c == nil {
|
||||
return ""
|
||||
}
|
||||
c := r.TLS.PeerCertificates[0]
|
||||
name, ok := strings.CutPrefix(c.Subject.CommonName, "admin:")
|
||||
if !ok || name == "" {
|
||||
return ""
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
package main
|
||||
|
||||
// Where machine-side tokens may be used from. Job tokens (rollcall.job=) and
|
||||
// node tokens (rollcall.node=) travel in the per-MAC pxelinux file, which any
|
||||
// host on the provisioning network can fetch over TFTP; without a hardware
|
||||
// root of trust the machine holds no secret of its own. What the region does
|
||||
// know is the address it gives the machine (a static DHCP lease from the
|
||||
// network model), so a token is only accepted from there. A machine without
|
||||
// a fixed address (proxy DHCP, a site's DHCP server) binds a job to the
|
||||
// first address that uses it. Refusals are events on the machine.
|
||||
//
|
||||
// agent_source_check: false in config.json turns this off (NAT between the
|
||||
// machines and Rollcall).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// clientIP is the machine's address: a remote rack controller proxies the
|
||||
// node API and appends it to X-Forwarded-For (the last entry is the rack's).
|
||||
func (a *App) clientIP(r *http.Request) string {
|
||||
host, _, err := net.SplitHostPort(r.RemoteAddr)
|
||||
if err != nil {
|
||||
host = r.RemoteAddr
|
||||
}
|
||||
if rack := a.fromRack(r); rack != "" && rack != a.cfg.RackName {
|
||||
if xff := r.Header.Get("X-Forwarded-For"); xff != "" {
|
||||
parts := strings.Split(xff, ",")
|
||||
host = strings.TrimSpace(parts[len(parts)-1])
|
||||
}
|
||||
}
|
||||
if ad, ok := parseAddr(host); ok {
|
||||
return ad.String()
|
||||
}
|
||||
return host
|
||||
}
|
||||
|
||||
// expectedIPs are the addresses the region hands this machine itself.
|
||||
func (a *App) expectedIPs(ctx context.Context, m *Machine) []string {
|
||||
var out []string
|
||||
add := func(ip string) {
|
||||
if ad, ok := parseAddr(ip); ok && !hasTag(out, ad.String()) {
|
||||
out = append(out, ad.String())
|
||||
}
|
||||
}
|
||||
subnets := a.subnetsSnapshot()
|
||||
if ips, err := a.store.IPs(ctx, "", m.ID); err == nil {
|
||||
for _, x := range ips {
|
||||
if sn := subnetFor(subnets, x.IP); sn == nil || sn.DHCPMode != "proxy" {
|
||||
add(x.IP)
|
||||
}
|
||||
}
|
||||
}
|
||||
if mg := deref(m.MgmtIP); mg != "" {
|
||||
if sn := subnetFor(subnets, mg); sn == nil || sn.DHCPMode != "proxy" {
|
||||
add(mg)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (s *Store) bindJobIP(ctx context.Context, id, ip string) (string, error) {
|
||||
var bound string
|
||||
err := s.pool.QueryRow(ctx, `UPDATE jobs SET bound_ip=coalesce(bound_ip, $2) WHERE id=$1 RETURNING bound_ip`, id, ip).Scan(&bound)
|
||||
return bound, err
|
||||
}
|
||||
|
||||
// tokenSource checks where a machine token is used from. job is nil for node
|
||||
// tokens and phone-home, which are checked against fixed addresses only.
|
||||
func (a *App) tokenSource(ctx context.Context, r *http.Request, m *Machine, j *Job, what string) error {
|
||||
if a.cfg.AgentSourceCheck != nil && !*a.cfg.AgentSourceCheck {
|
||||
return nil
|
||||
}
|
||||
ip := a.clientIP(r)
|
||||
want := a.expectedIPs(ctx, m)
|
||||
var err error
|
||||
switch {
|
||||
case len(want) > 0:
|
||||
if !hasTag(want, ip) {
|
||||
err = fmt.Errorf("%s token used from %s; this machine is %s", what, ip, strings.Join(want, ", "))
|
||||
}
|
||||
case j != nil:
|
||||
bound, berr := a.store.bindJobIP(ctx, j.ID, ip)
|
||||
if berr != nil {
|
||||
return berr
|
||||
}
|
||||
if bound != ip {
|
||||
err = fmt.Errorf("%s token used from %s; job %s is bound to %s, where it was first used", what, ip, j.ID, bound)
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
_ = a.store.AddEvent(ctx, m.ID, "token refused", err.Error())
|
||||
log.Printf("%s: %v", m.ID, err)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
package main
|
||||
|
||||
// TLS for the agents (commissioning, deploy, stateless nodes). Rack
|
||||
// controllers serve the node API and images over HTTPS too, with a server
|
||||
// certificate the region issues for the rack's own address only (never one
|
||||
// of the region's). The pxelinux command line names the HTTPS URL and the
|
||||
// CA's fingerprint (rollcall.tls=, rollcall.ca=); an agent fetches the CA
|
||||
// certificate, accepts it only by that fingerprint, and then talks HTTPS
|
||||
// with it. The embedded rack uses the region's own certificate.
|
||||
//
|
||||
// This keeps what agents send and receive (BMC passwords, seed files) from
|
||||
// anyone who can only listen, and from an HTTP server posing as the rack.
|
||||
// PXE itself (TFTP, the kernel, the initrd) stays unauthenticated: an
|
||||
// attacker who can rewrite the boot files on the wire is not stopped here;
|
||||
// that takes UEFI Secure Boot with signed images.
|
||||
|
||||
import (
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// rackServerCN is the subject of a rack controller's server certificate;
|
||||
// regionLeaf and the CLI refuse it as the region.
|
||||
const rackServerCN = "rack-server:"
|
||||
|
||||
// handleRackServerCert issues a rack's server certificate for the host of its
|
||||
// registered URL: {"csr"} -> {"cert"}.
|
||||
func (a *App) handleRackServerCert(w http.ResponseWriter, r *http.Request, rack string) {
|
||||
if a.peerRack(r) != rack || a.ca == nil {
|
||||
fail(w, 401, errors.New("a server certificate needs the rack's own certificate"))
|
||||
return
|
||||
}
|
||||
rc, ok := a.racksSnapshot()[rack]
|
||||
if !ok {
|
||||
fail(w, 409, fmt.Errorf("rack controller %s is not registered", rack))
|
||||
return
|
||||
}
|
||||
host, err := rackHost(rc.URL)
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
if a.isRegionName(host) {
|
||||
fail(w, 403, fmt.Errorf("%s is the region's own address: no rack certificate for it", host))
|
||||
return
|
||||
}
|
||||
var req enrollRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
csr, err := parseCSR(req.CSR)
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
certPEM, cert, err := a.ca.sign(csr.PublicKey, rackServerCN+rack, []string{host}, x509.ExtKeyUsageServerAuth)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
log.Printf("rack controller %s: server certificate for %s until %s", rack, host, cert.NotAfter.Format("2006-01-02"))
|
||||
writeJSON(w, 200, obj{"cert": string(certPEM), "ca": string(a.ca.PEM)})
|
||||
}
|
||||
|
||||
// rackHost is the host part of a rack URL (an address or a name).
|
||||
func rackHost(raw string) (string, error) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil || u.Hostname() == "" {
|
||||
return "", fmt.Errorf("rack URL %q has no host", raw)
|
||||
}
|
||||
return u.Hostname(), nil
|
||||
}
|
||||
|
||||
// isRegionName: one of the names or addresses the region's own certificate covers.
|
||||
func (a *App) isRegionName(host string) bool {
|
||||
h := strings.ToLower(strings.Trim(host, "[]"))
|
||||
if ad, ok := parseAddr(h); ok {
|
||||
h = ad.String()
|
||||
}
|
||||
for _, n := range append(hostAddresses(), a.cfg.TLSNames...) {
|
||||
n = strings.ToLower(n)
|
||||
if ad, ok := parseAddr(n); ok {
|
||||
n = ad.String()
|
||||
}
|
||||
if n == h {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if su, err := url.Parse(a.cfg.ServerURL); err == nil && strings.EqualFold(su.Hostname(), h) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// validTLSURL: what a rack reports must be https on its own URL's host.
|
||||
func validTLSURL(rc RackController) string {
|
||||
if rc.TLSURL == "" {
|
||||
return ""
|
||||
}
|
||||
u, err := url.Parse(rc.TLSURL)
|
||||
host, herr := rackHost(rc.URL)
|
||||
if err != nil || herr != nil || u.Scheme != "https" || !strings.EqualFold(u.Hostname(), host) || u.Path != "" && u.Path != "/" {
|
||||
return ""
|
||||
}
|
||||
return strings.TrimRight(rc.TLSURL, "/")
|
||||
}
|
||||
|
||||
// agentTLS is the HTTPS base agents behind this base URL use, "" when none.
|
||||
func (a *App) agentTLS(base string) string {
|
||||
if a.ca == nil {
|
||||
return ""
|
||||
}
|
||||
if base == a.cfg.ServerURL { // the embedded rack: the region's own listener
|
||||
if a.cfg.TLSListen == "" {
|
||||
return ""
|
||||
}
|
||||
u, err := url.Parse(base)
|
||||
if err != nil || u.Hostname() == "" {
|
||||
return ""
|
||||
}
|
||||
_, port, _ := net.SplitHostPort(a.cfg.TLSListen)
|
||||
if port == "" || port == "443" {
|
||||
return "https://" + hostPort(u.Hostname(), "")
|
||||
}
|
||||
return "https://" + hostPort(u.Hostname(), port)
|
||||
}
|
||||
for _, rc := range a.racksSnapshot() {
|
||||
if strings.TrimRight(rc.URL, "/") == base {
|
||||
return validTLSURL(rc)
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func hostPort(h, port string) string {
|
||||
if strings.Contains(h, ":") {
|
||||
h = "[" + h + "]"
|
||||
}
|
||||
if port == "" {
|
||||
return h
|
||||
}
|
||||
return h + ":" + port
|
||||
}
|
||||
|
||||
// agentTLSArgs are the command-line words that switch an agent to HTTPS.
|
||||
func (a *App) agentTLSArgs(base string) []string {
|
||||
t := a.agentTLS(base)
|
||||
if t == "" {
|
||||
return nil
|
||||
}
|
||||
fp := certFingerprint(a.ca.Cert)
|
||||
return []string{"rollcall.tls=" + t, "rollcall.ca=" + hex.EncodeToString(fp[:])}
|
||||
}
|
||||
|
||||
// overTLS: the agent's request came over HTTPS (to the region, or to a rack
|
||||
// controller that says so in X-Forwarded-Proto).
|
||||
func (a *App) overTLS(r *http.Request) bool {
|
||||
if rack := a.fromRack(r); rack != "" && rack != a.cfg.RackName {
|
||||
return r.Header.Get("X-Forwarded-Proto") == "https"
|
||||
}
|
||||
return r.TLS != nil
|
||||
}
|
||||
+81
-37
@@ -209,7 +209,14 @@ func parsePermits(list string) (dns []string, nets []*net.IPNet, err error) {
|
||||
if net.ParseIP(p) != nil {
|
||||
return nil, nil, fmt.Errorf("%s: give an address range as CIDR", p)
|
||||
}
|
||||
dns = append(dns, strings.TrimPrefix(strings.ToLower(p), "*."))
|
||||
d := strings.Trim(strings.TrimPrefix(strings.ToLower(p), "*."), ".")
|
||||
if !validDomain(d) {
|
||||
return nil, nil, fmt.Errorf("%s: give a domain (int.example.com) or a CIDR", p)
|
||||
}
|
||||
dns = append(dns, d)
|
||||
}
|
||||
if len(dns)+len(nets) == 0 {
|
||||
return nil, nil, errors.New("give at least one permitted domain or CIDR")
|
||||
}
|
||||
return dns, nets, nil
|
||||
}
|
||||
@@ -233,7 +240,16 @@ func (ca *CA) intermediate(pub crypto.PublicKey, cn string, dns []string, nets [
|
||||
tmpl := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: cn, Organization: []string{"rollcall"}},
|
||||
NotBefore: time.Now().Add(-backdate), NotAfter: time.Now().Add(validity),
|
||||
KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign | x509.KeyUsageDigitalSignature, BasicConstraintsValid: true, IsCA: true, MaxPathLenZero: true,
|
||||
PermittedDNSDomainsCritical: len(dns)+len(nets) > 0, PermittedDNSDomains: dns, PermittedIPRanges: nets}
|
||||
PermittedDNSDomainsCritical: true, PermittedDNSDomains: dns, PermittedIPRanges: nets,
|
||||
// server certificates only: nothing it issues can be a client (admin, rack) identity
|
||||
ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}
|
||||
// a constraint on one name type leaves the other open: close it
|
||||
if len(nets) == 0 {
|
||||
tmpl.ExcludedIPRanges = []*net.IPNet{{IP: net.IPv4zero, Mask: net.CIDRMask(0, 32)}, {IP: net.IPv6zero, Mask: net.CIDRMask(0, 128)}}
|
||||
}
|
||||
if len(dns) == 0 {
|
||||
tmpl.PermittedDNSDomains = []string{"invalid"} // RFC 6761: no name below it exists
|
||||
}
|
||||
der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Cert, pub, ca.Key)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
@@ -357,6 +373,17 @@ func parseJoinCode(s string) (joinCode, error) {
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// regionLeaf accepts the region's own certificate only: "rollcall region ...",
|
||||
// signed by the root itself - not a rack's, not one from a delegated intermediate.
|
||||
func regionLeaf(cs tls.ConnectionState) error {
|
||||
for _, ch := range cs.VerifiedChains {
|
||||
if len(ch) == 2 && strings.HasPrefix(ch[0].Subject.CommonName, "rollcall region ") {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
return errors.New("the server's certificate is not the region's")
|
||||
}
|
||||
|
||||
// pinnedTLS trusts a server whose chain carries the CA with this fingerprint.
|
||||
func pinnedTLS(fp [16]byte) *tls.Config {
|
||||
return &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: true, //nolint:gosec // verified below against the pinned CA
|
||||
@@ -374,8 +401,11 @@ func pinnedTLS(fp [16]byte) *tls.Config {
|
||||
if c.IsCA && string(sum[:16]) == string(fp[:]) {
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(c)
|
||||
_, err := leaf.Verify(x509.VerifyOptions{Roots: pool})
|
||||
return err
|
||||
chains, err := leaf.Verify(x509.VerifyOptions{Roots: pool})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return regionLeaf(tls.ConnectionState{VerifiedChains: chains})
|
||||
}
|
||||
}
|
||||
return errors.New("the server's certificate is not from the authority this code is for")
|
||||
@@ -397,15 +427,30 @@ func (s *Store) UseJoinToken(ctx context.Context, hash string) (string, error) {
|
||||
return name, nil
|
||||
}
|
||||
|
||||
func (s *Store) SetRackCert(ctx context.Context, name, serial string, expires time.Time) error {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET cert_serial=$2, cert_expires=$3 WHERE name=$1`, name, serial, expires)
|
||||
// rackCertGrace: after a renewal the previous certificate still works this
|
||||
// long, so a renewal answer lost on the way does not lock the rack out.
|
||||
const rackCertGrace = 7 * 24 * time.Hour
|
||||
|
||||
// SetRackCert records a rack's certificate. A renewal keeps the previous
|
||||
// serial for rackCertGrace; an enrollment (a new join) does not.
|
||||
func (s *Store) SetRackCert(ctx context.Context, name, serial string, expires time.Time, renewal bool) error {
|
||||
if renewal {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET prev_cert_serial=cert_serial, prev_cert_until=now()+$4::interval,
|
||||
cert_serial=$2, cert_expires=$3 WHERE name=$1`, name, serial, expires, fmt.Sprintf("%d seconds", int(rackCertGrace.Seconds())))
|
||||
return err
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET cert_serial=$2, cert_expires=$3, prev_cert_serial=NULL, prev_cert_until=NULL WHERE name=$1`, name, serial, expires)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Store) RackCertSerial(ctx context.Context, name string) string {
|
||||
var serial string
|
||||
_ = s.pool.QueryRow(ctx, `SELECT coalesce(cert_serial,'') FROM rack_controllers WHERE name=$1`, name).Scan(&serial)
|
||||
return serial
|
||||
// rackCertOK: the serial is the rack's current certificate, or the previous one within the grace period.
|
||||
func (s *Store) rackCertOK(ctx context.Context, name, serial string) bool {
|
||||
var cur, prev string
|
||||
var until *time.Time
|
||||
if err := s.pool.QueryRow(ctx, `SELECT coalesce(cert_serial,''), coalesce(prev_cert_serial,''), prev_cert_until FROM rack_controllers WHERE name=$1`, name).Scan(&cur, &prev, &until); err != nil {
|
||||
return false
|
||||
}
|
||||
return serial != "" && (serial == cur || (serial == prev && until != nil && time.Now().Before(*until)))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- HTTP
|
||||
@@ -413,15 +458,15 @@ func (s *Store) RackCertSerial(ctx context.Context, name string) string {
|
||||
// peerRack is the rack controller a mutual-TLS request comes from, if any:
|
||||
// a verified client certificate whose serial is the one on record.
|
||||
func (a *App) peerRack(r *http.Request) string {
|
||||
if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 || len(r.TLS.PeerCertificates) == 0 {
|
||||
c := rootIssued(r)
|
||||
if c == nil {
|
||||
return ""
|
||||
}
|
||||
c := r.TLS.PeerCertificates[0]
|
||||
name := strings.TrimPrefix(c.Subject.CommonName, "rack:")
|
||||
if name == c.Subject.CommonName { // only rack certificates
|
||||
return ""
|
||||
}
|
||||
if a.store.RackCertSerial(r.Context(), name) != c.SerialNumber.Text(16) {
|
||||
if !a.store.rackCertOK(r.Context(), name, c.SerialNumber.Text(16)) {
|
||||
return ""
|
||||
}
|
||||
return name
|
||||
@@ -487,32 +532,26 @@ type enrollRequest struct {
|
||||
Rack RackController `json:"rack"`
|
||||
}
|
||||
|
||||
func (a *App) signRackCSR(csrPEM, name string, rc RackController) ([]byte, *x509.Certificate, error) {
|
||||
// parseCSR reads a PEM certificate request; only its key is used.
|
||||
func parseCSR(csrPEM string) (*x509.CertificateRequest, error) {
|
||||
blk, _ := pem.Decode([]byte(csrPEM))
|
||||
if blk == nil {
|
||||
return nil, nil, errors.New("csr is not PEM")
|
||||
return nil, errors.New("csr is not PEM")
|
||||
}
|
||||
csr, err := x509.ParseCertificateRequest(blk.Bytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return csr, csr.CheckSignature()
|
||||
}
|
||||
|
||||
func (a *App) signRackCSR(csrPEM, name string) ([]byte, *x509.Certificate, error) {
|
||||
csr, err := parseCSR(csrPEM)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if err := csr.CheckSignature(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
hosts := []string{name}
|
||||
if u := strings.TrimPrefix(strings.TrimPrefix(rc.URL, "http://"), "https://"); u != "" {
|
||||
if h, _, err := net.SplitHostPort(u); err == nil {
|
||||
hosts = append(hosts, h)
|
||||
} else {
|
||||
hosts = append(hosts, strings.Trim(u, "/[]"))
|
||||
}
|
||||
}
|
||||
for _, i := range rc.Interfaces {
|
||||
for _, ad := range i.Addrs {
|
||||
hosts = append(hosts, stripPrefix(ad))
|
||||
}
|
||||
}
|
||||
return a.ca.sign(csr.PublicKey, "rack:"+name, hosts, x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth)
|
||||
// racks serve no TLS: a client identity only, no names, so a rack cannot pose as the region
|
||||
return a.ca.sign(csr.PublicKey, "rack:"+name, nil, x509.ExtKeyUsageClientAuth)
|
||||
}
|
||||
|
||||
// handleRackEnroll: a rack controller presents a join code's secret and a CSR
|
||||
@@ -546,7 +585,13 @@ func (a *App) handleRackEnroll(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 400, fmt.Errorf("bad rack controller name %q", name))
|
||||
return
|
||||
}
|
||||
certPEM, cert, err := a.signRackCSR(req.CSR, name, req.Rack)
|
||||
// a code made for one name may re-enroll that rack; an unbound code may not
|
||||
// take over a name in use (its machines, BMC jobs and certificate)
|
||||
if _, exists := a.racksSnapshot()[name]; exists && bound == "" {
|
||||
fail(w, 409, fmt.Errorf("rack controller %s exists; make a code for it: rollcall controller join-code -name %s", name, name))
|
||||
return
|
||||
}
|
||||
certPEM, cert, err := a.signRackCSR(req.CSR, name)
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
@@ -557,7 +602,7 @@ func (a *App) handleRackEnroll(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
if err := a.store.SetRackCert(r.Context(), name, cert.SerialNumber.Text(16), cert.NotAfter); err != nil {
|
||||
if err := a.store.SetRackCert(r.Context(), name, cert.SerialNumber.Text(16), cert.NotAfter, false); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
@@ -578,13 +623,12 @@ func (a *App) handleRackRenew(w http.ResponseWriter, r *http.Request, rack strin
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
rc := a.racksSnapshot()[rack]
|
||||
certPEM, cert, err := a.signRackCSR(req.CSR, rack, rc)
|
||||
certPEM, cert, err := a.signRackCSR(req.CSR, rack)
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
if err := a.store.SetRackCert(r.Context(), rack, cert.SerialNumber.Text(16), cert.NotAfter); err != nil {
|
||||
if err := a.store.SetRackCert(r.Context(), rack, cert.SerialNumber.Text(16), cert.NotAfter, true); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
|
||||
+79
-2
@@ -9,6 +9,8 @@ import (
|
||||
"crypto/x509"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -91,7 +93,7 @@ func TestSignRackCSR(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_, cert, err := a.signRackCSR(csr, "hall-b", RackController{URL: "http://10.213.78.1", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"fd00:213:78::1/64"}}}})
|
||||
_, cert, err := a.signRackCSR(csr, "hall-b")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@@ -100,9 +102,13 @@ func TestSignRackCSR(t *testing.T) {
|
||||
if _, err := cert.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil {
|
||||
t.Errorf("rack certificate is no client certificate: %v", err)
|
||||
}
|
||||
if cert.Subject.CommonName != "rack:hall-b" || cert.VerifyHostname("10.213.78.1") != nil || cert.VerifyHostname("fd00:213:78::1") != nil {
|
||||
// a rack is a client only: no names, no server use, so it cannot pose as the region
|
||||
if cert.Subject.CommonName != "rack:hall-b" || len(cert.IPAddresses)+len(cert.DNSNames) > 0 {
|
||||
t.Errorf("subject %s, SANs %v %v", cert.Subject.CommonName, cert.IPAddresses, cert.DNSNames)
|
||||
}
|
||||
if _, err := cert.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}}); err == nil {
|
||||
t.Error("rack certificate verifies as a server certificate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestIntermediate(t *testing.T) {
|
||||
@@ -146,6 +152,30 @@ func TestIntermediate(t *testing.T) {
|
||||
if err := verify("node001.int.example.com", "192.168.1.1"); err == nil {
|
||||
t.Error("leaf outside the IP constraint verified")
|
||||
}
|
||||
// nothing it issues is a client identity (admin:, rack:)
|
||||
if _, leaf, err := sub.sign(&ikey.PublicKey, "admin:mallory", nil, x509.ExtKeyUsageClientAuth); err == nil {
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, Intermediates: inters, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err == nil {
|
||||
t.Error("the intermediate issued a client certificate that verifies")
|
||||
}
|
||||
}
|
||||
// DNS only: no address may be certified (the region's IP, say)
|
||||
dnsOnly, _, _ := parsePermits("int.example.com")
|
||||
_, ic2, err := root.intermediate(&ikey.PublicKey, "dns-only", dnsOnly, nil, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
inters2 := x509.NewCertPool()
|
||||
inters2.AddCert(ic2)
|
||||
if _, leaf, err := (&CA{Cert: ic2, Key: ikey}).sign(&ikey.PublicKey, "x", []string{"10.0.0.26"}, x509.ExtKeyUsageServerAuth); err == nil {
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{Roots: roots, Intermediates: inters2}); err == nil {
|
||||
t.Error("a DNS-only intermediate certified an IP address")
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"*.", "", "foo bar"} {
|
||||
if _, _, err := parsePermits(bad); err == nil {
|
||||
t.Errorf("permit %q accepted", bad)
|
||||
}
|
||||
}
|
||||
// a second level below the intermediate is refused by path length 0
|
||||
k2, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
_, c2, err := sub.intermediate(&k2.PublicKey, "deeper", dns, nil, time.Hour)
|
||||
@@ -234,3 +264,50 @@ func TestAdminClientCertificate(t *testing.T) {
|
||||
t.Error("token no longer works")
|
||||
}
|
||||
}
|
||||
|
||||
// A renewed certificate is kept in the rack's own directory and preferred
|
||||
// when it is this rack's, from the CA and newer.
|
||||
func TestRackRenewedCertificate(t *testing.T) {
|
||||
ca, _ := newCA("test CA")
|
||||
a := &App{ca: ca, cfg: &Config{}}
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(ca.Cert)
|
||||
issue := func(name string) ([]byte, []byte, *x509.Certificate) {
|
||||
key, csr, _ := newCSR("rack:" + name)
|
||||
certPEM, cert, err := a.signRackCSR(csr, name)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return certPEM, key, cert
|
||||
}
|
||||
_, _, joined := issue("hall-b")
|
||||
dir := t.TempDir()
|
||||
p := filepath.Join(dir, "rack.pem")
|
||||
if loadRenewed(p, pool, "hall-b", joined) != nil {
|
||||
t.Error("nothing renewed yet")
|
||||
}
|
||||
time.Sleep(1100 * time.Millisecond) // a later NotAfter
|
||||
certPEM, key, renewed := issue("hall-b")
|
||||
if err := writeAtomic(p, append(certPEM, key...), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := loadRenewed(p, pool, "hall-b", joined)
|
||||
if got == nil || got.Leaf.SerialNumber.Cmp(renewed.SerialNumber) != 0 {
|
||||
t.Fatalf("renewed certificate not used: %v", got)
|
||||
}
|
||||
if loadRenewed(p, pool, "hall-c", joined) != nil {
|
||||
t.Error("another rack's certificate was used")
|
||||
}
|
||||
if loadRenewed(p, pool, "hall-b", renewed) != nil {
|
||||
t.Error("an older or equal certificate replaced the current one")
|
||||
}
|
||||
other, _ := newCA("other CA")
|
||||
op := x509.NewCertPool()
|
||||
op.AddCert(other.Cert)
|
||||
if loadRenewed(p, op, "hall-b", joined) != nil {
|
||||
t.Error("a certificate from another CA was used")
|
||||
}
|
||||
if st, _ := os.Stat(p); st.Mode().Perm() != 0o600 {
|
||||
t.Errorf("mode %v", st.Mode())
|
||||
}
|
||||
}
|
||||
+16
-3
@@ -39,6 +39,16 @@ func clientTLS(caPEM string) *tls.Config {
|
||||
}
|
||||
}
|
||||
tc := &tls.Config{MinVersion: tls.VersionTLS12}
|
||||
// a rack controller's certificate (its client one, or the server one it
|
||||
// shows agents) is never the region, whatever names it carries
|
||||
tc.VerifyConnection = func(cs tls.ConnectionState) error {
|
||||
if len(cs.PeerCertificates) > 0 {
|
||||
if cn := cs.PeerCertificates[0].Subject.CommonName; strings.HasPrefix(cn, "rack:") || strings.HasPrefix(cn, rackServerCN) {
|
||||
return fmt.Errorf("the server shows the certificate of a rack controller (%s), not the region's", cn)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if cert := clientCertificate(); cert != nil {
|
||||
tc.Certificates = []tls.Certificate{*cert}
|
||||
}
|
||||
@@ -232,13 +242,16 @@ func runClient(cmd string, args []string) error {
|
||||
if len(ids) > 1 {
|
||||
fmt.Printf("%s: ", id)
|
||||
}
|
||||
fmt.Println(str(d["output"]))
|
||||
if o := str(d["output"]); o != "" {
|
||||
fmt.Println(o)
|
||||
}
|
||||
if ok, _ := d["ok"].(bool); !ok {
|
||||
fmt.Println("error:", orDefault(str(d["error"]), "the BMC command failed"))
|
||||
failed++
|
||||
}
|
||||
}
|
||||
if failed > 0 {
|
||||
return fmt.Errorf("ipmitool failed on %d machine(s)", failed)
|
||||
return fmt.Errorf("power %s failed on %d machine(s)", args[1], failed)
|
||||
}
|
||||
case "bootdev":
|
||||
_ = need(2)
|
||||
@@ -276,7 +289,7 @@ func runClient(cmd string, args []string) error {
|
||||
return c.locationCmd(args)
|
||||
case "bmc":
|
||||
return c.bmcCmd(args)
|
||||
case "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp", "ipam":
|
||||
case "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp", "ipam", "dns", "reservations":
|
||||
return c.netCmd(cmd, args)
|
||||
case "admin-cert":
|
||||
return c.adminCertCmd(args)
|
||||
|
||||
+17
-5
@@ -31,8 +31,10 @@ type Config struct {
|
||||
ClusterVars map[string]any `json:"cluster_vars"` // available to overlay templates as .Vars (profile vars override)
|
||||
|
||||
// disk-based deployment
|
||||
OSDir string `json:"os_dir"` // <os_dir>/<name>/<version>/{disk.raw.zst,meta.json}, served under /os/
|
||||
OSSources string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync`
|
||||
OSDir string `json:"os_dir"` // <os_dir>/<name>/<version>/{disk.raw.zst,meta.json}, served under /os/
|
||||
// FirmwareDir: <firmware_dir>/<id>/{meta.json,<package>}, served under /firmware/<id>/<package>
|
||||
FirmwareDir string `json:"firmware_dir"`
|
||||
OSSources string `json:"os_sources"` // JSON list of upstream cloud images for `rollcall os sync`
|
||||
|
||||
// region/rack: this server runs an embedded rack controller named RackName;
|
||||
// remote rack controllers (rollcall rack) authenticate with RackSecret
|
||||
@@ -45,12 +47,19 @@ type Config struct {
|
||||
CADir string `json:"ca_dir"` // ca.pem, ca-key.pem, region.pem, region-key.pem (default /etc/rollcall/ca)
|
||||
TLSListen string `json:"tls_listen"` // HTTPS listener, default ":443"; "off" disables
|
||||
TLSNames []string `json:"tls_names"` // extra names/addresses for the region certificate
|
||||
AdminHTTP string `json:"admin_http"` // admin API over plain HTTP: allow (default) | loopback | deny
|
||||
// AgentSourceCheck: machine tokens only from the machine's own address (default true; see agentsrc.go)
|
||||
AgentSourceCheck *bool `json:"agent_source_check,omitempty"`
|
||||
AdminHTTP string `json:"admin_http"` // admin API over plain HTTP: loopback (default) | allow | deny
|
||||
|
||||
// a deploy/release job the machine has not started by then fails (default 30m;
|
||||
// servers can take 4-5 minutes to POST, longer when many boot together)
|
||||
JobStartTimeout string `json:"job_start_timeout"`
|
||||
|
||||
// DNS records for machines written directly (PowerDNS API or RFC 2136)
|
||||
DNS DNSConfig `json:"dns"`
|
||||
// fixed leases on the site's DHCP server for machines on proxy-mode VLANs (Kea, OpenWrt)
|
||||
DHCPRes DHCPResConfig `json:"dhcp_reservations"`
|
||||
|
||||
// external IPAM authority (NetBox; Infoblox at sites that run it)
|
||||
IPAM IPAMConfig `json:"ipam"`
|
||||
}
|
||||
@@ -100,6 +109,9 @@ func loadConfig(path string) (*Config, error) {
|
||||
if c.OSDir == "" {
|
||||
c.OSDir = filepath.Join(filepath.Dir(c.ImageDir), "os")
|
||||
}
|
||||
if c.FirmwareDir == "" {
|
||||
c.FirmwareDir = filepath.Join(filepath.Dir(c.ImageDir), "firmware")
|
||||
}
|
||||
if c.OSSources == "" {
|
||||
c.OSSources = "/etc/rollcall/os-sources.json"
|
||||
}
|
||||
@@ -115,8 +127,8 @@ func loadConfig(path string) (*Config, error) {
|
||||
case "off", "none":
|
||||
c.TLSListen = ""
|
||||
}
|
||||
if c.AdminHTTP == "" {
|
||||
c.AdminHTTP = "allow"
|
||||
if c.AdminHTTP == "" { // the token must not cross the provisioning network in clear
|
||||
c.AdminHTTP = "loopback"
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
+82
-9
@@ -161,6 +161,12 @@ func (s *Store) FinishJob(ctx context.Context, id, state, errMsg string) error {
|
||||
return err
|
||||
}
|
||||
|
||||
// FailPendingJob fails a job that has not started; false when it started meanwhile.
|
||||
func (s *Store) FailPendingJob(ctx context.Context, id, errMsg string) (bool, error) {
|
||||
tag, err := s.pool.Exec(ctx, `UPDATE jobs SET state='failed', finished=now(), started=coalesce(started, now()), error=nullif($2,'') WHERE id=$1 AND state='pending'`, id, errMsg)
|
||||
return err == nil && tag.RowsAffected() == 1, err
|
||||
}
|
||||
|
||||
func (s *Store) SetDeployed(ctx context.Context, id, os, version string) error {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE machines SET state='deployed', os=$2, os_version=$3, deployed_at=now(), booted_at=NULL, last_seen=now() WHERE id=$1`, id, os, version)
|
||||
return err
|
||||
@@ -176,6 +182,9 @@ func (s *Store) SetReleased(ctx context.Context, id string) error {
|
||||
}
|
||||
|
||||
func (s *Store) SetNodeAddr(ctx context.Context, id, name, mgmt, ib string) error {
|
||||
if name != "" && !validDomain(name) {
|
||||
return fmt.Errorf("node name %q is not a host name (letters, digits, hyphens)", name)
|
||||
}
|
||||
_, err := s.pool.Exec(ctx, `UPDATE machines SET node_name=$2, mgmt_ip=$3, ib_ip=$4 WHERE id=$1`, id, name, nilIfEmpty(mgmt), nilIfEmpty(ib))
|
||||
return err
|
||||
}
|
||||
@@ -213,6 +222,9 @@ func inventoryDisks(m *Machine) []invDisk {
|
||||
|
||||
// pickDisk: the smallest non-removable, non-USB disk that fits the image (boot/OS disks are
|
||||
// usually the small ones on servers; data disks stay untouched). Explicit choice wins.
|
||||
// remoteTran: transports of disks that are not the machine's own (lsblk TRAN).
|
||||
var remoteTran = map[string]bool{"usb": true, "iscsi": true, "fc": true, "fcoe": true, "srp": true, "iser": true}
|
||||
|
||||
func pickDisk(disks []invDisk, want string, need int64) (*invDisk, error) {
|
||||
if want != "" {
|
||||
for i := range disks {
|
||||
@@ -230,7 +242,7 @@ func pickDisk(disks []invDisk, want string, need int64) (*invDisk, error) {
|
||||
}
|
||||
var cands []invDisk
|
||||
for _, d := range disks {
|
||||
if d.Removable || d.Tran == "usb" || d.Size < need {
|
||||
if d.Removable || remoteTran[d.Tran] || d.Size < need { // never a USB stick or a SAN LUN by itself
|
||||
continue
|
||||
}
|
||||
cands = append(cands, d)
|
||||
@@ -285,6 +297,9 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile {
|
||||
if len(sp.Packages) > 0 {
|
||||
fmt.Fprintf(&b, "packages: %s\n", yamlList(sp.Packages))
|
||||
}
|
||||
// plain HTTP on purpose: nothing in it is secret (the token only works from this
|
||||
// machine's address), and cloud-init spends its 10 tries within milliseconds, so a
|
||||
// TLS hiccup on first boot (seen once on Debian: bad record MAC) loses "booted"
|
||||
fmt.Fprintf(&b, "phone_home:\n url: %s/api/v1/phone-home/%s\n post: [instance_id, hostname, fqdn, pub_key_ed25519]\n tries: 10\n", base, token)
|
||||
b.WriteString("final_message: \"rollcall: deployed, up after $UPTIME s\"\n") // only $UPTIME/$TIMESTAMP/$DATASOURCE/$VERSION are expanded
|
||||
ud = b.String()
|
||||
@@ -468,6 +483,20 @@ func (a *App) handleDeploy(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 400, fmt.Errorf("os is required"))
|
||||
return
|
||||
}
|
||||
req.Hostname = strings.ToLower(req.Hostname)
|
||||
if req.Hostname != "" && !validDomain(req.Hostname) {
|
||||
fail(w, 400, fmt.Errorf("hostname %q is not a host name (letters, digits, hyphens)", req.Hostname))
|
||||
return
|
||||
}
|
||||
// refuse before anything is allocated or renamed
|
||||
if m.Profile != nil {
|
||||
fail(w, 409, fmt.Errorf("machine is a stateless node (profile %s); unassign it first", *m.Profile))
|
||||
return
|
||||
}
|
||||
if j, err := a.store.ActiveJob(r.Context(), m.ID); err == nil {
|
||||
fail(w, 409, fmt.Errorf("job %s (%s, %s) is still active; cancel it first", j.ID, j.Kind, j.State))
|
||||
return
|
||||
}
|
||||
meta, err := osResolve(a.cfg.OSDir, req.OS, req.Version)
|
||||
if err != nil {
|
||||
fail(w, 404, err)
|
||||
@@ -550,6 +579,17 @@ func (a *App) handleRelease(w http.ResponseWriter, r *http.Request) {
|
||||
if req.Wipe == "" {
|
||||
req.Wipe = "quick"
|
||||
}
|
||||
if req.Disk == "" { // the disk the OS went to, not "the smallest one"
|
||||
if jobs, err := a.store.Jobs(r.Context(), m.ID, 50); err == nil {
|
||||
for _, j := range jobs {
|
||||
var sp DeploySpec
|
||||
if j.Kind == "deploy" && j.State == "done" && json.Unmarshal(j.Spec, &sp) == nil && sp.Disk != "" {
|
||||
req.Disk = sp.Disk
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
disk, err := pickDisk(inventoryDisks(m), req.Disk, 0)
|
||||
if err != nil {
|
||||
fail(w, 409, err)
|
||||
@@ -589,6 +629,10 @@ func (a *App) handleJobCancel(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 404, fmt.Errorf("no active job"))
|
||||
return
|
||||
}
|
||||
if j.Kind == "firmware" {
|
||||
fail(w, 409, errors.New("a firmware update cannot be stopped from here once the BMC has it (it would leave the component half-flashed); it ends by itself"))
|
||||
return
|
||||
}
|
||||
_ = a.store.FinishJob(r.Context(), j.ID, "cancelled", "cancelled by operator")
|
||||
a.afterJob(r.Context(), m, j, "cancelled", "cancelled by operator")
|
||||
writeJSON(w, 200, obj{"id": m.ID, "job": j.ID, "state": "cancelled"})
|
||||
@@ -648,16 +692,41 @@ func (a *App) jobFromRequest(w http.ResponseWriter, r *http.Request) (*Job, stri
|
||||
return j, tok
|
||||
}
|
||||
|
||||
// jobMachine checks where the job token came from; nil when refused (answered).
|
||||
func (a *App) jobMachine(w http.ResponseWriter, r *http.Request, j *Job) *Machine {
|
||||
m, err := a.store.Get(r.Context(), j.MachineID)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return nil
|
||||
}
|
||||
if err := a.tokenSource(r.Context(), r, m, j, "job"); err != nil {
|
||||
fail(w, 403, err)
|
||||
return nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func (a *App) handleJobSelf(w http.ResponseWriter, r *http.Request) {
|
||||
j, tok := a.jobFromRequest(w, r)
|
||||
if j == nil {
|
||||
return
|
||||
}
|
||||
if a.jobMachine(w, r, j) == nil {
|
||||
return
|
||||
}
|
||||
if j.State != "pending" && j.State != "running" {
|
||||
fail(w, 409, fmt.Errorf("job %s is %s", j.ID, j.State))
|
||||
return
|
||||
}
|
||||
sp := j.spec()
|
||||
if a.overTLS(r) { // an agent speaking HTTPS gets the image over HTTPS too
|
||||
if m, err := a.store.Get(r.Context(), j.MachineID); err == nil {
|
||||
base := a.urlFor(m)
|
||||
if t := a.agentTLS(base); t != "" && strings.HasPrefix(sp.ImageURL, base+"/") {
|
||||
sp.ImageURL = t + strings.TrimPrefix(sp.ImageURL, base)
|
||||
}
|
||||
}
|
||||
}
|
||||
out := obj{"id": j.ID, "machine_id": j.MachineID, "kind": j.Kind, "spec": sp}
|
||||
if j.Kind == "deploy" {
|
||||
out["files"] = a.renderSeed(j, sp, tok)
|
||||
@@ -690,13 +759,12 @@ func (a *App) handleJobProgress(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 200, obj{"job": j.ID, "state": j.State, "next": "poweroff"})
|
||||
return
|
||||
}
|
||||
prog, _ := json.Marshal(obj{"phase": p.Phase, "pct": p.Pct, "msg": p.Msg, "details": p.Details, "updated": time.Now()})
|
||||
_ = a.store.JobProgress(r.Context(), j.ID, prog)
|
||||
m, err := a.store.Get(r.Context(), j.MachineID)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
m := a.jobMachine(w, r, j)
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
prog, _ := json.Marshal(obj{"phase": p.Phase, "pct": p.Pct, "msg": p.Msg, "details": p.Details, "updated": time.Now()})
|
||||
_ = a.store.JobProgress(r.Context(), j.ID, prog)
|
||||
next := "continue"
|
||||
switch p.State {
|
||||
case "done":
|
||||
@@ -725,6 +793,10 @@ func (a *App) handlePhoneHome(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 404, errNotFound)
|
||||
return
|
||||
}
|
||||
if m, err := a.store.Get(r.Context(), j.MachineID); err != nil || a.tokenSource(r.Context(), r, m, nil, "phone-home") != nil {
|
||||
fail(w, 403, errors.New("not from this machine"))
|
||||
return
|
||||
}
|
||||
_ = r.ParseForm()
|
||||
_ = a.store.MarkPhoneHome(r.Context(), j.MachineID)
|
||||
detail := fmt.Sprintf("job=%s from %s hostname=%s fqdn=%s", j.ID, r.RemoteAddr, r.Form.Get("hostname"), r.Form.Get("fqdn"))
|
||||
@@ -755,7 +827,7 @@ func (a *App) jobStartTimeout() time.Duration {
|
||||
func staleJobs(active map[string]*Job, cutoff time.Time) []*Job {
|
||||
var out []*Job
|
||||
for _, j := range active {
|
||||
if j.State == "pending" && j.Created.Before(cutoff) {
|
||||
if j.State == "pending" && j.Kind != "firmware" && j.Created.Before(cutoff) { // firmware jobs have their own watch
|
||||
out = append(out, j)
|
||||
}
|
||||
}
|
||||
@@ -775,8 +847,9 @@ func (a *App) jobWatchdog() {
|
||||
continue
|
||||
}
|
||||
msg := fmt.Sprintf("the machine did not start the job within %s: check its power, that it PXE-boots on the provisioning network, and its console for a hung POST (events show the power command's result; job_start_timeout raises the limit)", limit)
|
||||
_ = a.store.FinishJob(ctx, j.ID, "failed", msg)
|
||||
a.afterJob(ctx, m, j, "failed", msg)
|
||||
if ok, _ := a.store.FailPendingJob(ctx, j.ID, msg); ok { // not if it reported progress since the snapshot
|
||||
a.afterJob(ctx, m, j, "failed", msg)
|
||||
}
|
||||
}
|
||||
}
|
||||
cancel()
|
||||
|
||||
@@ -0,0 +1,335 @@
|
||||
package main
|
||||
|
||||
// DHCP reservations on the site's DHCP server. On a VLAN in proxy mode the
|
||||
// site's server hands out addresses and Rollcall only answers PXE; with a
|
||||
// reservation driver the machine still gets the address Rollcall planned for
|
||||
// it, as a fixed lease (MAC -> IP) on that server (Infoblox's fixedaddress).
|
||||
//
|
||||
// kea ISC Kea with the host_cmds hook (open source since Kea 3.0):
|
||||
// reservation-add/-del/-get-all over the control API; Rollcall's
|
||||
// reservations carry user-context {"rollcall": <machine>}
|
||||
// openwrt rpcd/ubus: "host" sections in /etc/config/dhcp with an
|
||||
// option rollcall '<machine>'
|
||||
//
|
||||
// Reservations Rollcall did not make are never changed or deleted; one in
|
||||
// the way (same MAC or address) is reported as a conflict.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type DHCPResConfig struct {
|
||||
Provider string `json:"provider"` // "" | kea | openwrt
|
||||
URL string `json:"url"` // kea: control API (http://kea:8000); openwrt: http(s)://router (rpcd at /ubus)
|
||||
User string `json:"user"`
|
||||
PasswordFile string `json:"password_file"`
|
||||
CAFile string `json:"ca_file"`
|
||||
SyncInterval string `json:"sync_interval"`
|
||||
// kea: write the configuration file after changes (reservations kept in
|
||||
// the server's memory, i.e. no hosts database); default true
|
||||
Persist *bool `json:"persist,omitempty"`
|
||||
}
|
||||
|
||||
// dhcpRes is one fixed lease.
|
||||
type dhcpRes struct {
|
||||
MAC string `json:"mac"`
|
||||
IP string `json:"ip"`
|
||||
Hostname string `json:"hostname,omitempty"`
|
||||
Machine string `json:"machine,omitempty"`
|
||||
Subnet string `json:"subnet"` // CIDR
|
||||
ref string // provider's handle (OpenWrt section)
|
||||
}
|
||||
|
||||
func (r dhcpRes) key() string { return r.Subnet + " " + r.MAC }
|
||||
|
||||
type dhcpResProvider interface {
|
||||
Name() string
|
||||
// Owned: Rollcall's reservations in these subnets, and the MAC/IP pairs others hold.
|
||||
Owned(ctx context.Context, subnets []string) (owned []dhcpRes, foreign []dhcpRes, err error)
|
||||
Put(ctx context.Context, r dhcpRes) error
|
||||
Delete(ctx context.Context, r dhcpRes) error
|
||||
// Done is called after a pass that changed something (Kea: config-write).
|
||||
Done(ctx context.Context) error
|
||||
}
|
||||
|
||||
type dhcpResReport struct {
|
||||
Provider string `json:"provider"`
|
||||
At time.Time `json:"at"`
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
Put []string `json:"put"`
|
||||
Deleted []string `json:"deleted"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Skipped []string `json:"skipped,omitempty"` // addresses without a known MAC
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (r *dhcpResReport) fail(e string) { r.Error = joinErr(r.Error, e) }
|
||||
|
||||
type dhcpResState struct {
|
||||
mu sync.Mutex
|
||||
last dhcpResReport
|
||||
kick chan struct{}
|
||||
}
|
||||
|
||||
func newDHCPResProvider(c DHCPResConfig) (dhcpResProvider, error) {
|
||||
if c.Provider == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if c.URL == "" {
|
||||
return nil, errors.New("dhcp_reservations.url is required")
|
||||
}
|
||||
pw := ""
|
||||
if c.PasswordFile != "" {
|
||||
b, err := os.ReadFile(c.PasswordFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dhcp_reservations.password_file: %w", err)
|
||||
}
|
||||
pw = strings.TrimSpace(string(b))
|
||||
}
|
||||
hc, err := httpClientCA(c.CAFile, 30*time.Second)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dhcp_reservations.ca_file: %w", err)
|
||||
}
|
||||
switch c.Provider {
|
||||
case "kea":
|
||||
return &keaProvider{url: strings.TrimRight(c.URL, "/"), user: c.User, pass: pw, hc: hc, persist: c.Persist == nil || *c.Persist}, nil
|
||||
case "openwrt":
|
||||
return &openwrtProvider{url: strings.TrimRight(c.URL, "/") + "/ubus", user: orDefault(c.User, "root"), pass: pw, hc: hc}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("dhcp_reservations.provider %q: kea or openwrt", c.Provider)
|
||||
}
|
||||
|
||||
func (a *App) startDHCPRes() error {
|
||||
p, err := newDHCPResProvider(a.cfg.DHCPRes)
|
||||
if err != nil || p == nil {
|
||||
return err
|
||||
}
|
||||
a.dhcpRes = p
|
||||
a.dhcpResSt = &dhcpResState{kick: make(chan struct{}, 1)}
|
||||
every := 5 * time.Minute
|
||||
if d, err := time.ParseDuration(a.cfg.DHCPRes.SyncInterval); err == nil && d >= 10*time.Second {
|
||||
every = d
|
||||
}
|
||||
go func() {
|
||||
a.kickDHCPRes()
|
||||
for {
|
||||
select {
|
||||
case <-a.dhcpResSt.kick:
|
||||
case <-time.After(every):
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
rep := a.reconcileDHCPRes(ctx, false)
|
||||
cancel()
|
||||
if rep.Error != "" {
|
||||
log.Printf("dhcp reservations %s: %s", p.Name(), rep.Error)
|
||||
} else if n := len(rep.Put) + len(rep.Deleted) + len(rep.Conflicts); n > 0 {
|
||||
log.Printf("dhcp reservations %s: put %d, deleted %d, conflicts %d %v", p.Name(), len(rep.Put), len(rep.Deleted), len(rep.Conflicts), rep.Conflicts)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("dhcp reservations: %s at %s", p.Name(), a.cfg.DHCPRes.URL)
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) kickDHCPRes() {
|
||||
if a.dhcpResSt == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case a.dhcpResSt.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// desiredDHCPRes: every address Rollcall gave a machine on a proxy-mode
|
||||
// subnet, with the MAC of the interface it belongs to.
|
||||
func desiredDHCPRes(ips []IPAddr, subnets []Subnet, machines map[string]*Machine) (map[string]dhcpRes, []string) {
|
||||
want := map[string]dhcpRes{}
|
||||
var skipped []string
|
||||
for _, x := range ips {
|
||||
sn := subnetFor(subnets, x.IP)
|
||||
m := machines[x.MachineID]
|
||||
if sn == nil || sn.DHCPMode != "proxy" || m == nil || len(m.MACs) == 0 {
|
||||
continue
|
||||
}
|
||||
ad, ok := parseAddr(x.IP)
|
||||
if !ok || !ad.Is4() { // DHCPv6 reservations key on DUIDs, not MACs
|
||||
continue
|
||||
}
|
||||
mac := ""
|
||||
if x.IP == deref(m.MgmtIP) || x.Iface == "" {
|
||||
mac = m.MACs[0]
|
||||
}
|
||||
for _, n := range machineNICs(m) {
|
||||
if n.Name == x.Iface && n.MAC != "" {
|
||||
mac = n.MAC
|
||||
}
|
||||
}
|
||||
if mac == "" {
|
||||
skipped = append(skipped, x.IP+" ("+x.Iface+": no single MAC)")
|
||||
continue
|
||||
}
|
||||
r := dhcpRes{MAC: mac, IP: ad.String(), Hostname: deref(m.NodeName), Machine: m.ID, Subnet: sn.CIDR}
|
||||
want[r.key()] = r
|
||||
}
|
||||
return want, skipped
|
||||
}
|
||||
|
||||
func (a *App) reconcileDHCPRes(ctx context.Context, dry bool) dhcpResReport {
|
||||
rep := dhcpResReport{At: time.Now().UTC(), DryRun: dry, Put: []string{}, Deleted: []string{}, Conflicts: []string{}}
|
||||
if a.dhcpRes == nil {
|
||||
rep.Error = "no DHCP reservation provider configured (dhcp_reservations.provider in config.json)"
|
||||
return rep
|
||||
}
|
||||
rep.Provider = a.dhcpRes.Name()
|
||||
a.dhcpResSt.mu.Lock()
|
||||
defer a.dhcpResSt.mu.Unlock()
|
||||
defer func() {
|
||||
if !dry {
|
||||
a.dhcpResSt.last = rep
|
||||
}
|
||||
}()
|
||||
ips, err := a.store.IPs(ctx, "", "")
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
list, err := a.store.List(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
machines := map[string]*Machine{}
|
||||
for _, m := range list {
|
||||
machines[m.ID] = m
|
||||
}
|
||||
subnets := a.subnetsSnapshot()
|
||||
want, skipped := desiredDHCPRes(ips, subnets, machines)
|
||||
rep.Skipped = skipped
|
||||
var proxied []string
|
||||
for _, sn := range subnets {
|
||||
if sn.DHCPMode == "proxy" && familyOf(sn.CIDR) == 4 {
|
||||
proxied = append(proxied, sn.CIDR)
|
||||
}
|
||||
}
|
||||
if len(proxied) == 0 {
|
||||
return rep
|
||||
}
|
||||
a.dhcpResWith(ctx, want, proxied, &rep)
|
||||
return rep
|
||||
}
|
||||
|
||||
func (a *App) dhcpResWith(ctx context.Context, want map[string]dhcpRes, subnets []string, rep *dhcpResReport) {
|
||||
owned, foreign, err := a.dhcpRes.Owned(ctx, subnets)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return
|
||||
}
|
||||
have := map[string]dhcpRes{}
|
||||
for _, o := range owned {
|
||||
have[o.key()] = o
|
||||
}
|
||||
keys := make([]string, 0, len(want))
|
||||
for k := range want {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
changed := false
|
||||
for _, k := range keys {
|
||||
w := want[k]
|
||||
if o, ok := have[k]; ok {
|
||||
delete(have, k)
|
||||
if o.IP == w.IP && o.Hostname == w.Hostname && o.Machine == w.Machine {
|
||||
continue
|
||||
}
|
||||
if rep.DryRun {
|
||||
rep.Put = append(rep.Put, w.MAC+" "+w.IP)
|
||||
continue
|
||||
}
|
||||
if err := a.dhcpRes.Delete(ctx, o); err != nil { // replaced: the old one goes first
|
||||
rep.fail(fmt.Sprintf("replace %s: %v", w.MAC, err))
|
||||
continue
|
||||
}
|
||||
changed = true
|
||||
} else if c := clash(foreign, w); c != "" {
|
||||
rep.Conflicts = append(rep.Conflicts, fmt.Sprintf("%s %s: %s", w.MAC, w.IP, c))
|
||||
continue
|
||||
}
|
||||
if rep.DryRun {
|
||||
rep.Put = append(rep.Put, w.MAC+" "+w.IP)
|
||||
continue
|
||||
}
|
||||
if err := a.dhcpRes.Put(ctx, w); err != nil {
|
||||
rep.fail(fmt.Sprintf("put %s %s: %v", w.MAC, w.IP, err))
|
||||
continue
|
||||
}
|
||||
changed = true
|
||||
rep.Put = append(rep.Put, w.MAC+" "+w.IP)
|
||||
}
|
||||
gone := make([]dhcpRes, 0, len(have))
|
||||
for _, o := range have {
|
||||
gone = append(gone, o)
|
||||
}
|
||||
sort.Slice(gone, func(i, j int) bool { return gone[i].key() < gone[j].key() })
|
||||
for _, o := range gone {
|
||||
if !rep.DryRun {
|
||||
if err := a.dhcpRes.Delete(ctx, o); err != nil {
|
||||
rep.fail(fmt.Sprintf("delete %s: %v", o.MAC, err))
|
||||
continue
|
||||
}
|
||||
changed = true
|
||||
}
|
||||
rep.Deleted = append(rep.Deleted, o.MAC+" "+o.IP)
|
||||
}
|
||||
if changed {
|
||||
if err := a.dhcpRes.Done(ctx); err != nil {
|
||||
rep.fail("save: " + err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// clash: a reservation someone else holds for this MAC or this address.
|
||||
func clash(foreign []dhcpRes, w dhcpRes) string {
|
||||
for _, f := range foreign {
|
||||
if f.Subnet != w.Subnet {
|
||||
continue
|
||||
}
|
||||
switch {
|
||||
case strings.EqualFold(f.MAC, w.MAC):
|
||||
return "the MAC is reserved there for " + f.IP + " (not Rollcall's)"
|
||||
case f.IP == w.IP:
|
||||
return "the address is reserved there for " + f.MAC + " (not Rollcall's)"
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (a *App) handleDHCPRes(w http.ResponseWriter, r *http.Request) {
|
||||
if a.dhcpRes == nil {
|
||||
writeJSON(w, 200, obj{"provider": "", "comment": "no DHCP reservation provider (dhcp_reservations in config.json)"})
|
||||
return
|
||||
}
|
||||
a.dhcpResSt.mu.Lock()
|
||||
last := a.dhcpResSt.last
|
||||
a.dhcpResSt.mu.Unlock()
|
||||
writeJSON(w, 200, obj{"provider": a.dhcpRes.Name(), "url": a.cfg.DHCPRes.URL, "last_sync": last})
|
||||
}
|
||||
|
||||
func (a *App) handleDHCPResSync(w http.ResponseWriter, r *http.Request) {
|
||||
if a.dhcpRes == nil {
|
||||
fail(w, 409, errors.New("no DHCP reservation provider configured"))
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 2*time.Minute)
|
||||
defer cancel()
|
||||
writeJSON(w, 200, a.reconcileDHCPRes(ctx, r.URL.Query().Get("dry") == "1"))
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
package main
|
||||
|
||||
// Kea: the control API (Kea 3.0's own HTTP control socket, or the Control
|
||||
// Agent before it) with the host_cmds hook. Subnet ids come from
|
||||
// config-get; reservations carry user-context {"rollcall": <machine>}.
|
||||
// A server with a hosts database keeps them there; one without answers
|
||||
// "Host database not available", and then they go into its running
|
||||
// configuration (operation-target memory), which config-write saves.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type keaProvider struct {
|
||||
url, user, pass string
|
||||
hc *http.Client
|
||||
persist bool
|
||||
service *bool // send "service": ["dhcp4"] (a Control Agent wants it; Kea 3.0's own socket refuses it)
|
||||
memory bool // no hosts database: reservations go into the running configuration
|
||||
}
|
||||
|
||||
func (k *keaProvider) Name() string { return "kea" }
|
||||
|
||||
type keaAnswer struct {
|
||||
Result int `json:"result"`
|
||||
Text string `json:"text"`
|
||||
Arguments json.RawMessage `json:"arguments"`
|
||||
}
|
||||
|
||||
// call sends one command; result 0 (success) and 3 (empty) are not errors.
|
||||
func (k *keaProvider) call(ctx context.Context, command string, args any) (*keaAnswer, error) {
|
||||
try := func(service bool) (*keaAnswer, error) {
|
||||
body := map[string]any{"command": command}
|
||||
if args != nil {
|
||||
body["arguments"] = args
|
||||
}
|
||||
if service {
|
||||
body["service"] = []string{"dhcp4"}
|
||||
}
|
||||
b, _ := json.Marshal(body)
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", k.url, bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
if k.user != "" {
|
||||
req.SetBasicAuth(k.user, k.pass)
|
||||
}
|
||||
resp, err := k.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
|
||||
if resp.StatusCode >= 300 {
|
||||
return nil, fmt.Errorf("kea %s: %s %s", command, resp.Status, oneLine(string(data)))
|
||||
}
|
||||
var many []keaAnswer // the Control Agent answers with a list, one per service
|
||||
if json.Unmarshal(data, &many) == nil && len(many) > 0 {
|
||||
return &many[0], nil
|
||||
}
|
||||
var one keaAnswer
|
||||
if err := json.Unmarshal(data, &one); err != nil {
|
||||
return nil, fmt.Errorf("kea %s: %w", command, err)
|
||||
}
|
||||
return &one, nil
|
||||
}
|
||||
if k.service == nil { // learn once whether this endpoint wants "service"
|
||||
ans, err := try(false)
|
||||
if err == nil && !(ans.Result == 1 && strings.Contains(strings.ToLower(ans.Text), "service")) {
|
||||
f := false
|
||||
k.service = &f
|
||||
return ans, nil
|
||||
}
|
||||
t := true
|
||||
k.service = &t
|
||||
}
|
||||
return try(*k.service)
|
||||
}
|
||||
|
||||
func (k *keaProvider) ok(ctx context.Context, command string, args any) (*keaAnswer, error) {
|
||||
ans, err := k.call(ctx, command, args)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
switch ans.Result {
|
||||
case 0, 3:
|
||||
return ans, nil
|
||||
case 2:
|
||||
return nil, fmt.Errorf("kea does not know %s (the host_cmds hook is needed: open source since Kea 3.0)", command)
|
||||
}
|
||||
return nil, fmt.Errorf("kea %s: %s", command, ans.Text)
|
||||
}
|
||||
|
||||
// subnetIDs maps CIDR -> Kea subnet id (also inside shared networks).
|
||||
func (k *keaProvider) subnetIDs(ctx context.Context) (map[string]int, error) {
|
||||
ans, err := k.ok(ctx, "config-get", nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
type sub struct {
|
||||
ID int `json:"id"`
|
||||
Subnet string `json:"subnet"`
|
||||
}
|
||||
var cfg struct {
|
||||
Dhcp4 struct {
|
||||
Subnet4 []sub `json:"subnet4"`
|
||||
SharedNetworks []struct {
|
||||
Subnet4 []sub `json:"subnet4"`
|
||||
} `json:"shared-networks"`
|
||||
} `json:"Dhcp4"`
|
||||
}
|
||||
if err := json.Unmarshal(ans.Arguments, &cfg); err != nil {
|
||||
return nil, fmt.Errorf("kea config-get: %w", err)
|
||||
}
|
||||
out := map[string]int{}
|
||||
add := func(s sub) {
|
||||
if p, err := netip.ParsePrefix(s.Subnet); err == nil {
|
||||
out[p.Masked().String()] = s.ID
|
||||
}
|
||||
}
|
||||
for _, s := range cfg.Dhcp4.Subnet4 {
|
||||
add(s)
|
||||
}
|
||||
for _, n := range cfg.Dhcp4.SharedNetworks {
|
||||
for _, s := range n.Subnet4 {
|
||||
add(s)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (k *keaProvider) Owned(ctx context.Context, subnets []string) ([]dhcpRes, []dhcpRes, error) {
|
||||
ids, err := k.subnetIDs(ctx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var owned, foreign []dhcpRes
|
||||
for _, cidr := range subnets {
|
||||
id, ok := ids[cidr]
|
||||
if !ok {
|
||||
return nil, nil, fmt.Errorf("kea serves no subnet %s", cidr)
|
||||
}
|
||||
ans, err := k.ok(ctx, "reservation-get-all", map[string]any{"subnet-id": id})
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var got struct {
|
||||
Hosts []struct {
|
||||
HWAddress string `json:"hw-address"`
|
||||
IPAddress string `json:"ip-address"`
|
||||
Hostname string `json:"hostname"`
|
||||
UserContext map[string]any `json:"user-context"`
|
||||
} `json:"hosts"`
|
||||
}
|
||||
_ = json.Unmarshal(ans.Arguments, &got)
|
||||
for _, h := range got.Hosts {
|
||||
r := dhcpRes{MAC: normMAC(h.HWAddress), IP: h.IPAddress, Hostname: h.Hostname, Subnet: cidr}
|
||||
if m, ok := h.UserContext["rollcall"].(string); ok && m != "" {
|
||||
r.Machine = m
|
||||
owned = append(owned, r)
|
||||
} else {
|
||||
foreign = append(foreign, r)
|
||||
}
|
||||
}
|
||||
}
|
||||
return owned, foreign, nil
|
||||
}
|
||||
|
||||
func (k *keaProvider) subnetID(ctx context.Context, cidr string) (int, error) {
|
||||
ids, err := k.subnetIDs(ctx)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
id, ok := ids[cidr]
|
||||
if !ok {
|
||||
return 0, fmt.Errorf("kea serves no subnet %s", cidr)
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func (k *keaProvider) Put(ctx context.Context, r dhcpRes) error {
|
||||
id, err := k.subnetID(ctx, r.Subnet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
res := map[string]any{"subnet-id": id, "hw-address": r.MAC, "ip-address": r.IP, "user-context": map[string]any{"rollcall": r.Machine}}
|
||||
if r.Hostname != "" {
|
||||
res["hostname"] = r.Hostname
|
||||
}
|
||||
return k.change(ctx, "reservation-add", map[string]any{"reservation": res})
|
||||
}
|
||||
|
||||
// change runs reservation-add/-del against the hosts database, or the
|
||||
// configuration in memory when the server has no hosts database.
|
||||
func (k *keaProvider) change(ctx context.Context, command string, args map[string]any) error {
|
||||
if k.memory {
|
||||
args["operation-target"] = "memory"
|
||||
}
|
||||
_, err := k.ok(ctx, command, args)
|
||||
if err != nil && !k.memory && strings.Contains(err.Error(), "database not available") {
|
||||
k.memory = true
|
||||
args["operation-target"] = "memory"
|
||||
_, err = k.ok(ctx, command, args)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (k *keaProvider) Delete(ctx context.Context, r dhcpRes) error {
|
||||
id, err := k.subnetID(ctx, r.Subnet)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return k.change(ctx, "reservation-del", map[string]any{"subnet-id": id, "identifier-type": "hw-address", "identifier": r.MAC})
|
||||
}
|
||||
|
||||
func (k *keaProvider) Done(ctx context.Context) error {
|
||||
if !k.persist || !k.memory { // a hosts database keeps them by itself
|
||||
return nil
|
||||
}
|
||||
if _, err := k.ok(ctx, "config-write", map[string]any{}); err != nil {
|
||||
return errors.New("reservations are in effect but not saved: " + err.Error())
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,204 @@
|
||||
package main
|
||||
|
||||
// OpenWrt: static leases are "host" sections of /etc/config/dhcp, reached
|
||||
// through rpcd's ubus JSON-RPC (POST /ubus). Rollcall's carry option
|
||||
// rollcall '<machine>'. The rpcd user needs uci read/write on "dhcp" (and rc
|
||||
// to reload dnsmasq): see the README for the ACL.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"strings"
|
||||
"sync"
|
||||
)
|
||||
|
||||
type openwrtProvider struct {
|
||||
url, user, pass string
|
||||
hc *http.Client
|
||||
mu sync.Mutex
|
||||
sid string
|
||||
}
|
||||
|
||||
func (o *openwrtProvider) Name() string { return "openwrt" }
|
||||
|
||||
var errUbusDenied = errors.New("ubus: access denied")
|
||||
|
||||
// rpc is one ubus call; it logs in (again) when the session is missing or expired.
|
||||
func (o *openwrtProvider) rpc(ctx context.Context, object, method string, args map[string]any) (json.RawMessage, error) {
|
||||
o.mu.Lock()
|
||||
sid := o.sid
|
||||
o.mu.Unlock()
|
||||
if sid == "" {
|
||||
if err := o.login(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sid = o.sid
|
||||
}
|
||||
res, err := o.raw(ctx, sid, object, method, args)
|
||||
if errors.Is(err, errUbusDenied) { // expired session
|
||||
if err := o.login(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
res, err = o.raw(ctx, o.sid, object, method, args)
|
||||
}
|
||||
return res, err
|
||||
}
|
||||
|
||||
func (o *openwrtProvider) login(ctx context.Context) error {
|
||||
res, err := o.raw(ctx, "00000000000000000000000000000000", "session", "login", map[string]any{"username": o.user, "password": o.pass})
|
||||
if err != nil {
|
||||
return fmt.Errorf("openwrt login: %w", err)
|
||||
}
|
||||
var s struct {
|
||||
SID string `json:"ubus_rpc_session"`
|
||||
}
|
||||
if json.Unmarshal(res, &s) != nil || s.SID == "" {
|
||||
return errors.New("openwrt login: no session")
|
||||
}
|
||||
o.mu.Lock()
|
||||
o.sid = s.SID
|
||||
o.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
var ubusErrors = map[int]string{1: "invalid command", 2: "invalid argument", 3: "method not found", 4: "not found", 5: "no data", 6: "permission denied", 7: "timeout", 8: "not supported", 9: "unknown error", 10: "connection failed"}
|
||||
|
||||
func (o *openwrtProvider) raw(ctx context.Context, sid, object, method string, args map[string]any) (json.RawMessage, error) {
|
||||
if args == nil {
|
||||
args = map[string]any{}
|
||||
}
|
||||
b, _ := json.Marshal(map[string]any{"jsonrpc": "2.0", "id": 1, "method": "call", "params": []any{sid, object, method, args}})
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", o.url, bytes.NewReader(b))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := o.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 8<<20))
|
||||
var out struct {
|
||||
Result []json.RawMessage `json:"result"`
|
||||
Error *struct {
|
||||
Code int `json:"code"`
|
||||
Message string `json:"message"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return nil, fmt.Errorf("ubus %s %s: %s %s", object, method, resp.Status, oneLine(string(data)))
|
||||
}
|
||||
if out.Error != nil {
|
||||
if out.Error.Code == -32002 {
|
||||
return nil, errUbusDenied
|
||||
}
|
||||
return nil, fmt.Errorf("ubus %s %s: %s", object, method, out.Error.Message)
|
||||
}
|
||||
if len(out.Result) == 0 {
|
||||
return nil, fmt.Errorf("ubus %s %s: empty answer", object, method)
|
||||
}
|
||||
var code int
|
||||
_ = json.Unmarshal(out.Result[0], &code)
|
||||
if code != 0 {
|
||||
if code == 6 {
|
||||
return nil, fmt.Errorf("ubus %s %s: permission denied (rpcd ACL for this user)", object, method)
|
||||
}
|
||||
return nil, fmt.Errorf("ubus %s %s: %s", object, method, orDefault(ubusErrors[code], fmt.Sprint(code)))
|
||||
}
|
||||
if len(out.Result) < 2 {
|
||||
return json.RawMessage("{}"), nil
|
||||
}
|
||||
return out.Result[1], nil
|
||||
}
|
||||
|
||||
func (o *openwrtProvider) Owned(ctx context.Context, subnets []string) ([]dhcpRes, []dhcpRes, error) {
|
||||
res, err := o.rpc(ctx, "uci", "get", map[string]any{"config": "dhcp", "type": "host"})
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "not found") { // no host sections yet
|
||||
return nil, nil, nil
|
||||
}
|
||||
return nil, nil, err
|
||||
}
|
||||
var got struct {
|
||||
Values map[string]map[string]any `json:"values"`
|
||||
}
|
||||
if err := json.Unmarshal(res, &got); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var prefixes []netip.Prefix
|
||||
for _, s := range subnets {
|
||||
if p, err := netip.ParsePrefix(s); err == nil {
|
||||
prefixes = append(prefixes, p)
|
||||
}
|
||||
}
|
||||
var owned, foreign []dhcpRes
|
||||
for name, sec := range got.Values {
|
||||
ip, _ := sec["ip"].(string)
|
||||
ad, err := netip.ParseAddr(ip)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
subnet := ""
|
||||
for _, p := range prefixes {
|
||||
if p.Contains(ad) {
|
||||
subnet = p.Masked().String()
|
||||
}
|
||||
}
|
||||
if subnet == "" {
|
||||
continue
|
||||
}
|
||||
var mac string
|
||||
switch v := sec["mac"].(type) { // a string, or a list of MACs
|
||||
case string:
|
||||
mac = strings.Fields(v)[0]
|
||||
case []any:
|
||||
if len(v) > 0 {
|
||||
mac, _ = v[0].(string)
|
||||
}
|
||||
}
|
||||
r := dhcpRes{MAC: normMAC(mac), IP: ad.String(), Subnet: subnet, ref: name}
|
||||
r.Hostname, _ = sec["name"].(string)
|
||||
if m, _ := sec["rollcall"].(string); m != "" {
|
||||
r.Machine = m
|
||||
owned = append(owned, r)
|
||||
} else {
|
||||
foreign = append(foreign, r)
|
||||
}
|
||||
}
|
||||
return owned, foreign, nil
|
||||
}
|
||||
|
||||
func (o *openwrtProvider) Put(ctx context.Context, r dhcpRes) error {
|
||||
values := map[string]any{"mac": r.MAC, "ip": r.IP, "rollcall": r.Machine}
|
||||
if r.Hostname != "" {
|
||||
values["name"] = r.Hostname
|
||||
}
|
||||
_, err := o.rpc(ctx, "uci", "add", map[string]any{"config": "dhcp", "type": "host", "values": values})
|
||||
return err
|
||||
}
|
||||
|
||||
func (o *openwrtProvider) Delete(ctx context.Context, r dhcpRes) error {
|
||||
if r.ref == "" {
|
||||
return errors.New("no section to delete")
|
||||
}
|
||||
_, err := o.rpc(ctx, "uci", "delete", map[string]any{"config": "dhcp", "section": r.ref})
|
||||
return err
|
||||
}
|
||||
|
||||
// Done commits /etc/config/dhcp and reloads dnsmasq.
|
||||
func (o *openwrtProvider) Done(ctx context.Context) error {
|
||||
if _, err := o.rpc(ctx, "uci", "commit", map[string]any{"config": "dhcp"}); err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := o.rpc(ctx, "rc", "init", map[string]any{"name": "dnsmasq", "action": "reload"}); err != nil {
|
||||
return fmt.Errorf("committed, but dnsmasq not reloaded: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDesiredDHCPRes(t *testing.T) {
|
||||
inv, _ := json.Marshal(obj{"network": []obj{{"name": "eth0", "mac": "52:54:00:00:00:01"}, {"name": "eth1", "mac": "52:54:00:00:00:02"}}})
|
||||
mg := "10.214.0.50"
|
||||
name := "fresh01"
|
||||
m := &Machine{ID: "m-1", MACs: []string{"52:54:00:00:00:01", "52:54:00:00:00:02"}, MgmtIP: &mg, NodeName: &name, Inventory: inv}
|
||||
subnets := []Subnet{{CIDR: "10.214.0.0/24", DHCPMode: "proxy"}, {CIDR: "10.215.0.0/24", DHCPMode: "proxy"}, {CIDR: "10.216.0.0/24"}, {CIDR: "fd00:214::/64", DHCPMode: "proxy"}}
|
||||
ips := []IPAddr{
|
||||
{IP: "10.214.0.50", MachineID: "m-1", Iface: "eth0"},
|
||||
{IP: "10.215.0.50", MachineID: "m-1", Iface: "eth1"},
|
||||
{IP: "10.216.0.50", MachineID: "m-1", Iface: "eth1"}, // served by Rollcall itself: no reservation
|
||||
{IP: "fd00:214::50", MachineID: "m-1", Iface: "eth0"}, // DHCPv6 keys on DUIDs
|
||||
{IP: "10.215.0.60", MachineID: "m-1", Iface: "bond0"}, // no single MAC
|
||||
{IP: "10.214.0.70", MachineID: "", Kind: "reserved"}, // not a machine's
|
||||
}
|
||||
want, skipped := desiredDHCPRes(ips, subnets, map[string]*Machine{"m-1": m})
|
||||
var got []string
|
||||
for _, r := range want {
|
||||
got = append(got, r.MAC+"="+r.IP+"/"+r.Hostname)
|
||||
}
|
||||
sort.Strings(got)
|
||||
if strings.Join(got, " ") != "52:54:00:00:00:01=10.214.0.50/fresh01 52:54:00:00:00:02=10.215.0.50/fresh01" || len(skipped) != 1 {
|
||||
t.Errorf("want %v skipped %v", got, skipped)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeKea answers the control API like Kea 3.0's own socket (no "service")
|
||||
// or a Control Agent (wants "service", answers with a list).
|
||||
type fakeKea struct {
|
||||
mu sync.Mutex
|
||||
agent bool
|
||||
noHooks bool
|
||||
noHostDB bool // like Kea with a configuration file only: add/del need operation-target memory
|
||||
hosts []obj
|
||||
commands []string
|
||||
written int
|
||||
}
|
||||
|
||||
func (f *fakeKea) handler() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if u, p, ok := r.BasicAuth(); !ok || u != "rollcall" || p != "pw" {
|
||||
w.WriteHeader(401)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
Command string `json:"command"`
|
||||
Service []string `json:"service"`
|
||||
Arguments json.RawMessage `json:"arguments"`
|
||||
}
|
||||
_ = json.NewDecoder(r.Body).Decode(&req)
|
||||
f.commands = append(f.commands, req.Command)
|
||||
answer := func(o obj) {
|
||||
if f.agent {
|
||||
_ = json.NewEncoder(w).Encode([]obj{o})
|
||||
return
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(o)
|
||||
}
|
||||
if f.agent && len(req.Service) == 0 {
|
||||
answer(obj{"result": 1, "text": "service not specified"})
|
||||
return
|
||||
}
|
||||
if !f.agent && len(req.Service) > 0 {
|
||||
answer(obj{"result": 1, "text": "'service' parameter is not supported here"})
|
||||
return
|
||||
}
|
||||
var args obj
|
||||
_ = json.Unmarshal(req.Arguments, &args)
|
||||
switch req.Command {
|
||||
case "config-get":
|
||||
answer(obj{"result": 0, "arguments": obj{"Dhcp4": obj{"subnet4": []obj{{"id": 7, "subnet": "10.214.0.0/24"}}}}})
|
||||
case "reservation-get-all":
|
||||
if f.noHooks {
|
||||
answer(obj{"result": 2, "text": "'reservation-get-all' command not supported."})
|
||||
return
|
||||
}
|
||||
answer(obj{"result": 0, "arguments": obj{"hosts": f.hosts}})
|
||||
case "reservation-add", "reservation-del":
|
||||
if f.noHostDB && args["operation-target"] != "memory" {
|
||||
answer(obj{"result": 1, "text": "Host database not available, cannot add host."})
|
||||
return
|
||||
}
|
||||
}
|
||||
switch req.Command {
|
||||
case "reservation-add":
|
||||
res := args["reservation"].(map[string]any)
|
||||
for _, h := range f.hosts {
|
||||
if h["hw-address"] == res["hw-address"] || h["ip-address"] == res["ip-address"] {
|
||||
answer(obj{"result": 1, "text": "Host already exists."})
|
||||
return
|
||||
}
|
||||
}
|
||||
f.hosts = append(f.hosts, res)
|
||||
answer(obj{"result": 0, "text": "Host added."})
|
||||
case "reservation-del":
|
||||
for i, h := range f.hosts {
|
||||
if h["hw-address"] == args["identifier"] {
|
||||
f.hosts = append(f.hosts[:i], f.hosts[i+1:]...)
|
||||
answer(obj{"result": 0, "text": "Host deleted."})
|
||||
return
|
||||
}
|
||||
}
|
||||
answer(obj{"result": 3, "text": "Host not deleted (not found)."})
|
||||
case "config-write":
|
||||
f.written++
|
||||
answer(obj{"result": 0})
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func keaFor(t *testing.T, f *fakeKea) (*keaProvider, func()) {
|
||||
srv := httptest.NewServer(f.handler())
|
||||
return &keaProvider{url: srv.URL, user: "rollcall", pass: "pw", hc: srv.Client(), persist: true}, srv.Close
|
||||
}
|
||||
|
||||
func TestKeaReservations(t *testing.T) {
|
||||
for _, agent := range []bool{false, true} {
|
||||
f := &fakeKea{agent: agent, noHostDB: agent, hosts: []obj{
|
||||
{"hw-address": "52:54:00:aa:aa:aa", "ip-address": "10.214.0.9", "hostname": "printer"}, // someone else's
|
||||
{"hw-address": "52:54:00:00:00:09", "ip-address": "10.214.0.99", "user-context": obj{"rollcall": "m-old"}},
|
||||
}}
|
||||
k, done := keaFor(t, f)
|
||||
a := &App{dhcpRes: k, dhcpResSt: &dhcpResState{}}
|
||||
want := map[string]dhcpRes{}
|
||||
for _, r := range []dhcpRes{
|
||||
{MAC: "52:54:00:00:00:01", IP: "10.214.0.50", Hostname: "fresh01", Machine: "m-1", Subnet: "10.214.0.0/24"},
|
||||
{MAC: "52:54:00:00:00:02", IP: "10.214.0.9", Machine: "m-2", Subnet: "10.214.0.0/24"}, // the printer's address
|
||||
} {
|
||||
want[r.key()] = r
|
||||
}
|
||||
rep := dhcpResReport{}
|
||||
a.dhcpResWith(context.Background(), want, []string{"10.214.0.0/24"}, &rep)
|
||||
if rep.Error != "" || strings.Join(rep.Put, ",") != "52:54:00:00:00:01 10.214.0.50" || len(rep.Conflicts) != 1 || strings.Join(rep.Deleted, ",") != "52:54:00:00:00:09 10.214.0.99" {
|
||||
t.Errorf("agent=%v report %+v", agent, rep)
|
||||
}
|
||||
var added obj
|
||||
for _, h := range f.hosts {
|
||||
if h["hw-address"] == "52:54:00:00:00:01" {
|
||||
added = h
|
||||
}
|
||||
}
|
||||
wantWrites := 0 // a hosts database keeps reservations by itself
|
||||
if f.noHostDB {
|
||||
wantWrites = 1 // in memory: config-write saves them
|
||||
}
|
||||
if dig(added, "user-context", "rollcall") != "m-1" || added["hostname"] != "fresh01" || added["subnet-id"] != float64(7) || f.written != wantWrites {
|
||||
t.Errorf("agent=%v added %v written %d", agent, added, f.written)
|
||||
}
|
||||
// a second pass changes nothing
|
||||
rep = dhcpResReport{}
|
||||
a.dhcpResWith(context.Background(), want, []string{"10.214.0.0/24"}, &rep)
|
||||
if len(rep.Put)+len(rep.Deleted) != 0 || f.written != wantWrites {
|
||||
t.Errorf("agent=%v second pass %+v", agent, rep)
|
||||
}
|
||||
done()
|
||||
}
|
||||
f := &fakeKea{noHooks: true}
|
||||
k, done := keaFor(t, f)
|
||||
defer done()
|
||||
if _, _, err := k.Owned(context.Background(), []string{"10.214.0.0/24"}); err == nil || !strings.Contains(err.Error(), "host_cmds") {
|
||||
t.Errorf("without host_cmds: %v", err)
|
||||
}
|
||||
if _, _, err := k.Owned(context.Background(), []string{"10.99.0.0/24"}); err == nil || !strings.Contains(err.Error(), "no subnet") {
|
||||
t.Errorf("unknown subnet: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// fakeUbus is rpcd's JSON-RPC with uci on "dhcp" and rc.
|
||||
type fakeUbus struct {
|
||||
mu sync.Mutex
|
||||
sessions map[string]bool
|
||||
sections map[string]map[string]any
|
||||
n int
|
||||
commits int
|
||||
reloads int
|
||||
logins int
|
||||
}
|
||||
|
||||
func (f *fakeUbus) handler() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
var req struct {
|
||||
Params []json.RawMessage `json:"params"`
|
||||
}
|
||||
_ = json.Unmarshal(b, &req)
|
||||
var sid, object, method string
|
||||
var args map[string]any
|
||||
_ = json.Unmarshal(req.Params[0], &sid)
|
||||
_ = json.Unmarshal(req.Params[1], &object)
|
||||
_ = json.Unmarshal(req.Params[2], &method)
|
||||
_ = json.Unmarshal(req.Params[3], &args)
|
||||
reply := func(v ...any) { _ = json.NewEncoder(w).Encode(obj{"jsonrpc": "2.0", "id": 1, "result": v}) }
|
||||
if object == "session" && method == "login" {
|
||||
if args["username"] != "rollcall" || args["password"] != "pw" {
|
||||
reply(6)
|
||||
return
|
||||
}
|
||||
f.logins++
|
||||
s := "s" + strings.Repeat("x", f.logins)
|
||||
f.sessions[s] = true
|
||||
reply(0, obj{"ubus_rpc_session": s})
|
||||
return
|
||||
}
|
||||
if !f.sessions[sid] {
|
||||
_ = json.NewEncoder(w).Encode(obj{"jsonrpc": "2.0", "id": 1, "error": obj{"code": -32002, "message": "Access denied"}})
|
||||
return
|
||||
}
|
||||
switch object + "." + method {
|
||||
case "uci.get":
|
||||
vals := obj{}
|
||||
for k, v := range f.sections {
|
||||
vals[k] = v
|
||||
}
|
||||
reply(0, obj{"values": vals})
|
||||
case "uci.add":
|
||||
f.n++
|
||||
name := "cfg" + strings.Repeat("0", f.n)
|
||||
sec := map[string]any{".type": "host"}
|
||||
for k, v := range args["values"].(map[string]any) {
|
||||
sec[k] = v
|
||||
}
|
||||
f.sections[name] = sec
|
||||
reply(0, obj{"section": name})
|
||||
case "uci.delete":
|
||||
delete(f.sections, args["section"].(string))
|
||||
reply(0)
|
||||
case "uci.commit":
|
||||
f.commits++
|
||||
reply(0)
|
||||
case "rc.init":
|
||||
f.reloads++
|
||||
reply(0)
|
||||
default:
|
||||
reply(3)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func TestOpenWrtReservations(t *testing.T) {
|
||||
f := &fakeUbus{sessions: map[string]bool{}, sections: map[string]map[string]any{
|
||||
"lan_printer": {".type": "host", "name": "printer", "mac": "52:54:00:aa:aa:aa", "ip": "10.214.0.9"},
|
||||
"other_net": {".type": "host", "mac": []any{"52:54:00:bb:bb:bb"}, "ip": "192.168.9.9"},
|
||||
}}
|
||||
srv := httptest.NewServer(f.handler())
|
||||
defer srv.Close()
|
||||
o := &openwrtProvider{url: srv.URL + "/ubus", user: "rollcall", pass: "pw", hc: srv.Client()}
|
||||
a := &App{dhcpRes: o, dhcpResSt: &dhcpResState{}}
|
||||
want := map[string]dhcpRes{}
|
||||
r := dhcpRes{MAC: "52:54:00:00:00:01", IP: "10.214.0.50", Hostname: "fresh01", Machine: "m-1", Subnet: "10.214.0.0/24"}
|
||||
want[r.key()] = r
|
||||
rep := dhcpResReport{}
|
||||
a.dhcpResWith(context.Background(), want, []string{"10.214.0.0/24"}, &rep)
|
||||
if rep.Error != "" || len(rep.Put) != 1 || f.commits != 1 || f.reloads != 1 {
|
||||
t.Fatalf("report %+v commits %d reloads %d", rep, f.commits, f.reloads)
|
||||
}
|
||||
var mine map[string]any
|
||||
for _, s := range f.sections {
|
||||
if s["rollcall"] == "m-1" {
|
||||
mine = s
|
||||
}
|
||||
}
|
||||
if mine == nil || mine["ip"] != "10.214.0.50" || mine["name"] != "fresh01" {
|
||||
t.Errorf("sections %v", f.sections)
|
||||
}
|
||||
// session expired: logs in again; the machine goes away: its section is deleted, the printer stays
|
||||
f.sessions = map[string]bool{}
|
||||
rep = dhcpResReport{}
|
||||
a.dhcpResWith(context.Background(), map[string]dhcpRes{}, []string{"10.214.0.0/24"}, &rep)
|
||||
if rep.Error != "" || len(rep.Deleted) != 1 || f.logins != 2 || f.sections["lan_printer"] == nil || f.sections["other_net"] == nil {
|
||||
t.Errorf("report %+v logins %d sections %v", rep, f.logins, f.sections)
|
||||
}
|
||||
}
|
||||
@@ -195,7 +195,7 @@ func discoSuggest(d discoIface) string {
|
||||
case v4 == "":
|
||||
return "link up, no IPv4 address" + vlan + ": give it its address on the provisioning network, then bootstrap.sh --iface " + d.Name
|
||||
case foreign > 0:
|
||||
return v4 + " already has DHCP (" + strings.Join(d.DHCP, ",") + "): either the network team turns it off for this VLAN, or keep it and point it at this host (next-server " + strings.Split(d.Addrs[0], "/")[0] + ", or a DHCP relay); then bootstrap.sh --iface " + d.Name + " --no-dhcp"
|
||||
return v4 + " already has DHCP (" + strings.Join(d.DHCP, ",") + "): keep it and answer only PXE from here - bootstrap.sh --iface " + d.Name + " --proxy-dhcp - or the network team turns it off for this VLAN"
|
||||
default:
|
||||
return "candidate provisioning network " + v4 + vlan + ": bootstrap.sh --iface " + d.Name + " (or Network -> New subnet in the web UI)"
|
||||
}
|
||||
|
||||
@@ -0,0 +1,570 @@
|
||||
package main
|
||||
|
||||
// DNS provider: Rollcall writes its machines' A/AAAA records into the site's
|
||||
// DNS itself - for sites without an IPAM authority whose DNS sync would do it.
|
||||
//
|
||||
// - powerdns: the HTTP API; Rollcall's RRsets carry a comment with
|
||||
// account "rollcall" and only those are changed or deleted.
|
||||
// - rfc2136: dynamic update with TSIG (BIND, Infoblox, Windows DNS,
|
||||
// PowerDNS, Knot). Next to each name Rollcall writes a marker TXT at
|
||||
// _rollcall.<name>; every change of an existing record requires that
|
||||
// marker (a prerequisite), a new record requires the name to be free,
|
||||
// so a record someone else owns is never overwritten. What Rollcall
|
||||
// wrote is kept in dns_records, since a zone cannot be listed; each
|
||||
// sync queries those records, so changes made behind its back are seen.
|
||||
//
|
||||
// Names follow the IPAM rule: <host>.<zone> for a machine's main address,
|
||||
// <host>-<iface>.<zone> elsewhere.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"slices"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
)
|
||||
|
||||
type DNSConfig struct {
|
||||
Provider string `json:"provider"` // "" | powerdns | rfc2136
|
||||
Zone string `json:"zone"` // e.g. int.example.com
|
||||
TTL int `json:"ttl"` // default 300
|
||||
SyncInterval string `json:"sync_interval"`
|
||||
// powerdns
|
||||
URL string `json:"url"` // http://dns:8081 (server "localhost")
|
||||
APIKeyFile string `json:"api_key_file"` // X-API-Key
|
||||
// rfc2136
|
||||
Server string `json:"server"` // host:53
|
||||
TSIGName string `json:"tsig_name"` // key name
|
||||
TSIGAlgorithm string `json:"tsig_algorithm"` // hmac-sha256 (default), hmac-sha512 ...
|
||||
TSIGSecretFile string `json:"tsig_secret_file"` // base64 secret
|
||||
}
|
||||
|
||||
const dnsMarkerPrefix = "_rollcall."
|
||||
const dnsMarkerValue = "heritage=rollcall"
|
||||
|
||||
type dnsRecord struct {
|
||||
Name string `json:"name"` // fqdn with the trailing dot
|
||||
Type string `json:"type"` // A | AAAA
|
||||
Values []string `json:"values"`
|
||||
}
|
||||
|
||||
func (r dnsRecord) key() string { return r.Name + " " + r.Type }
|
||||
|
||||
type dnsReport struct {
|
||||
Provider string `json:"provider"`
|
||||
Zone string `json:"zone"`
|
||||
At time.Time `json:"at"`
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
Upserted []string `json:"upserted"`
|
||||
Deleted []string `json:"deleted"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (r *dnsReport) fail(e string) { r.Error = joinErr(r.Error, e) }
|
||||
|
||||
type dnsProvider interface {
|
||||
Name() string
|
||||
// Owned: Rollcall's records in the zone, and the names+types someone else holds.
|
||||
Owned(ctx context.Context) (owned map[string]dnsRecord, foreign map[string]bool, err error)
|
||||
// Upsert writes one RRset; conflict when the provider refuses because it is not ours.
|
||||
Upsert(ctx context.Context, r dnsRecord, existed bool) (conflict bool, err error)
|
||||
Delete(ctx context.Context, r dnsRecord) error
|
||||
}
|
||||
|
||||
type dnsState struct {
|
||||
mu sync.Mutex
|
||||
kick chan struct{}
|
||||
last dnsReport
|
||||
}
|
||||
|
||||
func (c DNSConfig) zone() string { return dns.Fqdn(strings.ToLower(strings.TrimSuffix(c.Zone, "."))) }
|
||||
func (c DNSConfig) ttl() int {
|
||||
if c.TTL > 0 {
|
||||
return c.TTL
|
||||
}
|
||||
return 300
|
||||
}
|
||||
|
||||
func newDNSProvider(c DNSConfig, st *Store) (dnsProvider, error) {
|
||||
if c.Provider == "" {
|
||||
return nil, nil
|
||||
}
|
||||
if c.Zone == "" {
|
||||
return nil, errors.New("dns.zone is required")
|
||||
}
|
||||
switch c.Provider {
|
||||
case "powerdns":
|
||||
key, err := os.ReadFile(c.APIKeyFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dns.api_key_file: %w", err)
|
||||
}
|
||||
if c.URL == "" {
|
||||
return nil, errors.New("dns.url is required")
|
||||
}
|
||||
return &pdnsProvider{base: strings.TrimRight(c.URL, "/") + "/api/v1/servers/localhost", key: strings.TrimSpace(string(key)),
|
||||
zone: c.zone(), ttl: c.ttl(), hc: &http.Client{Timeout: 30 * time.Second}}, nil
|
||||
case "rfc2136":
|
||||
sec, err := os.ReadFile(c.TSIGSecretFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("dns.tsig_secret_file: %w", err)
|
||||
}
|
||||
alg := strings.ToLower(orDefault(c.TSIGAlgorithm, "hmac-sha256"))
|
||||
if !strings.HasSuffix(alg, ".") {
|
||||
alg += "."
|
||||
}
|
||||
if c.Server == "" || c.TSIGName == "" {
|
||||
return nil, errors.New("dns.server and dns.tsig_name are required")
|
||||
}
|
||||
return &rfc2136Provider{server: c.Server, zone: c.zone(), ttl: c.ttl(), keyName: dns.Fqdn(c.TSIGName), alg: alg,
|
||||
secret: strings.TrimSpace(string(sec)), store: st}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("dns.provider %q: powerdns or rfc2136", c.Provider)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- App side
|
||||
|
||||
func (a *App) startDNS() error {
|
||||
p, err := newDNSProvider(a.cfg.DNS, a.store)
|
||||
if err != nil || p == nil {
|
||||
return err
|
||||
}
|
||||
a.dnsp = p
|
||||
a.dnsSt = &dnsState{kick: make(chan struct{}, 1)}
|
||||
every := 5 * time.Minute
|
||||
if d, err := time.ParseDuration(a.cfg.DNS.SyncInterval); err == nil && d >= 10*time.Second {
|
||||
every = d
|
||||
}
|
||||
go func() {
|
||||
a.kickDNS()
|
||||
for {
|
||||
select {
|
||||
case <-a.dnsSt.kick:
|
||||
case <-time.After(every):
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
rep := a.reconcileDNS(ctx, false)
|
||||
cancel()
|
||||
if rep.Error != "" {
|
||||
log.Printf("dns %s: %s", p.Name(), rep.Error)
|
||||
} else if n := len(rep.Upserted) + len(rep.Deleted) + len(rep.Conflicts); n > 0 {
|
||||
log.Printf("dns %s: wrote %d, deleted %d, conflicts %d %v", p.Name(), len(rep.Upserted), len(rep.Deleted), len(rep.Conflicts), rep.Conflicts)
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("dns: %s for zone %s", p.Name(), a.cfg.DNS.zone())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (a *App) kickDNS() {
|
||||
if a.dnsSt == nil {
|
||||
return
|
||||
}
|
||||
select {
|
||||
case a.dnsSt.kick <- struct{}{}:
|
||||
default:
|
||||
}
|
||||
}
|
||||
|
||||
// desiredDNS groups Rollcall's addresses into RRsets inside the zone.
|
||||
func desiredDNS(local []IPAddr, subnets []Subnet, zone string) map[string]dnsRecord {
|
||||
want := map[string]dnsRecord{}
|
||||
z := strings.TrimSuffix(zone, ".")
|
||||
for _, x := range local {
|
||||
name := dnsNameIn(x, subnetByRef(subnets, x.Subnet), z)
|
||||
if name == "" {
|
||||
continue
|
||||
}
|
||||
fq := dns.Fqdn(name)
|
||||
if !dns.IsSubDomain(dns.Fqdn(z), fq) {
|
||||
continue
|
||||
}
|
||||
ip, ok := parseAddr(x.IP)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
t := "A"
|
||||
if ip.Is6() {
|
||||
t = "AAAA"
|
||||
}
|
||||
r := want[fq+" "+t]
|
||||
r.Name, r.Type = fq, t
|
||||
if !hasTag(r.Values, ip.String()) {
|
||||
r.Values = append(r.Values, ip.String())
|
||||
}
|
||||
sort.Strings(r.Values)
|
||||
want[r.key()] = r
|
||||
}
|
||||
return want
|
||||
}
|
||||
|
||||
func (a *App) reconcileDNS(ctx context.Context, dry bool) dnsReport {
|
||||
rep := dnsReport{At: time.Now().UTC(), DryRun: dry, Upserted: []string{}, Deleted: []string{}, Conflicts: []string{}}
|
||||
if a.dnsp == nil {
|
||||
rep.Error = "no DNS provider configured (dns.provider in config.json)"
|
||||
return rep
|
||||
}
|
||||
rep.Provider, rep.Zone = a.dnsp.Name(), a.cfg.DNS.zone()
|
||||
a.dnsSt.mu.Lock()
|
||||
defer a.dnsSt.mu.Unlock()
|
||||
defer func() {
|
||||
if !dry {
|
||||
a.dnsSt.last = rep
|
||||
}
|
||||
}()
|
||||
local, err := a.store.IPs(ctx, "", "")
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
a.reconcileDNSWith(ctx, desiredDNS(local, a.subnetsSnapshot(), a.cfg.DNS.zone()), &rep)
|
||||
return rep
|
||||
}
|
||||
|
||||
func (a *App) reconcileDNSWith(ctx context.Context, want map[string]dnsRecord, rep *dnsReport) {
|
||||
owned, foreign, err := a.dnsp.Owned(ctx)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return
|
||||
}
|
||||
keys := make([]string, 0, len(want))
|
||||
for k := range want {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
w := want[k]
|
||||
o, mine := owned[k]
|
||||
if mine && strings.Join(o.Values, ",") == strings.Join(w.Values, ",") {
|
||||
continue
|
||||
}
|
||||
if !mine && foreign[k] {
|
||||
rep.Conflicts = append(rep.Conflicts, strings.TrimSuffix(k, " A")+" exists and is not Rollcall's")
|
||||
continue
|
||||
}
|
||||
if rep.DryRun {
|
||||
rep.Upserted = append(rep.Upserted, k)
|
||||
continue
|
||||
}
|
||||
conflict, err := a.dnsp.Upsert(ctx, w, mine)
|
||||
switch {
|
||||
case err != nil:
|
||||
rep.fail(fmt.Sprintf("%s: %v", k, err))
|
||||
continue
|
||||
case conflict:
|
||||
rep.Conflicts = append(rep.Conflicts, k+" exists and is not Rollcall's")
|
||||
default:
|
||||
rep.Upserted = append(rep.Upserted, k)
|
||||
}
|
||||
}
|
||||
var gone []string
|
||||
for k := range owned {
|
||||
if _, ok := want[k]; !ok {
|
||||
gone = append(gone, k)
|
||||
}
|
||||
}
|
||||
sort.Strings(gone)
|
||||
for _, k := range gone {
|
||||
rep.Deleted = append(rep.Deleted, k)
|
||||
if !rep.DryRun {
|
||||
if err := a.dnsp.Delete(ctx, owned[k]); err != nil {
|
||||
rep.fail(fmt.Sprintf("delete %s: %v", k, err))
|
||||
rep.Deleted = rep.Deleted[:len(rep.Deleted)-1]
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleDNS(w http.ResponseWriter, r *http.Request) {
|
||||
if a.dnsp == nil {
|
||||
writeJSON(w, 200, obj{"provider": "", "comment": "no DNS provider (an IPAM authority's DNS sync, or dnsmasq host files only)"})
|
||||
return
|
||||
}
|
||||
a.dnsSt.mu.Lock()
|
||||
last := a.dnsSt.last
|
||||
a.dnsSt.mu.Unlock()
|
||||
writeJSON(w, 200, obj{"provider": a.dnsp.Name(), "zone": a.cfg.DNS.zone(), "last_sync": last})
|
||||
}
|
||||
|
||||
func (a *App) handleDNSSync(w http.ResponseWriter, r *http.Request) {
|
||||
rep := a.reconcileDNS(r.Context(), r.URL.Query().Get("dry") != "")
|
||||
code := 200
|
||||
if rep.Error != "" {
|
||||
code = 502
|
||||
if a.dnsp == nil {
|
||||
code = 400
|
||||
}
|
||||
}
|
||||
writeJSON(w, code, rep)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- PowerDNS
|
||||
|
||||
type pdnsProvider struct {
|
||||
base, key, zone string
|
||||
ttl int
|
||||
hc *http.Client
|
||||
}
|
||||
|
||||
func (p *pdnsProvider) Name() string { return "powerdns" }
|
||||
|
||||
func (p *pdnsProvider) do(ctx context.Context, method, path string, body, out any) error {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, p.base+path, rd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Set("X-API-Key", p.key)
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := p.hc.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
|
||||
if resp.StatusCode >= 300 {
|
||||
return fmt.Errorf("HTTP %d: %s", resp.StatusCode, trunc(strings.TrimSpace(string(b)), 300))
|
||||
}
|
||||
if out != nil {
|
||||
return json.Unmarshal(b, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *pdnsProvider) Owned(ctx context.Context) (map[string]dnsRecord, map[string]bool, error) {
|
||||
var z struct {
|
||||
RRsets []struct {
|
||||
Name string `json:"name"`
|
||||
Type string `json:"type"`
|
||||
Records []struct{ Content string } `json:"records"`
|
||||
Comments []struct{ Account string } `json:"comments"`
|
||||
} `json:"rrsets"`
|
||||
}
|
||||
if err := p.do(ctx, "GET", "/zones/"+url.PathEscape(p.zone), nil, &z); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
owned, foreign := map[string]dnsRecord{}, map[string]bool{}
|
||||
for _, rr := range z.RRsets {
|
||||
if rr.Type != "A" && rr.Type != "AAAA" {
|
||||
continue
|
||||
}
|
||||
r := dnsRecord{Name: strings.ToLower(rr.Name), Type: rr.Type}
|
||||
for _, x := range rr.Records {
|
||||
r.Values = append(r.Values, x.Content)
|
||||
}
|
||||
sort.Strings(r.Values)
|
||||
mine := false
|
||||
for _, c := range rr.Comments {
|
||||
mine = mine || c.Account == "rollcall"
|
||||
}
|
||||
if mine {
|
||||
owned[r.key()] = r
|
||||
} else {
|
||||
foreign[r.key()] = true
|
||||
}
|
||||
}
|
||||
return owned, foreign, nil
|
||||
}
|
||||
|
||||
func (p *pdnsProvider) Upsert(ctx context.Context, r dnsRecord, _ bool) (bool, error) {
|
||||
recs := []obj{}
|
||||
for _, v := range r.Values {
|
||||
recs = append(recs, obj{"content": v, "disabled": false})
|
||||
}
|
||||
return false, p.do(ctx, "PATCH", "/zones/"+url.PathEscape(p.zone), obj{"rrsets": []obj{{"name": r.Name, "type": r.Type, "ttl": p.ttl,
|
||||
"changetype": "REPLACE", "records": recs, "comments": []obj{{"content": "rollcall", "account": "rollcall"}}}}}, nil)
|
||||
}
|
||||
|
||||
func (p *pdnsProvider) Delete(ctx context.Context, r dnsRecord) error {
|
||||
return p.do(ctx, "PATCH", "/zones/"+url.PathEscape(p.zone), obj{"rrsets": []obj{{"name": r.Name, "type": r.Type, "changetype": "DELETE"}}}, nil)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- RFC 2136
|
||||
|
||||
const dnsRecordsSchema = `
|
||||
CREATE TABLE IF NOT EXISTS dns_records (
|
||||
zone text NOT NULL,
|
||||
name text NOT NULL,
|
||||
type text NOT NULL,
|
||||
vals jsonb NOT NULL,
|
||||
updated timestamptz NOT NULL DEFAULT now(),
|
||||
PRIMARY KEY (zone, name, type)
|
||||
);
|
||||
`
|
||||
|
||||
type rfc2136Provider struct {
|
||||
server, zone, keyName, alg, secret string
|
||||
ttl int
|
||||
store *Store
|
||||
}
|
||||
|
||||
func (p *rfc2136Provider) Name() string { return "rfc2136" }
|
||||
|
||||
func (p *rfc2136Provider) Owned(ctx context.Context) (map[string]dnsRecord, map[string]bool, error) {
|
||||
rows, err := p.store.pool.Query(ctx, `SELECT name, type, vals FROM dns_records WHERE zone=$1`, p.zone)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
owned := map[string]dnsRecord{}
|
||||
for rows.Next() {
|
||||
var r dnsRecord
|
||||
var vals []byte
|
||||
if err := rows.Scan(&r.Name, &r.Type, &vals); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
_ = json.Unmarshal(vals, &r.Values)
|
||||
owned[r.key()] = r
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
rows.Close()
|
||||
// the table says what we wrote; the server says what is there now. A record
|
||||
// changed or a marker removed behind our back shows up as a difference, and
|
||||
// the next Upsert either repairs it or finds someone else's record (conflict).
|
||||
for k, r := range owned {
|
||||
live, err := p.lookup(ctx, r.Name, r.Type)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
marker, err := p.lookup(ctx, dnsMarkerPrefix+r.Name, "TXT")
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if !slices.Contains(marker, strconv.Quote(dnsMarkerValue)) {
|
||||
live = nil
|
||||
}
|
||||
r.Values = live
|
||||
owned[k] = r
|
||||
}
|
||||
return owned, map[string]bool{}, nil // foreign names show up as refused updates
|
||||
}
|
||||
|
||||
// lookup asks the primary for one RRset; it returns the values, sorted, nil if none.
|
||||
func (p *rfc2136Provider) lookup(ctx context.Context, name, typ string) ([]string, error) {
|
||||
m := new(dns.Msg)
|
||||
m.SetQuestion(name, dns.StringToType[typ])
|
||||
m.SetTsig(p.keyName, p.alg, 300, time.Now().Unix())
|
||||
c := &dns.Client{Net: "tcp", Timeout: 10 * time.Second, TsigSecret: map[string]string{p.keyName: p.secret}}
|
||||
r, _, err := c.ExchangeContext(ctx, m, p.server)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("query %s %s: %w", name, typ, err)
|
||||
}
|
||||
if r.Rcode != dns.RcodeSuccess && r.Rcode != dns.RcodeNameError {
|
||||
return nil, fmt.Errorf("query %s %s: %s", name, typ, dns.RcodeToString[r.Rcode])
|
||||
}
|
||||
var out []string
|
||||
for _, rr := range r.Answer {
|
||||
if rr.Header().Rrtype == dns.StringToType[typ] && strings.EqualFold(rr.Header().Name, name) {
|
||||
out = append(out, strings.TrimPrefix(rr.String(), rr.Header().String()))
|
||||
}
|
||||
}
|
||||
sort.Strings(out)
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *rfc2136Provider) marker(name string) dns.RR {
|
||||
rr, _ := dns.NewRR(fmt.Sprintf("%s%s %d IN TXT %q", dnsMarkerPrefix, name, p.ttl, dnsMarkerValue))
|
||||
return rr
|
||||
}
|
||||
|
||||
func (p *rfc2136Provider) rrs(r dnsRecord) []dns.RR {
|
||||
var out []dns.RR
|
||||
for _, v := range r.Values {
|
||||
if rr, err := dns.NewRR(fmt.Sprintf("%s %d IN %s %s", r.Name, p.ttl, r.Type, v)); err == nil {
|
||||
out = append(out, rr)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// send signs and sends an update; it returns the response code.
|
||||
func (p *rfc2136Provider) send(m *dns.Msg) (int, error) {
|
||||
m.SetTsig(p.keyName, p.alg, 300, time.Now().Unix())
|
||||
c := &dns.Client{Net: "tcp", Timeout: 10 * time.Second, TsigSecret: map[string]string{p.keyName: p.secret}}
|
||||
r, _, err := c.Exchange(m, p.server)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return r.Rcode, nil
|
||||
}
|
||||
|
||||
func (p *rfc2136Provider) Upsert(ctx context.Context, r dnsRecord, existed bool) (bool, error) {
|
||||
conflict, err := p.upsert(ctx, r, existed)
|
||||
if conflict && existed && err == nil {
|
||||
// our marker is gone: if the name is free now, write it again as new;
|
||||
// if someone else's record is there, that is a real conflict
|
||||
return p.upsert(ctx, r, false)
|
||||
}
|
||||
return conflict, err
|
||||
}
|
||||
|
||||
func (p *rfc2136Provider) upsert(ctx context.Context, r dnsRecord, existed bool) (bool, error) {
|
||||
m := new(dns.Msg)
|
||||
m.SetUpdate(p.zone)
|
||||
set := &dns.ANY{Hdr: dns.RR_Header{Name: r.Name, Rrtype: dns.StringToType[r.Type], Class: dns.ClassINET}}
|
||||
if existed {
|
||||
m.Used([]dns.RR{p.marker(r.Name)}) // our marker must still be there
|
||||
m.RemoveRRset([]dns.RR{set})
|
||||
} else {
|
||||
m.RRsetNotUsed([]dns.RR{set}) // and the name must be free of this type
|
||||
}
|
||||
m.Insert(p.rrs(r))
|
||||
m.Insert([]dns.RR{p.marker(r.Name)})
|
||||
rc, err := p.send(m)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
switch rc {
|
||||
case dns.RcodeSuccess:
|
||||
vals, _ := json.Marshal(r.Values)
|
||||
_, err := p.store.pool.Exec(ctx, `INSERT INTO dns_records(zone, name, type, vals) VALUES ($1,$2,$3,$4)
|
||||
ON CONFLICT (zone, name, type) DO UPDATE SET vals=$4, updated=now()`, p.zone, r.Name, r.Type, vals)
|
||||
return false, err
|
||||
case dns.RcodeYXRrset, dns.RcodeNXRrset, dns.RcodeYXDomain, dns.RcodeNameError:
|
||||
return true, nil
|
||||
}
|
||||
return false, fmt.Errorf("update refused: %s", dns.RcodeToString[rc])
|
||||
}
|
||||
|
||||
func (p *rfc2136Provider) Delete(ctx context.Context, r dnsRecord) error {
|
||||
m := new(dns.Msg)
|
||||
m.SetUpdate(p.zone)
|
||||
m.Used([]dns.RR{p.marker(r.Name)})
|
||||
m.RemoveRRset([]dns.RR{&dns.ANY{Hdr: dns.RR_Header{Name: r.Name, Rrtype: dns.StringToType[r.Type], Class: dns.ClassINET}}})
|
||||
var others int
|
||||
if err := p.store.pool.QueryRow(ctx, `SELECT count(*) FROM dns_records WHERE zone=$1 AND name=$2 AND type<>$3`, p.zone, r.Name, r.Type).Scan(&others); err != nil {
|
||||
return err // not knowing, the marker could go while an AAAA of ours still needs it
|
||||
}
|
||||
if others == 0 { // the last of our types at this name: the marker goes too
|
||||
m.RemoveRRset([]dns.RR{&dns.ANY{Hdr: dns.RR_Header{Name: dnsMarkerPrefix + r.Name, Rrtype: dns.TypeTXT, Class: dns.ClassINET}}})
|
||||
}
|
||||
rc, err := p.send(m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if rc != dns.RcodeSuccess && rc != dns.RcodeNXRrset { // NXRRSET: our marker is gone, it is not ours any more
|
||||
return fmt.Errorf("update refused: %s", dns.RcodeToString[rc])
|
||||
}
|
||||
_, err = p.store.pool.Exec(ctx, `DELETE FROM dns_records WHERE zone=$1 AND name=$2 AND type=$3`, p.zone, r.Name, r.Type)
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestDesiredDNS(t *testing.T) {
|
||||
subnets := []Subnet{{CIDR: "10.0.0.0/24", Purpose: "provisioning"}, {CIDR: "10.20.0.0/24", Purpose: "storage"}, {CIDR: "fd00::/64", Purpose: "provisioning"}}
|
||||
local := []IPAddr{
|
||||
{IP: "10.0.0.50", Subnet: "10.0.0.0/24", MachineID: "m1", Hostname: "node01", Iface: "eno1", Kind: "auto"},
|
||||
{IP: "fd00::50", Subnet: "fd00::/64", MachineID: "m1", Hostname: "node01", Iface: "eno1", Kind: "auto"},
|
||||
{IP: "10.20.0.50", Subnet: "10.20.0.0/24", MachineID: "m1", Hostname: "node01", Iface: "bond0.200", Kind: "auto"},
|
||||
{IP: "10.0.0.5", Subnet: "10.0.0.0/24", Hostname: "core switch", Kind: "reserved"}, // not a DNS label: skipped
|
||||
{IP: "10.0.0.6", Subnet: "10.0.0.0/24", Hostname: "vip", Kind: "reserved"},
|
||||
}
|
||||
got := desiredDNS(local, subnets, "int.example.")
|
||||
var keys []string
|
||||
for k, r := range got {
|
||||
keys = append(keys, k+"="+strings.Join(r.Values, ","))
|
||||
}
|
||||
sort.Strings(keys)
|
||||
want := "node01-bond0-200.int.example. A=10.20.0.50 node01.int.example. A=10.0.0.50 node01.int.example. AAAA=fd00::50 vip.int.example. A=10.0.0.6"
|
||||
if strings.Join(keys, " ") != want {
|
||||
t.Errorf("desired:\n got %s\nwant %s", strings.Join(keys, " "), want)
|
||||
}
|
||||
}
|
||||
|
||||
// fakePDNS: a zone held in memory behind the PowerDNS zone API.
|
||||
type fakePDNS struct {
|
||||
mu sync.Mutex
|
||||
rrsets map[string]map[string]any // name type -> rrset
|
||||
}
|
||||
|
||||
func (f *fakePDNS) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if r.Header.Get("X-API-Key") != "k" {
|
||||
http.Error(w, "auth", 401)
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
out := []any{}
|
||||
for _, v := range f.rrsets {
|
||||
out = append(out, v)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{"rrsets": out})
|
||||
case "PATCH":
|
||||
var b struct{ RRsets []map[string]any }
|
||||
_ = json.NewDecoder(r.Body).Decode(&b)
|
||||
for _, rr := range b.RRsets {
|
||||
k := rr["name"].(string) + " " + rr["type"].(string)
|
||||
if rr["changetype"] == "DELETE" {
|
||||
delete(f.rrsets, k)
|
||||
} else {
|
||||
f.rrsets[k] = rr
|
||||
}
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
}
|
||||
|
||||
func rrset(name, typ string, account string, vals ...string) map[string]any {
|
||||
recs := []any{}
|
||||
for _, v := range vals {
|
||||
recs = append(recs, map[string]any{"content": v})
|
||||
}
|
||||
c := []any{}
|
||||
if account != "" {
|
||||
c = append(c, map[string]any{"account": account})
|
||||
}
|
||||
return map[string]any{"name": name, "type": typ, "records": recs, "comments": c}
|
||||
}
|
||||
|
||||
func TestDNSReconcilePowerDNS(t *testing.T) {
|
||||
f := &fakePDNS{rrsets: map[string]map[string]any{
|
||||
"printer.int.example. A": rrset("printer.int.example.", "A", "", "10.0.0.9"), // someone else's
|
||||
"node02.int.example. A": rrset("node02.int.example.", "A", "", "10.0.0.99"), // someone else's, same name as ours
|
||||
"node01.int.example. A": rrset("node01.int.example.", "A", "rollcall", "10.0.0.40"), // ours, outdated
|
||||
"gone.int.example. A": rrset("gone.int.example.", "A", "rollcall", "10.0.0.41"), // ours, released
|
||||
"other.int.example. A": rrset("other.int.example.", "A", "netbox", "10.0.0.42"), // another tool's
|
||||
}}
|
||||
srv := httptest.NewServer(f)
|
||||
defer srv.Close()
|
||||
keyFile := filepath.Join(t.TempDir(), "key")
|
||||
_ = os.WriteFile(keyFile, []byte("k\n"), 0o600)
|
||||
p, err := newDNSProvider(DNSConfig{Provider: "powerdns", Zone: "int.example", URL: srv.URL, APIKeyFile: keyFile}, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &App{cfg: &Config{}, dnsp: p, dnsSt: &dnsState{kick: make(chan struct{}, 1)}}
|
||||
want := map[string]dnsRecord{
|
||||
"node01.int.example. A": {Name: "node01.int.example.", Type: "A", Values: []string{"10.0.0.50"}},
|
||||
"node02.int.example. A": {Name: "node02.int.example.", Type: "A", Values: []string{"10.0.0.51"}},
|
||||
"node03.int.example. A": {Name: "node03.int.example.", Type: "A", Values: []string{"10.0.0.52"}},
|
||||
}
|
||||
rep := dnsReport{}
|
||||
a.reconcileDNSWith(context.Background(), want, &rep)
|
||||
if rep.Error != "" {
|
||||
t.Fatal(rep.Error)
|
||||
}
|
||||
if strings.Join(rep.Upserted, ",") != "node01.int.example. A,node03.int.example. A" || strings.Join(rep.Deleted, ",") != "gone.int.example. A" || len(rep.Conflicts) != 1 {
|
||||
t.Errorf("report: %+v", rep)
|
||||
}
|
||||
vals := func(k string) string {
|
||||
rr, ok := f.rrsets[k]
|
||||
if !ok {
|
||||
return "-"
|
||||
}
|
||||
var v []string
|
||||
for _, r := range rr["records"].([]any) {
|
||||
v = append(v, r.(map[string]any)["content"].(string))
|
||||
}
|
||||
return strings.Join(v, ",")
|
||||
}
|
||||
for k, w := range map[string]string{"printer.int.example. A": "10.0.0.9", "node02.int.example. A": "10.0.0.99", "node01.int.example. A": "10.0.0.50",
|
||||
"node03.int.example. A": "10.0.0.52", "gone.int.example. A": "-", "other.int.example. A": "10.0.0.42"} {
|
||||
if got := vals(k); got != w {
|
||||
t.Errorf("%s: %s, want %s", k, got, w)
|
||||
}
|
||||
}
|
||||
again := dnsReport{}
|
||||
a.reconcileDNSWith(context.Background(), want, &again)
|
||||
if len(again.Upserted)+len(again.Deleted) != 0 {
|
||||
t.Errorf("second pass changed things: %+v", again)
|
||||
}
|
||||
}
|
||||
+30
-3
@@ -178,7 +178,7 @@ func doctorCmd(args []string) error {
|
||||
// dnsmasq
|
||||
if out, _ := exec.Command("systemctl", "is-active", "dnsmasq").Output(); strings.TrimSpace(string(out)) != "active" {
|
||||
d.fail("dnsmasq", "not running", "journalctl -u dnsmasq -n 20")
|
||||
} else if _, err := os.Stat(filepath.Join(cfg.DnsmasqDir, "rollcall.conf.rejected")); err == nil {
|
||||
} else if _, err := os.Stat("/var/lib/rollcall-dnsmasq/rollcall.conf.rejected"); err == nil {
|
||||
d.fail("dnsmasq", "the generated configuration was rejected; the last working one is in use",
|
||||
"journalctl -u rollcall-dnsmasq; a hand-written DHCP/TFTP file in /etc/dnsmasq.d usually conflicts")
|
||||
} else {
|
||||
@@ -212,7 +212,18 @@ func doctorCmd(args []string) error {
|
||||
}
|
||||
}
|
||||
|
||||
// other DHCP servers on the provisioning network
|
||||
// other DHCP servers on the provisioning network (expected when this rack proxies)
|
||||
proxying := false
|
||||
if c, err := newClient(); err == nil {
|
||||
var subs []Subnet
|
||||
if c.do("GET", "/api/v1/subnets", nil, &subs) == nil {
|
||||
for _, s := range subs {
|
||||
if s.DHCPMode == "proxy" && (s.PrimaryRack == cfg.RackName || s.SecondaryRack == cfg.RackName) {
|
||||
proxying = true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if iface != "" && !*noProbe && !ports["udp/67"] {
|
||||
d.ok("dhcp probe", "skipped: this host serves no DHCP yet")
|
||||
} else if iface != "" && !*noProbe {
|
||||
@@ -228,9 +239,13 @@ func doctorCmd(args []string) error {
|
||||
}
|
||||
}
|
||||
switch {
|
||||
case proxying && len(foreign) > 0:
|
||||
d.ok("dhcp probe", "the site's DHCP server "+strings.Join(foreign, ", ")+" hands out addresses; this host answers PXE (proxy)")
|
||||
case proxying:
|
||||
d.fail("dhcp probe", "proxy DHCP, but no DHCP server answers on "+iface, "machines would get no address: the site's DHCP server must serve this VLAN (or a relay must reach it)")
|
||||
case len(foreign) > 0:
|
||||
d.fail("dhcp probe", "another DHCP server answers on "+iface+": "+strings.Join(foreign, ", "),
|
||||
"machines may PXE-boot from it; switch it off for this VLAN, or use --no-dhcp and let it hand out next-server "+srvHost)
|
||||
"machines may PXE-boot from it: switch it off for this VLAN, or keep it and proxy (bootstrap.sh --iface IFACE --proxy-dhcp, or DHCP mode proxy in the web UI)")
|
||||
case len(servers) == 0:
|
||||
d.warn("dhcp probe", "no DHCP server answered on "+iface, "is a VLAN with dhcp assigned to this rack? (rollcall vlan add ... -dhcp)")
|
||||
default:
|
||||
@@ -268,6 +283,18 @@ func doctorCmd(args []string) error {
|
||||
d.ok("os images", fmt.Sprintf("%d in the catalog", len(oss)))
|
||||
}
|
||||
}
|
||||
var dnsSt map[string]any
|
||||
if err := c.do("GET", "/api/v1/dns", nil, &dnsSt); err == nil && dnsSt["provider"] != "" && dnsSt["provider"] != nil {
|
||||
last, _ := dnsSt["last_sync"].(map[string]any)
|
||||
switch e, _ := last["error"].(string); {
|
||||
case e != "":
|
||||
d.fail("dns provider", e, "check dns.* in config.json; rollcall dns sync")
|
||||
case last["conflicts"] != nil && len(last["conflicts"].([]any)) > 0:
|
||||
d.warn("dns provider", fmt.Sprintf("%v: %d names held by someone else", dnsSt["provider"], len(last["conflicts"].([]any))), "rollcall dns: those records are left alone")
|
||||
default:
|
||||
d.ok("dns provider", fmt.Sprintf("%v, zone %v, in sync", dnsSt["provider"], dnsSt["zone"]))
|
||||
}
|
||||
}
|
||||
var ipam map[string]any
|
||||
if err := c.do("GET", "/api/v1/ipam", nil, &ipam); err == nil && ipam["provider"] != "" && ipam["provider"] != nil {
|
||||
last, _ := ipam["last_sync"].(map[string]any)
|
||||
|
||||
@@ -0,0 +1,893 @@
|
||||
package main
|
||||
|
||||
// Firmware: a repository of vendor packages on the region, each machine's
|
||||
// firmware inventory read from its BMC, and update jobs.
|
||||
//
|
||||
// <firmware_dir>/<id>/meta.json + the package file, served under /firmware/
|
||||
//
|
||||
// A package is for one vendor profile (the Redfish driver: idrac, ilo, ...),
|
||||
// one component (bios, bmc, nic, raid, cpld, psu, drive, other), a version,
|
||||
// and optionally models. An update is a job (kind "firmware") that the region
|
||||
// drives with short rack commands: fw-start hands the package to the BMC,
|
||||
// fw-poll watches the BMC's task until it ends, fw-inventory reads the
|
||||
// versions afterwards. The task is kept in the job, so a region restart
|
||||
// resumes watching instead of losing the update.
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type FWPackage struct {
|
||||
ID string `json:"id"`
|
||||
Vendor string `json:"vendor"` // a Redfish driver: idrac, ilo, xcc, supermicro, ...
|
||||
Component string `json:"component"` // bios | bmc | nic | raid | cpld | psu | drive | other
|
||||
Version string `json:"version"`
|
||||
Models []string `json:"models,omitempty"` // model substrings it is for; none = any of the vendor's
|
||||
File string `json:"file"`
|
||||
Size int64 `json:"size"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Source string `json:"source,omitempty"`
|
||||
Notes string `json:"notes,omitempty"`
|
||||
Imported time.Time `json:"imported"`
|
||||
}
|
||||
|
||||
var fwComponents = []string{"bios", "bmc", "nic", "raid", "cpld", "psu", "drive", "other"}
|
||||
|
||||
var fwIDRe = regexp.MustCompile(`^[a-z0-9][a-z0-9._-]{0,80}$`)
|
||||
var fwFileRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._+-]{0,120}$`)
|
||||
|
||||
const firmwareSchema = `
|
||||
CREATE TABLE IF NOT EXISTS firmware_inventory (
|
||||
machine_id text PRIMARY KEY REFERENCES machines(id) ON DELETE CASCADE,
|
||||
at timestamptz NOT NULL DEFAULT now(),
|
||||
components jsonb NOT NULL
|
||||
);
|
||||
`
|
||||
|
||||
// ---------------------------------------------------------------- repository
|
||||
|
||||
func fwList(dir string) ([]FWPackage, error) {
|
||||
ents, err := os.ReadDir(dir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return []FWPackage{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []FWPackage{}
|
||||
for _, e := range ents {
|
||||
b, err := os.ReadFile(filepath.Join(dir, e.Name(), "meta.json"))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var p FWPackage
|
||||
if json.Unmarshal(b, &p) == nil && p.ID == e.Name() {
|
||||
out = append(out, p)
|
||||
}
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
a, b := out[i], out[j]
|
||||
if a.Vendor != b.Vendor {
|
||||
return a.Vendor < b.Vendor
|
||||
}
|
||||
if a.Component != b.Component {
|
||||
return a.Component < b.Component
|
||||
}
|
||||
return a.Imported.After(b.Imported)
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func fwGet(dir, id string) (FWPackage, error) {
|
||||
if !fwIDRe.MatchString(id) {
|
||||
return FWPackage{}, fmt.Errorf("no firmware package %q", id)
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(dir, id, "meta.json"))
|
||||
if err != nil {
|
||||
return FWPackage{}, fmt.Errorf("no firmware package %q", id)
|
||||
}
|
||||
var p FWPackage
|
||||
return p, json.Unmarshal(b, &p)
|
||||
}
|
||||
|
||||
// fwCheck validates a package description against what Rollcall knows.
|
||||
func fwCheck(p *FWPackage) error {
|
||||
p.Vendor, p.Component = strings.ToLower(p.Vendor), strings.ToLower(p.Component)
|
||||
if d, ok := driverByName(p.Vendor); !ok || d.Family != "redfish" {
|
||||
var names []string
|
||||
for _, r := range redfishProfiles {
|
||||
names = append(names, r.Driver)
|
||||
}
|
||||
return fmt.Errorf("vendor is a Redfish driver: %s", strings.Join(names, ", "))
|
||||
}
|
||||
if !hasTag(fwComponents, p.Component) {
|
||||
return fmt.Errorf("component is one of %s", strings.Join(fwComponents, ", "))
|
||||
}
|
||||
if strings.TrimSpace(p.Version) == "" || strings.ContainsAny(p.Version, "\r\n") {
|
||||
return errors.New("version is required")
|
||||
}
|
||||
if !fwFileRe.MatchString(p.File) {
|
||||
return fmt.Errorf("file name %q: letters, digits and ._+- only", p.File)
|
||||
}
|
||||
if fs := redfishProfileFor(p.Vendor).FWFormats; len(fs) > 0 {
|
||||
ok := false
|
||||
for _, ext := range fs {
|
||||
ok = ok || strings.HasSuffix(strings.ToLower(p.File), ext)
|
||||
}
|
||||
if !ok {
|
||||
return fmt.Errorf("%s takes %s packages, not %s", p.Vendor, strings.Join(fs, " "), p.File)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fwImport copies (or downloads) a package into the repository.
|
||||
func fwImport(ctx context.Context, dir, src string, p FWPackage, wantSum string) (*FWPackage, error) {
|
||||
if p.File == "" {
|
||||
p.File = path.Base(strings.SplitN(src, "?", 2)[0])
|
||||
}
|
||||
if err := fwCheck(&p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var in io.ReadCloser
|
||||
if strings.HasPrefix(src, "https://") || strings.HasPrefix(src, "http://") {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", src, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
resp.Body.Close()
|
||||
return nil, fmt.Errorf("%s: %s", src, resp.Status)
|
||||
}
|
||||
in = resp.Body
|
||||
} else {
|
||||
f, err := os.Open(src)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
in = f
|
||||
}
|
||||
defer in.Close()
|
||||
if err := os.MkdirAll(dir, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
tmp, err := os.MkdirTemp(dir, ".import-")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer os.RemoveAll(tmp)
|
||||
f, err := os.Create(filepath.Join(tmp, p.File))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(f, h), in)
|
||||
if cerr := f.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p.Size, p.SHA256 = n, hex.EncodeToString(h.Sum(nil))
|
||||
if wantSum != "" && !strings.EqualFold(wantSum, p.SHA256) {
|
||||
return nil, fmt.Errorf("sha256 %s, expected %s", p.SHA256, wantSum)
|
||||
}
|
||||
if p.Vendor == "ilo" && strings.HasSuffix(strings.ToLower(p.File), ".fwpkg") {
|
||||
if err := fwpkgExtract(tmp, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
if p.Source == "" {
|
||||
p.Source = src
|
||||
}
|
||||
p.Imported = time.Now().UTC()
|
||||
slug := strings.Trim(regexp.MustCompile(`[^a-z0-9.]+`).ReplaceAllString(strings.ToLower(p.Vendor+"-"+p.Component+"-"+p.Version), "-"), "-")
|
||||
if len(slug) > 60 {
|
||||
slug = slug[:60]
|
||||
}
|
||||
p.ID = slug + "-" + p.SHA256[:8]
|
||||
mb, _ := json.MarshalIndent(p, "", " ")
|
||||
if err := os.WriteFile(filepath.Join(tmp, "meta.json"), append(mb, '\n'), 0o644); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := os.Chmod(tmp, 0o755); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
dst := filepath.Join(dir, p.ID)
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
return &p, nil // the same package again
|
||||
}
|
||||
if err := os.Rename(tmp, dst); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &p, nil
|
||||
}
|
||||
|
||||
// fwpkgExtract: iLO's SimpleUpdate flashes the iLO .bin or UEFI .signed.flash
|
||||
// inside an HPE Smart Component; given the .fwpkg it answers 200 and flashes
|
||||
// nothing. The .fwpkg is a zip: the one flashable file is kept instead.
|
||||
func fwpkgExtract(dir string, p *FWPackage) error {
|
||||
src := filepath.Join(dir, p.File)
|
||||
zr, err := zip.OpenReader(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s is not a readable .fwpkg: %w", p.File, err)
|
||||
}
|
||||
defer zr.Close()
|
||||
var cands []*zip.File
|
||||
for _, f := range zr.File {
|
||||
n := strings.ToLower(path.Base(f.Name))
|
||||
if !f.FileInfo().IsDir() && (strings.HasSuffix(n, ".bin") || strings.HasSuffix(n, ".flash")) && fwFileRe.MatchString(path.Base(f.Name)) {
|
||||
cands = append(cands, f)
|
||||
}
|
||||
}
|
||||
if len(cands) != 1 {
|
||||
var names []string
|
||||
for _, f := range cands {
|
||||
names = append(names, f.Name)
|
||||
}
|
||||
return fmt.Errorf("%s holds %d files iLO flashes directly (%s); a component only UEFI or an OS agent installs needs SUM/iLOrest", p.File, len(cands), strings.Join(names, ", "))
|
||||
}
|
||||
in, err := cands[0].Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer in.Close()
|
||||
name := path.Base(cands[0].Name)
|
||||
out, err := os.Create(filepath.Join(dir, name))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
h := sha256.New()
|
||||
n, err := io.Copy(io.MultiWriter(out, h), io.LimitReader(in, 1<<30))
|
||||
if cerr := out.Close(); err == nil {
|
||||
err = cerr
|
||||
}
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = os.Remove(src)
|
||||
p.Notes = strings.TrimSpace(p.Notes + fmt.Sprintf(" extracted from %s (sha256 %s)", p.File, p.SHA256))
|
||||
p.File, p.Size, p.SHA256 = name, n, hex.EncodeToString(h.Sum(nil))
|
||||
return nil
|
||||
}
|
||||
|
||||
func fwRemove(dir, id string) error {
|
||||
if !fwIDRe.MatchString(id) {
|
||||
return fmt.Errorf("no firmware package %q", id)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, id, "meta.json")); err != nil {
|
||||
return fmt.Errorf("no firmware package %q", id)
|
||||
}
|
||||
return os.RemoveAll(filepath.Join(dir, id))
|
||||
}
|
||||
|
||||
// fwFor: the packages for this machine (vendor, model), newest first per component.
|
||||
func fwFor(pkgs []FWPackage, vendor, model string) []FWPackage {
|
||||
var out []FWPackage
|
||||
for _, p := range pkgs {
|
||||
if p.Vendor != vendor {
|
||||
continue
|
||||
}
|
||||
if len(p.Models) > 0 {
|
||||
ok := false
|
||||
for _, m := range p.Models {
|
||||
ok = ok || strings.Contains(strings.ToLower(model), strings.ToLower(m))
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- inventory
|
||||
|
||||
type fwInventory struct {
|
||||
At time.Time `json:"at"`
|
||||
Components []fwComponent `json:"components"`
|
||||
}
|
||||
|
||||
func (s *Store) FirmwareInventory(ctx context.Context, id string) (*fwInventory, error) {
|
||||
var inv fwInventory
|
||||
var b []byte
|
||||
err := s.pool.QueryRow(ctx, `SELECT at, components FROM firmware_inventory WHERE machine_id=$1`, id).Scan(&inv.At, &b)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &inv, json.Unmarshal(b, &inv.Components)
|
||||
}
|
||||
|
||||
func (s *Store) PutFirmwareInventory(ctx context.Context, id string, cs []fwComponent) error {
|
||||
b, _ := json.Marshal(cs)
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO firmware_inventory(machine_id, at, components) VALUES ($1, now(), $2)
|
||||
ON CONFLICT (machine_id) DO UPDATE SET at=now(), components=$2`, id, b)
|
||||
return err
|
||||
}
|
||||
|
||||
func (a *App) readFirmware(ctx context.Context, m *Machine) ([]fwComponent, error) {
|
||||
out, err := a.firmwareVia(ctx, m, "fw-inventory", "")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var cs []fwComponent
|
||||
if err := json.Unmarshal([]byte(out), &cs); err != nil {
|
||||
return nil, fmt.Errorf("inventory: %w", err)
|
||||
}
|
||||
return cs, a.store.PutFirmwareInventory(ctx, m.ID, cs)
|
||||
}
|
||||
|
||||
// bmcVendor is the machine's Redfish driver and model, as probed.
|
||||
func bmcVendor(m *Machine) (driver, model string) {
|
||||
var b bmcRecord
|
||||
_ = json.Unmarshal(m.BMC, &b)
|
||||
var rf struct {
|
||||
Model string `json:"model"`
|
||||
}
|
||||
var raw obj
|
||||
_ = json.Unmarshal(m.BMC, &raw)
|
||||
if x, ok := raw["redfish"]; ok {
|
||||
bb, _ := json.Marshal(x)
|
||||
_ = json.Unmarshal(bb, &rf)
|
||||
}
|
||||
model = rf.Model
|
||||
if model == "" {
|
||||
model = deref(m.Product)
|
||||
}
|
||||
return b.Driver, model
|
||||
}
|
||||
|
||||
// fwCompare marks, per component, whether a package in the repository is newer than what is installed.
|
||||
type fwOffer struct {
|
||||
Package FWPackage `json:"package"`
|
||||
Installed string `json:"installed"` // the version now on the component ("" when not seen)
|
||||
Same bool `json:"same"` // the package's version is what is installed
|
||||
}
|
||||
|
||||
func fwOffers(pkgs []FWPackage, inv []fwComponent) []fwOffer {
|
||||
var out []fwOffer
|
||||
for _, p := range pkgs {
|
||||
o := fwOffer{Package: p}
|
||||
for _, c := range inv {
|
||||
if c.Component == p.Component {
|
||||
if o.Installed == "" {
|
||||
o.Installed = c.Version
|
||||
}
|
||||
if fwSameVersion(c.Version, p.Version) {
|
||||
o.Installed, o.Same = c.Version, true
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// fwSameVersion compares loosely: "2.19.1" matches "2.19.1", "v2.19.1" and "2.19.1 (Build 4)".
|
||||
func fwSameVersion(installed, pkg string) bool {
|
||||
norm := func(s string) string { return strings.TrimPrefix(strings.ToLower(strings.TrimSpace(s)), "v") }
|
||||
i, p := norm(installed), norm(pkg)
|
||||
return i == p || (p != "" && (strings.HasPrefix(i, p+" ") || strings.HasPrefix(i, p+"(") || strings.HasPrefix(i, p+"-")))
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- jobs
|
||||
|
||||
type fwSpec struct {
|
||||
Package string `json:"package"`
|
||||
Component string `json:"component"`
|
||||
Version string `json:"version"`
|
||||
Apply string `json:"apply,omitempty"` // Immediate | OnReset
|
||||
Method string `json:"method,omitempty"` // forced upload method
|
||||
Reboot bool `json:"reboot,omitempty"` // power-cycle the host when the update waits for a reset
|
||||
}
|
||||
|
||||
type fwProgress struct {
|
||||
Phase string `json:"phase"` // uploading | flashing | rebooting | verifying
|
||||
Pct int `json:"pct"`
|
||||
Msg string `json:"msg,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Task string `json:"task,omitempty"`
|
||||
}
|
||||
|
||||
func (a *App) setFWProgress(ctx context.Context, j *Job, p fwProgress) {
|
||||
b, _ := json.Marshal(p)
|
||||
_ = a.store.JobProgress(ctx, j.ID, b)
|
||||
}
|
||||
|
||||
// firmwareURL is where the BMC downloads the package: the machine's rack
|
||||
// controller, or firmware_base on the BMC record when BMCs reach another address.
|
||||
func (a *App) firmwareURL(m *Machine, p FWPackage) string {
|
||||
base := a.urlFor(m)
|
||||
var b bmcRecord
|
||||
if json.Unmarshal(m.BMC, &b) == nil && b.Extra["firmware_base"] != "" {
|
||||
base = strings.TrimRight(b.Extra["firmware_base"], "/")
|
||||
}
|
||||
return base + "/firmware/" + p.ID + "/" + p.File
|
||||
}
|
||||
|
||||
func (a *App) startFirmware(ctx context.Context, m *Machine, p FWPackage, sp fwSpec, force bool) (*Job, error) {
|
||||
drv, model := bmcVendor(m)
|
||||
if d, _ := driverByName(drv); d.Family != "redfish" {
|
||||
return nil, fmt.Errorf("%s: firmware updates need a Redfish BMC driver (now %s): rollcall bmc %s -probe", m.ID, orDefault(drv, "ipmi"), m.ID)
|
||||
}
|
||||
if !force {
|
||||
if p.Vendor != drv {
|
||||
return nil, fmt.Errorf("%s is a %s package; this BMC is %s (-force to send it anyway)", p.ID, p.Vendor, drv)
|
||||
}
|
||||
if len(fwFor([]FWPackage{p}, drv, model)) == 0 {
|
||||
return nil, fmt.Errorf("%s is for models %v, this is %q (-force to send it anyway)", p.ID, p.Models, model)
|
||||
}
|
||||
}
|
||||
if j, err := a.store.ActiveJob(ctx, m.ID); err == nil {
|
||||
return nil, fmt.Errorf("job %s (%s, %s) is still active", j.ID, j.Kind, j.State)
|
||||
}
|
||||
sp.Package, sp.Component, sp.Version = p.ID, p.Component, p.Version
|
||||
specB, _ := json.Marshal(sp)
|
||||
j := &Job{ID: "j-" + randomToken(5), MachineID: m.ID, Kind: "firmware", State: "pending", Spec: specB}
|
||||
if err := a.store.CreateJob(ctx, j, "", nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
a.setFWProgress(ctx, j, fwProgress{Phase: "uploading", Msg: "handing " + p.File + " to the BMC"})
|
||||
_ = a.store.AddEvent(ctx, m.ID, "firmware requested", fmt.Sprintf("job=%s %s %s %s apply=%s", j.ID, p.Component, p.Version, p.File, orDefault(sp.Apply, "default")))
|
||||
go a.runFirmware(j.ID, m.ID, false)
|
||||
return j, nil
|
||||
}
|
||||
|
||||
// runFirmware drives one update to its end; resume picks up a task already started.
|
||||
func (a *App) runFirmware(jobID, machineID string, resume bool) {
|
||||
ctx := context.Background()
|
||||
j, err := a.store.Job(ctx, jobID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
m, err := a.store.Get(ctx, machineID)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
var sp fwSpec
|
||||
_ = json.Unmarshal(j.Spec, &sp)
|
||||
var pr fwProgress
|
||||
_ = json.Unmarshal(j.Progress, &pr)
|
||||
// every call reads the record again: the BMC certificate pin may change on the way
|
||||
via := func(op, arg string) (string, error) {
|
||||
if mm, err := a.store.Get(ctx, machineID); err == nil {
|
||||
m = mm
|
||||
}
|
||||
out, err := a.firmwareVia(ctx, m, op, arg)
|
||||
if err != nil && sp.Component == "bmc" && strings.Contains(err.Error(), "certificate changed") {
|
||||
// the BMC we are updating came back with a new certificate: expected, pin it anew
|
||||
_ = a.setBMCExtra(ctx, machineID, "tls_fingerprint", "")
|
||||
_ = a.store.AddEvent(ctx, machineID, "bmc certificate", "changed during the BMC update; pinned again")
|
||||
if mm, gerr := a.store.Get(ctx, machineID); gerr == nil {
|
||||
m = mm
|
||||
}
|
||||
out, err = a.firmwareVia(ctx, m, op, arg)
|
||||
}
|
||||
return out, err
|
||||
}
|
||||
finish := func(state, msg string) {
|
||||
_ = a.store.FinishJob(ctx, j.ID, state, map[bool]string{true: "", false: msg}[state == "done"])
|
||||
ev := "firmware " + state
|
||||
_ = a.store.AddEvent(ctx, m.ID, ev, fmt.Sprintf("job=%s %s %s: %s", j.ID, sp.Component, sp.Version, msg))
|
||||
log.Printf("%s: firmware %s %s %s: %s", m.ID, sp.Component, sp.Version, state, msg)
|
||||
}
|
||||
if !resume || pr.Task == "" && pr.Phase == "uploading" {
|
||||
if resume { // the region went away while the upload was running: the BMC may or may not have it
|
||||
finish("failed", "the region restarted during the upload; check the BMC, then try again")
|
||||
return
|
||||
}
|
||||
p, err := fwGet(a.cfg.FirmwareDir, sp.Package)
|
||||
if err != nil {
|
||||
finish("failed", err.Error())
|
||||
return
|
||||
}
|
||||
arg, _ := json.Marshal(fwStartArg{URL: a.firmwareURL(m, p), File: p.File, Component: p.Component, Method: sp.Method, Apply: sp.Apply})
|
||||
out, err := via("fw-start", string(arg))
|
||||
if err != nil {
|
||||
finish("failed", err.Error())
|
||||
return
|
||||
}
|
||||
var res fwStartResult
|
||||
_ = json.Unmarshal([]byte(out), &res)
|
||||
pr = fwProgress{Phase: "flashing", Method: res.Method, Task: res.Task, Msg: "the BMC has the package"}
|
||||
a.setFWProgress(ctx, j, pr)
|
||||
}
|
||||
// watch the task; a BMC flashing itself drops off the network for minutes
|
||||
if pr.Task != "" && pr.Phase == "flashing" {
|
||||
var lastOK time.Time = time.Now()
|
||||
for {
|
||||
out, err := via("fw-poll", pr.Task)
|
||||
if err != nil {
|
||||
if time.Since(lastOK) > 25*time.Minute {
|
||||
finish("failed", "the BMC has not answered for 25 minutes: "+err.Error())
|
||||
return
|
||||
}
|
||||
pr.Msg = "BMC not answering (it may be restarting): " + oneLine(err.Error())
|
||||
a.setFWProgress(ctx, j, pr)
|
||||
time.Sleep(20 * time.Second)
|
||||
continue
|
||||
}
|
||||
lastOK = time.Now()
|
||||
var pl fwPollResult
|
||||
_ = json.Unmarshal([]byte(out), &pl)
|
||||
pr.Pct, pr.Msg = pl.Pct, pl.Msg
|
||||
a.setFWProgress(ctx, j, pr)
|
||||
if pl.State == "failed" {
|
||||
_, _ = via("fw-finish", "")
|
||||
finish("failed", orDefault(pl.Msg, "the BMC reported the update failed"))
|
||||
return
|
||||
}
|
||||
if pl.State == "done" || pl.State == "unknown" {
|
||||
break
|
||||
}
|
||||
if time.Since(*orNow(j.Started)) > 3*time.Hour {
|
||||
finish("failed", "the BMC task did not end within 3 hours")
|
||||
return
|
||||
}
|
||||
time.Sleep(15 * time.Second)
|
||||
}
|
||||
}
|
||||
_, _ = via("fw-finish", "") // e.g. Lenovo's busy flag
|
||||
// a component that applies at the next reset (BIOS, most NICs/RAID) waits for it
|
||||
if sp.Reboot && pr.Phase != "verifying" {
|
||||
pr.Phase, pr.Msg = "rebooting", "power cycle so the update applies"
|
||||
a.setFWProgress(ctx, j, pr)
|
||||
if _, err := a.power(ctx, m, "cycle", ""); err != nil {
|
||||
finish("failed", "update staged, but the power cycle failed: "+err.Error())
|
||||
return
|
||||
}
|
||||
}
|
||||
pr.Phase, pr.Msg = "verifying", "reading the firmware inventory"
|
||||
a.setFWProgress(ctx, j, pr)
|
||||
deadline := time.Now().Add(20 * time.Minute) // servers take minutes to POST, BMCs to come back
|
||||
for {
|
||||
if mm, gerr := a.store.Get(ctx, machineID); gerr == nil {
|
||||
m = mm
|
||||
}
|
||||
inv, err := a.readFirmware(ctx, m)
|
||||
if err != nil && sp.Component == "bmc" && strings.Contains(err.Error(), "certificate changed") {
|
||||
_ = a.setBMCExtra(ctx, machineID, "tls_fingerprint", "")
|
||||
time.Sleep(5 * time.Second)
|
||||
continue
|
||||
}
|
||||
if err == nil {
|
||||
for _, c := range inv {
|
||||
if c.Component == sp.Component && fwSameVersion(c.Version, sp.Version) {
|
||||
finish("done", "installed "+c.Name+" "+c.Version)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
if time.Now().After(deadline) || (!sp.Reboot && err == nil && pr.Task != "") {
|
||||
break
|
||||
}
|
||||
time.Sleep(60 * time.Second)
|
||||
}
|
||||
if sp.Reboot {
|
||||
finish("failed", "the inventory does not show "+sp.Version+" after the update and a reboot")
|
||||
return
|
||||
}
|
||||
// staged: most components switch at the next reset of the host
|
||||
finish("done", "accepted by the BMC; "+sp.Version+" is not in the inventory yet (applies at the next reboot, or rerun with -reboot)")
|
||||
}
|
||||
|
||||
func orNow(t *time.Time) *time.Time {
|
||||
if t == nil {
|
||||
n := time.Now()
|
||||
return &n
|
||||
}
|
||||
return t
|
||||
}
|
||||
|
||||
// resumeFirmware watches the updates that were running when the region stopped.
|
||||
func (a *App) resumeFirmware() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||
defer cancel()
|
||||
active, err := a.store.ActiveJobs(ctx)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, j := range active {
|
||||
if j.Kind == "firmware" {
|
||||
log.Printf("%s: resuming firmware job %s", j.MachineID, j.ID)
|
||||
go a.runFirmware(j.ID, j.MachineID, true)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- HTTP
|
||||
|
||||
func (a *App) handleFirmwareRepo(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.Method {
|
||||
case http.MethodGet:
|
||||
pkgs, err := fwList(a.cfg.FirmwareDir)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, pkgs)
|
||||
case http.MethodPost: // import from a URL: {"url", "vendor", "component", "version", "models", "sha256", "notes"}
|
||||
var req struct {
|
||||
FWPackage
|
||||
URL string `json:"url"`
|
||||
SHA256 string `json:"sha256"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || !(strings.HasPrefix(req.URL, "https://") || strings.HasPrefix(req.URL, "http://")) {
|
||||
fail(w, 400, errors.New("url (http or https), vendor, component and version are required"))
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 30*time.Minute)
|
||||
defer cancel()
|
||||
p, err := fwImport(ctx, a.cfg.FirmwareDir, req.URL, req.FWPackage, req.SHA256)
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
log.Printf("firmware %s imported by %s from %s", p.ID, a.who(r), req.URL)
|
||||
writeJSON(w, 200, p)
|
||||
}
|
||||
}
|
||||
|
||||
func (a *App) handleFirmwareRemove(w http.ResponseWriter, r *http.Request) {
|
||||
if err := fwRemove(a.cfg.FirmwareDir, r.PathValue("pkg")); err != nil {
|
||||
fail(w, 404, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, obj{"removed": r.PathValue("pkg")})
|
||||
}
|
||||
|
||||
// handleFirmwareFile serves one package file (no listings): BMCs download from here.
|
||||
func (a *App) handleFirmwareFile(w http.ResponseWriter, r *http.Request) {
|
||||
p, err := fwGet(a.cfg.FirmwareDir, r.PathValue("pkg"))
|
||||
if err != nil || r.PathValue("file") != p.File {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
http.ServeFile(w, r, filepath.Join(a.cfg.FirmwareDir, p.ID, p.File))
|
||||
}
|
||||
|
||||
// handleMachineFirmware: GET the stored inventory and what the repository offers; POST ?refresh reads it again.
|
||||
func (a *App) handleMachineFirmware(w http.ResponseWriter, r *http.Request) {
|
||||
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, 404, err)
|
||||
return
|
||||
}
|
||||
if r.Method == http.MethodPost {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), 6*time.Minute)
|
||||
defer cancel()
|
||||
if _, err := a.readFirmware(ctx, m); err != nil {
|
||||
fail(w, 502, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
inv, err := a.store.FirmwareInventory(r.Context(), m.ID)
|
||||
if err != nil {
|
||||
inv = &fwInventory{Components: []fwComponent{}}
|
||||
}
|
||||
drv, model := bmcVendor(m)
|
||||
pkgs, _ := fwList(a.cfg.FirmwareDir)
|
||||
offers := fwOffers(fwFor(pkgs, drv, model), inv.Components)
|
||||
if offers == nil {
|
||||
offers = []fwOffer{}
|
||||
}
|
||||
writeJSON(w, 200, obj{"id": m.ID, "driver": drv, "model": model, "inventory": inv, "offers": offers})
|
||||
}
|
||||
|
||||
// handleFirmwareUpdate starts one update: {"package", "apply", "reboot", "method", "force"}.
|
||||
func (a *App) handleFirmwareUpdate(w http.ResponseWriter, r *http.Request) {
|
||||
m, err := a.store.Get(r.Context(), r.PathValue("id"))
|
||||
if err != nil {
|
||||
fail(w, 404, err)
|
||||
return
|
||||
}
|
||||
var req struct {
|
||||
fwSpec
|
||||
Force bool `json:"force"`
|
||||
}
|
||||
if err := json.NewDecoder(r.Body).Decode(&req); err != nil || req.Package == "" {
|
||||
fail(w, 400, errors.New("package is required"))
|
||||
return
|
||||
}
|
||||
switch req.Apply {
|
||||
case "", "Immediate", "OnReset":
|
||||
default:
|
||||
fail(w, 400, errors.New("apply is Immediate or OnReset"))
|
||||
return
|
||||
}
|
||||
switch req.Method {
|
||||
case "", "multipart", "simple", "push":
|
||||
default:
|
||||
fail(w, 400, errors.New("method is multipart, simple or push"))
|
||||
return
|
||||
}
|
||||
p, err := fwGet(a.cfg.FirmwareDir, req.Package)
|
||||
if err != nil {
|
||||
fail(w, 404, err)
|
||||
return
|
||||
}
|
||||
j, err := a.startFirmware(r.Context(), m, p, req.fwSpec, req.Force)
|
||||
if err != nil {
|
||||
fail(w, 409, err)
|
||||
return
|
||||
}
|
||||
log.Printf("%s: firmware %s requested by %s (job %s)", m.ID, p.ID, a.who(r), j.ID)
|
||||
writeJSON(w, 200, obj{"id": m.ID, "job": j.ID, "package": p.ID, "component": p.Component, "version": p.Version})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- CLI
|
||||
|
||||
func firmwareCmd(args []string) error {
|
||||
if len(args) == 0 {
|
||||
args = []string{"list"}
|
||||
}
|
||||
switch args[0] {
|
||||
case "import":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
fs := newFlags("firmware import")
|
||||
vendor := fs.String("vendor", "", "Redfish driver the package is for: idrac, ilo, xcc, supermicro, xfusion, kaytus, irmc, cimc, gigabyte, asrockrack, openbmc, ami")
|
||||
comp := fs.String("component", "", strings.Join(fwComponents, " | "))
|
||||
ver := fs.String("version", "", "the version the component reports once installed")
|
||||
models := fs.String("models", "", "comma-separated model substrings it is for (default: any)")
|
||||
sum := fs.String("sha256", "", "expected SHA-256 (from the vendor's download page)")
|
||||
notes := fs.String("notes", "", "notes")
|
||||
_ = fs.Parse(args[2:])
|
||||
p := FWPackage{Vendor: *vendor, Component: *comp, Version: *ver, Notes: *notes}
|
||||
if *models != "" {
|
||||
p.Models = strings.Split(*models, ",")
|
||||
}
|
||||
src := args[1]
|
||||
if !strings.HasPrefix(src, "http://") && !strings.HasPrefix(src, "https://") { // a local file: on the region host
|
||||
cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("importing a file needs the region's config (run on the region, or give an https URL): %w", err)
|
||||
}
|
||||
got, err := fwImport(context.Background(), cfg.FirmwareDir, src, p, *sum)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("imported %s (%s, %s)\n", got.ID, human(float64(got.Size)), got.SHA256)
|
||||
return nil
|
||||
}
|
||||
c, err := newClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var got FWPackage
|
||||
req := obj{"url": src, "vendor": p.Vendor, "component": p.Component, "version": p.Version, "models": p.Models, "sha256": *sum, "notes": p.Notes}
|
||||
if err := c.do("POST", "/api/v1/firmware", req, &got); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("imported %s (%s, %s)\n", got.ID, human(float64(got.Size)), got.SHA256)
|
||||
return nil
|
||||
}
|
||||
c, err := newClient()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
switch args[0] {
|
||||
case "list":
|
||||
var pkgs []FWPackage
|
||||
if err := c.do("GET", "/api/v1/firmware", nil, &pkgs); err != nil {
|
||||
return err
|
||||
}
|
||||
f := "%-44s %-11s %-9s %-22s %9s %s\n"
|
||||
fmt.Printf(f, "PACKAGE", "VENDOR", "COMPONENT", "VERSION", "SIZE", "FILE / MODELS")
|
||||
for _, p := range pkgs {
|
||||
ms := ""
|
||||
if len(p.Models) > 0 {
|
||||
ms = " [" + strings.Join(p.Models, ", ") + "]"
|
||||
}
|
||||
fmt.Printf(f, p.ID, p.Vendor, p.Component, p.Version, human(float64(p.Size)), p.File+ms)
|
||||
}
|
||||
case "rm":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
return c.do("DELETE", "/api/v1/firmware/"+args[1], nil, nil)
|
||||
case "show":
|
||||
if len(args) < 2 {
|
||||
usage()
|
||||
}
|
||||
fs := newFlags("firmware show")
|
||||
refresh := fs.Bool("refresh", false, "read the inventory from the BMC now")
|
||||
_ = fs.Parse(args[2:])
|
||||
method := "GET"
|
||||
if *refresh {
|
||||
method = "POST"
|
||||
}
|
||||
var d struct {
|
||||
Driver string `json:"driver"`
|
||||
Model string `json:"model"`
|
||||
Inventory fwInventory `json:"inventory"`
|
||||
Offers []fwOffer `json:"offers"`
|
||||
}
|
||||
if err := c.do(method, "/api/v1/machines/"+args[1]+"/firmware", nil, &d); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s driver %s model %s inventory %s\n", args[1], orDash(d.Driver), orDash(d.Model), orDash(fwWhen(d.Inventory.At)))
|
||||
if len(d.Inventory.Components) == 0 {
|
||||
fmt.Println("no inventory yet: rollcall firmware show " + args[1] + " -refresh")
|
||||
}
|
||||
for _, x := range d.Inventory.Components {
|
||||
up := ""
|
||||
if !x.Updateable {
|
||||
up = " (not updateable)"
|
||||
}
|
||||
fmt.Printf(" %-9s %-40s %s%s\n", x.Component, trunc(x.Name, 40), x.Version, up)
|
||||
}
|
||||
if len(d.Offers) > 0 {
|
||||
fmt.Println("repository:")
|
||||
for _, o := range d.Offers {
|
||||
st := "update"
|
||||
if o.Same {
|
||||
st = "installed"
|
||||
}
|
||||
fmt.Printf(" %-9s %-22s %-9s (now %s) %s\n", o.Package.Component, o.Package.Version, st, orDash(o.Installed), o.Package.ID)
|
||||
}
|
||||
}
|
||||
case "update":
|
||||
if len(args) < 3 {
|
||||
usage()
|
||||
}
|
||||
fs := newFlags("firmware update")
|
||||
apply := fs.String("apply", "", "Immediate | OnReset (default: the BMC's)")
|
||||
reboot := fs.Bool("reboot", false, "power-cycle the host afterwards so a staged update applies, then verify")
|
||||
method := fs.String("method", "", "multipart | simple | push (default: the vendor profile's order)")
|
||||
force := fs.Bool("force", false, "send a package for another vendor or model")
|
||||
watch := fs.Bool("watch", false, "follow the job")
|
||||
_ = fs.Parse(args[3:])
|
||||
ids, err := c.targets(args[1])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req := obj{"package": args[2], "apply": *apply, "reboot": *reboot, "method": *method, "force": *force}
|
||||
for _, id := range ids {
|
||||
var d obj
|
||||
if err := c.do("POST", "/api/v1/machines/"+id+"/firmware/update", req, &d); err != nil {
|
||||
if len(ids) == 1 {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%s: error: %v\n", id, err)
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s: job %s %s %s\n", id, str(d["job"]), str(d["component"]), str(d["version"]))
|
||||
}
|
||||
if *watch && len(ids) == 1 {
|
||||
return c.watchJob(ids[0])
|
||||
}
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func fwWhen(t time.Time) string {
|
||||
if t.IsZero() {
|
||||
return ""
|
||||
}
|
||||
return t.Local().Format("2006-01-02 15:04")
|
||||
}
|
||||
@@ -0,0 +1,289 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fwBMC is an UpdateService: it takes packages the ways the flags allow and
|
||||
// keeps a task per upload.
|
||||
type fwBMC struct {
|
||||
mu sync.Mutex
|
||||
multipart, simple, push bool
|
||||
protocols []string
|
||||
got []string // method:file-content or method:ImageURI
|
||||
params []obj
|
||||
chunked bool
|
||||
polls int
|
||||
}
|
||||
|
||||
func (f *fwBMC) handler(t *testing.T) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if u, p, ok := r.BasicAuth(); !ok || u != "admin" || p != "secret" {
|
||||
w.WriteHeader(401)
|
||||
return
|
||||
}
|
||||
task := func() {
|
||||
w.Header().Set("Location", "/redfish/v1/TaskService/Tasks/3")
|
||||
w.WriteHeader(202)
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/redfish/v1/UpdateService":
|
||||
us := obj{}
|
||||
if f.multipart {
|
||||
us["MultipartHttpPushUri"] = "/redfish/v1/UpdateService/upload"
|
||||
}
|
||||
if f.push {
|
||||
us["HttpPushUri"] = "/redfish/v1/UpdateService/push"
|
||||
}
|
||||
if f.simple {
|
||||
act := obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}
|
||||
if f.protocols != nil {
|
||||
act["TransferProtocol@Redfish.AllowableValues"] = f.protocols
|
||||
}
|
||||
us["Actions"] = obj{"#UpdateService.SimpleUpdate": act}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(us)
|
||||
case r.URL.Path == "/redfish/v1/UpdateService/upload":
|
||||
f.chunked = f.chunked || r.ContentLength < 0 || len(r.TransferEncoding) > 0
|
||||
_, ps, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
|
||||
if err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
mr := multipart.NewReader(r.Body, ps["boundary"])
|
||||
for {
|
||||
p, err := mr.NextPart()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
b, _ := io.ReadAll(p)
|
||||
switch p.FormName() {
|
||||
case "UpdateParameters":
|
||||
var o obj
|
||||
_ = json.Unmarshal(b, &o)
|
||||
f.params = append(f.params, o)
|
||||
case "UpdateFile":
|
||||
f.got = append(f.got, "multipart:"+p.FileName()+":"+string(b))
|
||||
}
|
||||
}
|
||||
task()
|
||||
case r.URL.Path == "/redfish/v1/UpdateService/push":
|
||||
f.chunked = f.chunked || r.ContentLength < 0
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
f.got = append(f.got, "push:"+string(b))
|
||||
task()
|
||||
case r.URL.Path == "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate":
|
||||
var o obj
|
||||
_ = json.NewDecoder(r.Body).Decode(&o)
|
||||
f.got = append(f.got, "simple:"+str(o["ImageURI"])+":"+str(o["TransferProtocol"]))
|
||||
task()
|
||||
case r.URL.Path == "/redfish/v1/TaskService/Tasks/3":
|
||||
f.polls++
|
||||
st := obj{"TaskState": "Running", "PercentComplete": 40}
|
||||
if f.polls > 1 {
|
||||
st = obj{"TaskState": "Completed", "PercentComplete": 100, "Messages": []obj{{"Message": "Firmware update completed"}}}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(st)
|
||||
case r.URL.Path == "/redfish/v1/UpdateService/FirmwareInventory":
|
||||
_ = json.NewEncoder(w).Encode(obj{"Members": []obj{
|
||||
{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Installed-159-2.19.1"},
|
||||
{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Previous-159-2.18.0"},
|
||||
{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/Installed-25227-7.10.30.00"},
|
||||
}})
|
||||
case strings.HasPrefix(r.URL.Path, "/redfish/v1/UpdateService/FirmwareInventory/Installed-159"):
|
||||
_ = json.NewEncoder(w).Encode(obj{"Id": "Installed-159-2.19.1", "Name": "BIOS", "Version": "2.19.1", "Updateable": true})
|
||||
case strings.HasPrefix(r.URL.Path, "/redfish/v1/UpdateService/FirmwareInventory/Installed-25227"):
|
||||
_ = json.NewEncoder(w).Encode(obj{"Id": "Installed-25227-7.10.30.00", "Name": "Integrated Dell Remote Access Controller", "Version": "7.10.30.00", "Updateable": true})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func runFW(t *testing.T, f *fwBMC, op string, arg any) string {
|
||||
t.Helper()
|
||||
srv := httptest.NewTLSServer(f.handler(t))
|
||||
defer srv.Close()
|
||||
pkg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("FIRMWARE-BYTES")) }))
|
||||
defer pkg.Close()
|
||||
a := ""
|
||||
if s, ok := arg.(string); ok {
|
||||
a = s
|
||||
} else if arg != nil {
|
||||
sa := arg.(fwStartArg)
|
||||
if sa.URL == "" {
|
||||
sa.URL = pkg.URL + "/firmware/p/BIOS_X.EXE"
|
||||
}
|
||||
b, _ := json.Marshal(sa)
|
||||
a = string(b)
|
||||
}
|
||||
out, err := powerRun(context.Background(), "", creds(srv.URL, "redfish"), op, a)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", op, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestFirmwareMultipart(t *testing.T) {
|
||||
f := &fwBMC{multipart: true, simple: true}
|
||||
var res fwStartResult
|
||||
_ = json.Unmarshal([]byte(runFW(t, f, "fw-start", fwStartArg{File: "BIOS_X.EXE", Apply: "OnReset"})), &res)
|
||||
if res.Method != "multipart" || res.Task != "/redfish/v1/TaskService/Tasks/3" {
|
||||
t.Errorf("result %+v", res)
|
||||
}
|
||||
if len(f.got) != 1 || f.got[0] != "multipart:BIOS_X.EXE:FIRMWARE-BYTES" || f.chunked {
|
||||
t.Errorf("got %v chunked %v", f.got, f.chunked)
|
||||
}
|
||||
if f.params[0]["@Redfish.OperationApplyTime"] != "OnReset" {
|
||||
t.Errorf("params %v", f.params)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirmwareSimpleAndPush(t *testing.T) {
|
||||
f := &fwBMC{simple: true, protocols: []string{"HTTP", "HTTPS", "NFS"}}
|
||||
_ = runFW(t, f, "fw-start", fwStartArg{File: "x.bin"})
|
||||
if len(f.got) != 1 || !strings.HasPrefix(f.got[0], "simple:http://") || !strings.HasSuffix(f.got[0], ":HTTP") {
|
||||
t.Errorf("simple %v", f.got)
|
||||
}
|
||||
g := &fwBMC{push: true}
|
||||
_ = runFW(t, g, "fw-start", fwStartArg{File: "x.bin"})
|
||||
if len(g.got) != 1 || g.got[0] != "push:FIRMWARE-BYTES" || g.chunked {
|
||||
t.Errorf("push %v chunked %v", g.got, g.chunked)
|
||||
}
|
||||
// a service that pulls only over NFS cannot take our HTTP URL: refused clearly
|
||||
h := &fwBMC{simple: true, protocols: []string{"NFS"}}
|
||||
srv := httptest.NewTLSServer(h.handler(t))
|
||||
defer srv.Close()
|
||||
b, _ := json.Marshal(fwStartArg{URL: "http://rack/firmware/p/x.bin", File: "x.bin"})
|
||||
if _, err := powerRun(context.Background(), "", creds(srv.URL, "redfish"), "fw-start", string(b)); err == nil || !strings.Contains(err.Error(), "not HTTP") {
|
||||
t.Errorf("nfs-only: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirmwarePollAndInventory(t *testing.T) {
|
||||
f := &fwBMC{}
|
||||
var p fwPollResult
|
||||
_ = json.Unmarshal([]byte(runFW(t, f, "fw-poll", "/redfish/v1/TaskService/Tasks/3")), &p)
|
||||
if p.State != "running" || p.Pct != 40 {
|
||||
t.Errorf("first poll %+v", p)
|
||||
}
|
||||
var inv []fwComponent
|
||||
_ = json.Unmarshal([]byte(runFW(t, f, "fw-inventory", nil)), &inv)
|
||||
if len(inv) != 2 || inv[0].Component != "bios" || inv[1].Component != "bmc" || inv[0].Version != "2.19.1" {
|
||||
t.Errorf("inventory %+v", inv)
|
||||
}
|
||||
pkgs := []FWPackage{{ID: "a", Vendor: "idrac", Component: "bios", Version: "2.19.1"}, {ID: "b", Vendor: "idrac", Component: "bmc", Version: "7.20.10.50"}}
|
||||
offers := fwOffers(pkgs, inv)
|
||||
if !offers[0].Same || offers[1].Same || offers[1].Installed != "7.10.30.00" {
|
||||
t.Errorf("offers %+v", offers)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirmwareNeedsRedfish(t *testing.T) {
|
||||
if _, err := powerRun(context.Background(), "", &bmcCreds{Driver: "ipmi"}, "fw-inventory", ""); err == nil || !strings.Contains(err.Error(), "Redfish") {
|
||||
t.Errorf("ipmi: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirmwareRepository(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
src := filepath.Join(t.TempDir(), "BIOS_ABC_WN64_2.19.1.EXE")
|
||||
_ = os.WriteFile(src, []byte("dup"), 0o644)
|
||||
p, err := fwImport(context.Background(), dir, src, FWPackage{Vendor: "idrac", Component: "bios", Version: "2.19.1", Models: []string{"R650"}}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !fwIDRe.MatchString(p.ID) || p.Size != 3 || p.File != "BIOS_ABC_WN64_2.19.1.EXE" {
|
||||
t.Errorf("package %+v", p)
|
||||
}
|
||||
if _, err := fwImport(context.Background(), dir, src, FWPackage{Vendor: "idrac", Component: "bios", Version: "x"}, "00"); err == nil {
|
||||
t.Error("a wrong sha256 was accepted")
|
||||
}
|
||||
for _, bad := range []FWPackage{{Vendor: "ipmi", Component: "bios", Version: "1"}, {Vendor: "idrac", Component: "toaster", Version: "1"}, {Vendor: "idrac", Component: "bios"}} {
|
||||
if _, err := fwImport(context.Background(), dir, src, bad, ""); err == nil {
|
||||
t.Errorf("accepted %+v", bad)
|
||||
}
|
||||
}
|
||||
list, _ := fwList(dir)
|
||||
if len(list) != 1 || len(fwFor(list, "idrac", "PowerEdge R650")) != 1 || len(fwFor(list, "idrac", "PowerEdge R760")) != 0 || len(fwFor(list, "ilo", "R650")) != 0 {
|
||||
t.Errorf("list %+v", list)
|
||||
}
|
||||
if _, err := fwGet(dir, "../etc"); err == nil {
|
||||
t.Error("a path as package id")
|
||||
}
|
||||
if err := fwRemove(dir, p.ID); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, c := range [][2]string{{"2.19.1", "2.19.1"}, {"v2.19.1", "2.19.1"}, {"2.19.1 (Build 4)", "2.19.1"}} {
|
||||
if !fwSameVersion(c[0], c[1]) {
|
||||
t.Errorf("%q vs %q", c[0], c[1])
|
||||
}
|
||||
}
|
||||
if fwSameVersion("2.19.10", "2.19.1") {
|
||||
t.Error("2.19.10 is not 2.19.1")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirmwareKinds(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"Installed-159-2.19.1 BIOS": "bios",
|
||||
"Integrated Dell Remote Access Controller": "bmc",
|
||||
"PERC H755 Front": "raid",
|
||||
"RAID Controller": "raid",
|
||||
"iLO 6": "bmc",
|
||||
"Broadcom Adv. Dual 25Gb Ethernet": "nic",
|
||||
"Mellanox ConnectX-6": "nic",
|
||||
"PSU.Slot.1 Power Supply": "psu",
|
||||
"System CPLD": "cpld",
|
||||
"Dell Ent NVMe P5600": "drive",
|
||||
"Lifecycle Controller": "other",
|
||||
"OS Collector": "other",
|
||||
} {
|
||||
if got := fwKind("", in); got != want {
|
||||
t.Errorf("%q: %s, want %s", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFirmwareFwpkg(t *testing.T) {
|
||||
mk := func(files ...string) string {
|
||||
p := filepath.Join(t.TempDir(), "cp056789.fwpkg")
|
||||
f, _ := os.Create(p)
|
||||
zw := zip.NewWriter(f)
|
||||
for _, n := range files {
|
||||
w, _ := zw.Create(n)
|
||||
_, _ = w.Write([]byte("payload of " + n))
|
||||
}
|
||||
_ = zw.Close()
|
||||
_ = f.Close()
|
||||
return p
|
||||
}
|
||||
dir := t.TempDir()
|
||||
got, err := fwImport(context.Background(), dir, mk("payload.json", "ilo6_166.bin"), FWPackage{Vendor: "ilo", Component: "bmc", Version: "1.66"}, "")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got.File != "ilo6_166.bin" || got.Size != int64(len("payload of ilo6_166.bin")) || !strings.Contains(got.Notes, "cp056789.fwpkg") {
|
||||
t.Errorf("extracted %+v", got)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, got.ID, "ilo6_166.bin")); err != nil {
|
||||
t.Error(err)
|
||||
}
|
||||
if _, err := fwImport(context.Background(), dir, mk("payload.json", "setup.exe"), FWPackage{Vendor: "ilo", Component: "nic", Version: "1"}, ""); err == nil || !strings.Contains(err.Error(), "0 files") {
|
||||
t.Errorf("agent-only component: %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,306 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"mime"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// recBMC records what a firmware update sends: every request, the multipart
|
||||
// parts (JSON parsed), plain form fields and the file's field name.
|
||||
type recBMC struct {
|
||||
mu sync.Mutex
|
||||
vendor string
|
||||
us obj // the UpdateService resource
|
||||
inventory []string // FirmwareInventory ids
|
||||
missing []string // paths that answer 404
|
||||
reqs []recReq
|
||||
}
|
||||
|
||||
type recReq struct {
|
||||
method, path, ifMatch string
|
||||
json obj // JSON body, or the UpdateParameters part
|
||||
parts map[string]obj // JSON parts by name
|
||||
fields map[string]string
|
||||
fileField, fileName string
|
||||
fileBody string
|
||||
}
|
||||
|
||||
func (f *recBMC) find(method, pathPrefix string) *recReq {
|
||||
for i := range f.reqs {
|
||||
if f.reqs[i].method == method && strings.HasPrefix(f.reqs[i].path, pathPrefix) {
|
||||
return &f.reqs[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *recBMC) handler() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
for _, m := range f.missing {
|
||||
if r.URL.Path == m {
|
||||
http.NotFound(w, r)
|
||||
return
|
||||
}
|
||||
}
|
||||
if r.URL.Path == "/redfish/v1/SessionService/Sessions" && r.Method == "POST" {
|
||||
w.Header().Set("X-Auth-Token", "tok")
|
||||
w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/1")
|
||||
w.WriteHeader(201)
|
||||
return
|
||||
}
|
||||
if r.Method == "DELETE" {
|
||||
w.WriteHeader(204)
|
||||
return
|
||||
}
|
||||
rq := recReq{method: r.Method, path: r.URL.Path, ifMatch: r.Header.Get("If-Match"), parts: map[string]obj{}, fields: map[string]string{}}
|
||||
if mt, ps, err := mime.ParseMediaType(r.Header.Get("Content-Type")); err == nil && mt == "multipart/form-data" {
|
||||
mr := multipart.NewReader(r.Body, ps["boundary"])
|
||||
for {
|
||||
p, err := mr.NextPart()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
b, _ := io.ReadAll(p)
|
||||
switch {
|
||||
case p.FileName() != "":
|
||||
rq.fileField, rq.fileName, rq.fileBody = p.FormName(), p.FileName(), string(b)
|
||||
case strings.HasPrefix(p.Header.Get("Content-Type"), "application/json"):
|
||||
var o obj
|
||||
_ = json.Unmarshal(b, &o)
|
||||
rq.parts[p.FormName()] = o
|
||||
default:
|
||||
rq.fields[p.FormName()] = string(b)
|
||||
}
|
||||
}
|
||||
rq.json = rq.parts["UpdateParameters"]
|
||||
} else if r.Body != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(&rq.json)
|
||||
}
|
||||
if r.Method != "GET" {
|
||||
f.reqs = append(f.reqs, rq)
|
||||
}
|
||||
switch {
|
||||
case r.URL.Path == "/redfish/v1/":
|
||||
_ = json.NewEncoder(w).Encode(obj{"Vendor": f.vendor})
|
||||
case r.URL.Path == "/redfish/v1/UpdateService" && r.Method == "GET":
|
||||
w.Header().Set("ETag", `"us1"`)
|
||||
_ = json.NewEncoder(w).Encode(f.us)
|
||||
case r.URL.Path == "/redfish/v1/UpdateService" && r.Method == "PATCH":
|
||||
for k, v := range rq.json {
|
||||
f.us[k] = v
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
case r.URL.Path == "/redfish/v1/UpdateService/FirmwareInventory" && r.Method == "GET":
|
||||
var ms []obj
|
||||
for _, id := range f.inventory {
|
||||
ms = append(ms, obj{"@odata.id": "/redfish/v1/UpdateService/FirmwareInventory/" + id})
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(obj{"Members": ms})
|
||||
case strings.HasPrefix(r.URL.Path, "/redfish/v1/TaskService/Tasks/"):
|
||||
_ = json.NewEncoder(w).Encode(obj{"TaskState": "Completed", "PercentComplete": 100})
|
||||
case r.Method == "POST":
|
||||
w.Header().Set("Location", "/redfish/v1/TaskService/Tasks/9")
|
||||
w.WriteHeader(202)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
// fwStartWith runs fw-start for a vendor driver and returns the result and the recorder.
|
||||
func fwStartWith(t *testing.T, f *recBMC, driver string, a fwStartArg, extra map[string]string) (*fwStartResult, error) {
|
||||
t.Helper()
|
||||
srv := httptest.NewTLSServer(f.handler())
|
||||
defer srv.Close()
|
||||
pkg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("IMG")) }))
|
||||
defer pkg.Close()
|
||||
if a.URL == "" {
|
||||
a.URL = pkg.URL + "/firmware/p/" + a.File
|
||||
}
|
||||
c := creds(srv.URL, driver)
|
||||
for k, v := range extra {
|
||||
c.Extra[k] = v
|
||||
}
|
||||
b, _ := json.Marshal(a)
|
||||
out, err := powerRun(context.Background(), "", c, "fw-start", string(b))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var res fwStartResult
|
||||
_ = json.Unmarshal([]byte(out), &res)
|
||||
return &res, nil
|
||||
}
|
||||
|
||||
func mustStart(t *testing.T, f *recBMC, driver string, a fwStartArg) *fwStartResult {
|
||||
t.Helper()
|
||||
res, err := fwStartWith(t, f, driver, a, nil)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", driver, err)
|
||||
}
|
||||
return res
|
||||
}
|
||||
|
||||
func TestFWPlanDell(t *testing.T) {
|
||||
f := &recBMC{vendor: "Dell", us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/MultipartUpload"}}
|
||||
mustStart(t, f, "idrac", fwStartArg{File: "BIOS_X_WN64_2.19.1.EXE", Component: "bios"})
|
||||
rq := f.find("POST", "/redfish/v1/UpdateService/MultipartUpload")
|
||||
if rq == nil || rq.json["@Redfish.OperationApplyTime"] != "OnReset" || rq.fileField != "UpdateFile" || rq.fileBody != "IMG" {
|
||||
t.Errorf("dell %+v", rq)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFWPlanSupermicro(t *testing.T) {
|
||||
f := &recBMC{vendor: "Supermicro", us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/upload"}, inventory: []string{"BMC", "BIOS", "Motherboard_CPLD_1"}}
|
||||
mustStart(t, f, "supermicro", fwStartArg{File: "BIOS.bin", Component: "bios"})
|
||||
rq := f.find("POST", "/redfish/v1/UpdateService/upload")
|
||||
if rq == nil || dig(rq.json, "Oem", "Supermicro", "BIOS", "PreserveNVRAM") != true || toList(rq.json["Targets"])[0] != "/redfish/v1/Systems/1/Bios" {
|
||||
t.Errorf("bios params %v", rq.json)
|
||||
}
|
||||
f.reqs = nil
|
||||
mustStart(t, f, "supermicro", fwStartArg{File: "CPLD.bin", Component: "cpld"})
|
||||
if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); toList(rq.json["Targets"])[0] != "/redfish/v1/UpdateService/FirmwareInventory/Motherboard_CPLD_1" {
|
||||
t.Errorf("cpld params %v", rq.json)
|
||||
}
|
||||
// the operator's own parameters (Gen 13 preserve flags)
|
||||
f.reqs = nil
|
||||
if _, err := fwStartWith(t, f, "supermicro", fwStartArg{File: "BIOS.bin", Component: "bios"}, map[string]string{"fw_params": `{"Targets":["/redfish/v1/Systems/1/Bios"],"Oem":{"Supermicro":{"BIOS":{"PreserveOA":true}}}}`}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); dig(rq.json, "Oem", "Supermicro", "BIOS", "PreserveOA") != true {
|
||||
t.Errorf("fw_params %v", rq.json)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFWPlanAMI(t *testing.T) {
|
||||
f := &recBMC{us: obj{"MultipartHttpPushUri": "/redfish/v1/UpdateService/upload"}, inventory: []string{"BMC", "BMCImage1", "BIOS", "BIOS2", "MB_CPLD1"}}
|
||||
mustStart(t, f, "gigabyte", fwStartArg{File: "rom.ima_enc", Component: "bmc"})
|
||||
rq := f.find("POST", "/redfish/v1/UpdateService/upload")
|
||||
if rq.parts["OemParameters"]["ImageType"] != "BMC" || toList(rq.json["Targets"])[0] != "/redfish/v1/UpdateService/FirmwareInventory/BMCImage1" {
|
||||
t.Errorf("gigabyte bmc %+v", rq)
|
||||
}
|
||||
f.reqs = nil
|
||||
mustStart(t, f, "gigabyte", fwStartArg{File: "bios.hpm", Component: "bios"})
|
||||
if rq := f.find("POST", "/redfish/v1/UpdateService/upload"); rq.parts["OemParameters"]["ImageType"] != "HPM_BIOS" {
|
||||
t.Errorf("gigabyte hpm bios %+v", rq.parts)
|
||||
}
|
||||
if _, err := fwStartWith(t, &recBMC{us: obj{}, inventory: []string{"BMC"}}, "ami", fwStartArg{File: "x.cap", Component: "bios"}, nil); err == nil || !strings.Contains(err.Error(), "no firmware BIOS") {
|
||||
t.Errorf("missing target: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFWPlanXFusion(t *testing.T) {
|
||||
f := &recBMC{us: obj{"Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}}
|
||||
mustStart(t, f, "xfusion", fwStartArg{File: "image.hpm", Component: "bmc"})
|
||||
up, su := f.find("POST", "/redfish/v1/UpdateService/FirmwareInventory"), f.find("POST", "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate")
|
||||
if up == nil || up.fileField != "file" || len(up.parts) != 0 || su == nil || su.json["ImageURI"] != "/tmp/web/image.hpm" {
|
||||
t.Errorf("upload %+v simple %+v", up, su)
|
||||
}
|
||||
if _, err := fwStartWith(t, f, "xfusion", fwStartArg{File: "nic.hpm", Component: "nic"}, nil); err == nil {
|
||||
t.Error("in-band NIC firmware accepted")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFWPlanKaytus(t *testing.T) {
|
||||
f := &recBMC{us: obj{"ServiceEnabled": true, "Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}}
|
||||
res := mustStart(t, f, "kaytus", fwStartArg{File: "bios.bin", Component: "bios"})
|
||||
up, su := f.find("POST", "/redfish/v1/UpdateService/FirmwareInventory"), f.find("POST", "/redfish/v1/UpdateService/Actions")
|
||||
if up.fileField != "fwimage" || up.fields["CurrentPassword"] != "secret" || up.fields["EncryptFlag"] != "False" {
|
||||
t.Errorf("upload %+v", up)
|
||||
}
|
||||
if dig(su.json, "Oem", "Public", "FlashItem") != "BIOS" || dig(su.json, "Oem", "Public", "BiosFlash") != "Both" {
|
||||
t.Errorf("flash %v", su.json)
|
||||
}
|
||||
if res.Task == "" {
|
||||
t.Errorf("no task")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFWPlanIRMC(t *testing.T) {
|
||||
f := &recBMC{vendor: "ts_fujitsu", us: obj{}, missing: []string{"/redfish/v1/Managers/iRMC/Actions/FTSManager.FWUpdate"}}
|
||||
mustStart(t, f, "irmc", fwStartArg{File: "D3279_09.09F.bin", Component: "bmc"})
|
||||
rq := f.find("POST", "/redfish/v1/Managers/iRMC/Actions/Oem/FTSManager.FWUpdate")
|
||||
if rq == nil || rq.fileField != "data" || rq.json != nil {
|
||||
t.Errorf("irmc %+v (all %+v)", rq, f.reqs)
|
||||
}
|
||||
g := &recBMC{vendor: "Fsas", us: obj{}}
|
||||
mustStart(t, g, "irmc", fwStartArg{File: "D3279-B1x.R1.20.0.UPC", Component: "bios"})
|
||||
if g.find("POST", "/redfish/v1/Systems/0/Bios/Actions/Oem/FsasBios.BiosUpdate") == nil {
|
||||
t.Errorf("fsas bios %+v", g.reqs)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFWPlanLenovoClaims(t *testing.T) {
|
||||
f := &recBMC{vendor: "Lenovo", us: obj{"MultipartHttpPushUri": "/mfwupdate", "HttpPushUriTargetsBusy": false}}
|
||||
mustStart(t, f, "xcc", fwStartArg{File: "lnvgy_fw_xcc.uxz", Component: "bmc"})
|
||||
if f.us["HttpPushUriTargetsBusy"] != true || f.find("POST", "/mfwupdate") == nil || f.find("PATCH", "/redfish/v1/UpdateService").ifMatch != `"us1"` {
|
||||
t.Errorf("claim %v %+v", f.us, f.reqs)
|
||||
}
|
||||
if _, err := fwStartWith(t, f, "xcc", fwStartArg{File: "x.uxz", Component: "bmc"}, nil); err == nil || !strings.Contains(err.Error(), "busy") {
|
||||
t.Errorf("second upload while busy: %v", err)
|
||||
}
|
||||
srv := httptest.NewTLSServer(f.handler())
|
||||
defer srv.Close()
|
||||
if _, err := powerRun(context.Background(), "", creds(srv.URL, "xcc"), "fw-finish", ""); err != nil || f.us["HttpPushUriTargetsBusy"] != false {
|
||||
t.Errorf("release: %v %v", err, f.us)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFWPlanILOAndCisco(t *testing.T) {
|
||||
f := &recBMC{vendor: "HPE", us: obj{"Actions": obj{"#UpdateService.SimpleUpdate": obj{"target": "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"}}}}
|
||||
mustStart(t, f, "ilo", fwStartArg{File: "ilo6_166.bin", Component: "bmc"})
|
||||
if rq := f.find("POST", "/redfish/v1/UpdateService/Actions"); rq == nil || !strings.HasSuffix(str(rq.json["ImageURI"]), "/ilo6_166.bin") {
|
||||
t.Errorf("ilo %+v", f.reqs)
|
||||
}
|
||||
if _, err := fwStartWith(t, &recBMC{us: obj{}}, "cimc", fwStartArg{File: "x.bin", Component: "bios"}, nil); err == nil || !strings.Contains(err.Error(), "SCP") {
|
||||
t.Errorf("cimc: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFWPollStaged(t *testing.T) {
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/redfish/v1/TaskService/Tasks/JID_1":
|
||||
_ = json.NewEncoder(w).Encode(obj{"TaskState": "Running", "PercentComplete": 100, "Messages": []obj{{"Message": "Job scheduled successfully."}}})
|
||||
case "/redfish/v1/TaskService/Tasks/BIOS0":
|
||||
_ = json.NewEncoder(w).Encode(obj{"TaskState": "Completed", "PercentComplete": 100})
|
||||
case "/redfish/v1/TaskService/Tasks/BIOS1":
|
||||
_ = json.NewEncoder(w).Encode(obj{"TaskState": "Running", "PercentComplete": 30})
|
||||
case "/redfish/v1/JobService/Jobs/JID_2":
|
||||
_ = json.NewEncoder(w).Encode(obj{"JobState": "Completed", "PercentComplete": 100})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer srv.Close()
|
||||
poll := func(task string) fwPollResult {
|
||||
out, err := powerRun(context.Background(), "", creds(srv.URL, "idrac"), "fw-poll", task)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var p fwPollResult
|
||||
_ = json.Unmarshal([]byte(out), &p)
|
||||
return p
|
||||
}
|
||||
if p := poll("/redfish/v1/TaskService/Tasks/JID_1"); p.State != "done" {
|
||||
t.Errorf("scheduled job: %+v", p)
|
||||
}
|
||||
if p := poll("/redfish/v1/TaskService/Tasks/BIOS0,/redfish/v1/TaskService/Tasks/BIOS1"); p.State != "running" || p.Pct != 30 {
|
||||
t.Errorf("second of two: %+v", p)
|
||||
}
|
||||
if p := poll("/redfish/v1/TaskService/Tasks/JID_2"); p.State != "done" { // moved to the job service
|
||||
t.Errorf("job service: %+v", p)
|
||||
}
|
||||
if p := poll("/redfish/v1/TaskService/Tasks/GONE"); p.State != "unknown" {
|
||||
t.Errorf("gone: %+v", p)
|
||||
}
|
||||
}
|
||||
+7
-5
@@ -1,6 +1,6 @@
|
||||
module rollcall
|
||||
|
||||
go 1.24.2
|
||||
go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/charmbracelet/bubbletea v1.3.10
|
||||
@@ -9,7 +9,8 @@ require (
|
||||
github.com/jackc/pgx/v5 v5.7.5
|
||||
github.com/klauspost/compress v1.18.0
|
||||
github.com/mattn/go-isatty v0.0.24
|
||||
golang.org/x/crypto v0.37.0
|
||||
github.com/miekg/dns v1.1.73
|
||||
golang.org/x/crypto v0.54.0
|
||||
software.sslmate.com/src/go-pkcs12 v0.7.4
|
||||
)
|
||||
|
||||
@@ -33,7 +34,8 @@ require (
|
||||
github.com/muesli/termenv v0.16.0 // indirect
|
||||
github.com/rivo/uniseg v0.4.7 // indirect
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
|
||||
golang.org/x/sync v0.13.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.24.0 // indirect
|
||||
golang.org/x/net v0.57.0 // indirect
|
||||
golang.org/x/sync v0.22.0 // indirect
|
||||
golang.org/x/sys v0.47.0 // indirect
|
||||
golang.org/x/text v0.40.0 // indirect
|
||||
)
|
||||
+14
-10
@@ -41,6 +41,8 @@ github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2J
|
||||
github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88=
|
||||
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
|
||||
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
|
||||
github.com/miekg/dns v1.1.73 h1:uhT8nJxmTrPJYClxVxTCX+CVn6qnzSiybRk72Z6DgrE=
|
||||
github.com/miekg/dns v1.1.73/go.mod h1:RW2Obtfd5NZHvOFe3zYG0W8koWOQtAzyHaLo8vASBuQ=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI=
|
||||
github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo=
|
||||
github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA=
|
||||
@@ -58,19 +60,21 @@ github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKs
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
|
||||
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
|
||||
golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE=
|
||||
golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc=
|
||||
golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw=
|
||||
golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI=
|
||||
golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo=
|
||||
golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610=
|
||||
golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
|
||||
golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE=
|
||||
golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU=
|
||||
golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek=
|
||||
golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0=
|
||||
golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o=
|
||||
golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw=
|
||||
golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0=
|
||||
golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU=
|
||||
golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs=
|
||||
golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
|
||||
golang.org/x/term v0.45.0 h1:NwWyBmoJCbfTHpxrWoZ9C6/VxOf7ic219I8xZZFdrf0=
|
||||
golang.org/x/term v0.45.0/go.mod h1:9aqxs0blBcrm/n0L9QW0aRVD+ktan8ssZromtqJC43w=
|
||||
golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs=
|
||||
golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
package main
|
||||
|
||||
// UEFI machines boot through shim and GRUB, both signed (Debian's
|
||||
// shim-signed and grub-efi-amd64-signed), so a machine with Secure Boot on
|
||||
// PXE-boots at all (syslinux.efi is unsigned: the firmware refuses it) and
|
||||
// GRUB loads only kernels that shim verifies. BIOS machines keep pxelinux.
|
||||
//
|
||||
// TFTP root:
|
||||
//
|
||||
// bootx64.efi shim (shimx64.efi.signed)
|
||||
// grubx64.efi GRUB for network boot (grubnetx64.efi.signed)
|
||||
// grub/grub.cfg picks grub.cfg-<mac>, else grub.cfg-default
|
||||
// grub/grub.cfg-<mac> per machine, made from its pxelinux file
|
||||
//
|
||||
// The rack controller writes the GRUB files from the pxelinux ones it gets,
|
||||
// so both menus always say the same. Secure Boot checks the kernel; the
|
||||
// initrd and the squashfs it fetches are not covered by it.
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const grubDispatch = `# generated by rollcall: the machine's own menu, else the one for unknown machines
|
||||
terminal_output console
|
||||
set timeout=8
|
||||
configfile $prefix/grub.cfg-$net_default_mac
|
||||
configfile $prefix/grub.cfg-default
|
||||
`
|
||||
|
||||
var grubNameRe = regexp.MustCompile(`^grub\.cfg-([0-9a-f]{2}:){5}[0-9a-f]{2}$`)
|
||||
|
||||
// grubName is the GRUB file for a pxelinux one: 01-aa-bb-cc-dd-ee-ff -> grub.cfg-aa:bb:cc:dd:ee:ff.
|
||||
func grubName(pxe string) string {
|
||||
if !strings.HasPrefix(pxe, "01-") {
|
||||
return ""
|
||||
}
|
||||
n := "grub.cfg-" + strings.ReplaceAll(strings.TrimPrefix(pxe, "01-"), "-", ":")
|
||||
if !grubNameRe.MatchString(n) {
|
||||
return ""
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
type pxeEntry struct {
|
||||
label, kernel, initrd, append string
|
||||
local, poweroff bool
|
||||
}
|
||||
|
||||
// pxelinuxToGrub turns one of rollcall's pxelinux menus into the same GRUB menu.
|
||||
func pxelinuxToGrub(body string) string {
|
||||
var entries []*pxeEntry
|
||||
def, timeout := 0, 5
|
||||
var cur *pxeEntry
|
||||
for _, raw := range strings.Split(body, "\n") {
|
||||
line := strings.TrimSpace(raw)
|
||||
f := strings.Fields(line)
|
||||
if len(f) == 0 || strings.HasPrefix(line, "#") {
|
||||
continue
|
||||
}
|
||||
key := strings.ToUpper(f[0])
|
||||
rest := strings.TrimSpace(line[len(f[0]):])
|
||||
switch {
|
||||
case key == "TIMEOUT":
|
||||
if n, err := strconv.Atoi(rest); err == nil { // tenths of a second
|
||||
timeout = (n + 9) / 10
|
||||
}
|
||||
case key == "LABEL":
|
||||
cur = &pxeEntry{label: rest}
|
||||
entries = append(entries, cur)
|
||||
case cur == nil:
|
||||
case key == "MENU" && len(f) > 1 && strings.EqualFold(f[1], "LABEL"):
|
||||
cur.label = strings.TrimSpace(rest[len(f[1]):])
|
||||
case key == "MENU" && len(f) > 1 && strings.EqualFold(f[1], "DEFAULT"):
|
||||
def = len(entries) - 1
|
||||
case key == "KERNEL":
|
||||
if strings.HasSuffix(rest, "poweroff.c32") {
|
||||
cur.poweroff = true
|
||||
} else {
|
||||
cur.kernel = rest
|
||||
}
|
||||
case key == "INITRD":
|
||||
cur.initrd = rest
|
||||
case key == "APPEND":
|
||||
cur.append = rest
|
||||
case key == "LOCALBOOT":
|
||||
cur.local = true
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
b.WriteString("# generated by rollcall from the pxelinux menu\n")
|
||||
// the firmware's text console: GRUB does not switch the display mode, so the OS
|
||||
// it starts gets the firmware's own (OVMF showed a garbled console after a switch),
|
||||
// and serial/SOL consoles see the menu
|
||||
b.WriteString("terminal_output console\n")
|
||||
fmt.Fprintf(&b, "set default=%d\nset timeout=%d\n", def, timeout)
|
||||
for _, e := range entries {
|
||||
fmt.Fprintf(&b, "menuentry %s {\n", grubQuote(e.label))
|
||||
switch {
|
||||
case e.poweroff:
|
||||
b.WriteString(" halt\n")
|
||||
case e.local:
|
||||
b.WriteString(grubLocalBoot)
|
||||
case e.kernel != "":
|
||||
fmt.Fprintf(&b, " linux %s", grubQuote("/"+strings.TrimPrefix(e.kernel, "/")))
|
||||
for _, w := range strings.Fields(e.append) {
|
||||
b.WriteString(" " + grubQuote(w))
|
||||
}
|
||||
b.WriteString("\n")
|
||||
if e.initrd != "" {
|
||||
fmt.Fprintf(&b, " initrd %s\n", grubQuote("/"+strings.TrimPrefix(e.initrd, "/")))
|
||||
}
|
||||
default:
|
||||
b.WriteString(" exit 1\n")
|
||||
}
|
||||
b.WriteString("}\n")
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
// grubLocalBoot starts the installed OS by chainloading its signed loader
|
||||
// from the disk's EFI partition (shim, which Secure Boot accepts). Handing
|
||||
// back to the firmware with "exit" is not enough: firmware then tries its
|
||||
// other network boot options and ends in its own menu (OVMF does).
|
||||
const grubLocalBoot = ` insmod part_gpt
|
||||
insmod fat
|
||||
insmod chain
|
||||
for f in /EFI/debian/shimx64.efi /EFI/ubuntu/shimx64.efi /EFI/rocky/shimx64.efi /EFI/almalinux/shimx64.efi /EFI/redhat/shimx64.efi /EFI/centos/shimx64.efi /EFI/opensuse/shim.efi /EFI/sles/shim.efi /EFI/BOOT/BOOTX64.EFI; do
|
||||
if search --no-floppy --set=root --file $f; then
|
||||
chainloader $f
|
||||
boot
|
||||
fi
|
||||
done
|
||||
echo "no EFI loader found on the disks"
|
||||
sleep 10
|
||||
exit 1
|
||||
`
|
||||
|
||||
var grubPlain = regexp.MustCompile(`^[A-Za-z0-9_./:=,+@%-]+$`)
|
||||
|
||||
// grubQuote makes a word safe in GRUB's script language (no $ expansion, no ;).
|
||||
func grubQuote(s string) string {
|
||||
if grubPlain.MatchString(s) {
|
||||
return s
|
||||
}
|
||||
return "'" + strings.ReplaceAll(s, "'", `'\''`) + "'"
|
||||
}
|
||||
@@ -0,0 +1,40 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPxelinuxToGrub(t *testing.T) {
|
||||
commissioned := fmt.Sprintf(commissionedTmpl, "now", "m-1", "QEMU", "PC", "S1", "m-1", "QEMU", "PC", "S1",
|
||||
"boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs rollcall.server=http://10.213.77.1 rollcall.token=abc")
|
||||
g := pxelinuxToGrub(commissioned)
|
||||
for _, want := range []string{
|
||||
"set default=0\nset timeout=15\n",
|
||||
"menuentry 'Commissioned: power off (default in 15s)' {\n halt\n}",
|
||||
"menuentry 'Re-run commissioning' {\n linux /rollcall/vmlinuz boot=live components fetch=http://10.213.77.1/rollcall/filesystem.squashfs",
|
||||
" initrd /rollcall/initrd.img\n",
|
||||
"menuentry 'Boot from local disk' {\n insmod part_gpt\n",
|
||||
"chainloader $f",
|
||||
} {
|
||||
if !strings.Contains(g, want) {
|
||||
t.Errorf("missing %q in\n%s", want, g)
|
||||
}
|
||||
}
|
||||
deployed := fmt.Sprintf(deployedTmpl, "now", "m-1", "debian-13", "1", "node01", "debian-13", "m-1", "debian-13", "boot=live")
|
||||
if g := pxelinuxToGrub(deployed); !strings.Contains(g, "set default=0\n") || !strings.Contains(g, "/EFI/debian/shimx64.efi") {
|
||||
t.Errorf("deployed:\n%s", g)
|
||||
}
|
||||
// nothing from a cmdline can break out of the menu entry
|
||||
g = pxelinuxToGrub("LABEL x\n MENU DEFAULT\n KERNEL k\n APPEND a=$(x) b;reboot c='q'\n")
|
||||
if !strings.Contains(g, `'a=$(x)' 'b;reboot' 'c='\''q'\'''`) {
|
||||
t.Errorf("quoting:\n%s", g)
|
||||
}
|
||||
if n := grubName("01-bc-24-11-f7-53-6b"); n != "grub.cfg-bc:24:11:f7:53:6b" {
|
||||
t.Errorf("name %q", n)
|
||||
}
|
||||
if grubName("01-../x") != "" || grubName("default") != "" {
|
||||
t.Error("a non-MAC file name became a GRUB file")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,334 @@
|
||||
package main
|
||||
|
||||
// Infoblox WAPI driver for the IPAM authority. Each Rollcall address is one
|
||||
// record:host holding that address, named by its DNS name (rollcall-<address>
|
||||
// without one) and configured for DNS when it has a name and dns_view is not
|
||||
// "-" (Infoblox then publishes the A/AAAA record itself). Ownership is the extensible
|
||||
// attribute machine_field (default rollcall_machine), which must be defined
|
||||
// in the grid (String). Delegated networks carry the extensible attribute
|
||||
// named by tag (default rollcall). An address is taken when anything that is
|
||||
// not Rollcall's host uses it: a lease, a fixed address, a DHCP or reserved
|
||||
// range, another host, the network or broadcast address.
|
||||
//
|
||||
// Documentation: Infoblox WAPI reference (v2.12): record:host, ipv4address,
|
||||
// ipv6address, network, ipv6network; extensible attribute search (*EA~=),
|
||||
// "extattrs+" to change attributes without dropping others, paging
|
||||
// (_paging, _return_as_object, _page_id).
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type infoblox struct {
|
||||
base string // https://gm.example.com/wapi/v2.12
|
||||
user, pass string
|
||||
field, tag string
|
||||
networkView string
|
||||
dnsView string // "" = no DNS (IPAM-only hosts)
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
func newInfoblox(c IPAMConfig) (*infoblox, error) {
|
||||
if c.URL == "" || c.User == "" || c.PasswordFile == "" {
|
||||
return nil, fmt.Errorf("infoblox: url, user and password_file are required")
|
||||
}
|
||||
pw, err := os.ReadFile(c.PasswordFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("infoblox password: %w", err)
|
||||
}
|
||||
hc, err := httpClientCA(c.CAFile, 30*time.Second)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("infoblox ca_file: %w", err)
|
||||
}
|
||||
base := strings.TrimRight(c.URL, "/")
|
||||
if !strings.Contains(base, "/wapi/") {
|
||||
base += "/wapi/v2.12"
|
||||
}
|
||||
dv := orDefault(c.DNSView, "default")
|
||||
if dv == "-" {
|
||||
dv = ""
|
||||
}
|
||||
return &infoblox{base: base, user: c.User, pass: strings.TrimSpace(string(pw)), field: c.field(), tag: c.tag(),
|
||||
networkView: orDefault(c.NetworkView, "default"), dnsView: dv, client: hc}, nil
|
||||
}
|
||||
|
||||
func (ib *infoblox) Name() string { return "infoblox" }
|
||||
|
||||
type ibError struct {
|
||||
Code int
|
||||
Body string
|
||||
}
|
||||
|
||||
func (e *ibError) Error() string {
|
||||
var x struct{ Text, Code string }
|
||||
if json.Unmarshal([]byte(e.Body), &x) == nil && x.Text != "" {
|
||||
return fmt.Sprintf("HTTP %d: %s", e.Code, trunc(x.Text, 300))
|
||||
}
|
||||
return fmt.Sprintf("HTTP %d: %s", e.Code, trunc(e.Body, 300))
|
||||
}
|
||||
|
||||
// do calls WAPI; path is an object type or reference with its query.
|
||||
func (ib *infoblox) do(ctx context.Context, method, path string, q url.Values, body, out any) error {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
u := ib.base + "/" + strings.TrimPrefix(path, "/")
|
||||
if len(q) > 0 {
|
||||
u += "?" + q.Encode()
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, u, rd)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.SetBasicAuth(ib.user, ib.pass)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
resp, err := ib.client.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 16<<20))
|
||||
if resp.StatusCode >= 300 {
|
||||
return &ibError{resp.StatusCode, string(b)}
|
||||
}
|
||||
if out != nil && len(b) > 0 {
|
||||
return json.Unmarshal(b, out)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// all reads every object of a search through WAPI paging.
|
||||
func (ib *infoblox) all(ctx context.Context, objType string, q url.Values) ([]obj, error) {
|
||||
q.Set("_paging", "1")
|
||||
q.Set("_return_as_object", "1")
|
||||
q.Set("_max_results", "1000")
|
||||
var out []obj
|
||||
for {
|
||||
var p struct {
|
||||
Result []obj `json:"result"`
|
||||
NextPageID string `json:"next_page_id"`
|
||||
}
|
||||
if err := ib.do(ctx, "GET", objType, q, nil, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p.Result...)
|
||||
if p.NextPageID == "" {
|
||||
return out, nil
|
||||
}
|
||||
q = url.Values{"_page_id": {p.NextPageID}}
|
||||
}
|
||||
}
|
||||
|
||||
const ibHostFields = "name,comment,extattrs,configure_for_dns,ipv4addrs,ipv6addrs,view"
|
||||
|
||||
func ibIs6(ip string) bool {
|
||||
a, ok := parseAddr(ip)
|
||||
return ok && a.Is6()
|
||||
}
|
||||
|
||||
// owner is the machine of a host record, "" when it is not Rollcall's.
|
||||
func (ib *infoblox) owner(h obj) string {
|
||||
return str(asObj(asObj(h["extattrs"])[ib.field])["value"])
|
||||
}
|
||||
|
||||
func (ib *infoblox) hostIP(h obj) string {
|
||||
for _, k := range []string{"ipv4addrs", "ipv6addrs"} {
|
||||
for _, a := range toList(h[k]) {
|
||||
for _, f := range []string{"ipv4addr", "ipv6addr"} {
|
||||
if s := str(asObj(a)[f]); s != "" {
|
||||
if p, ok := parseAddr(s); ok {
|
||||
return p.String()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (ib *infoblox) toExt(h obj) extAddr {
|
||||
e := extAddr{ID: str(h["_ref"]), IP: ib.hostIP(h), Machine: ib.owner(h), Desc: str(h["comment"])}
|
||||
if n := strings.ToLower(str(h["name"])); n != ibHostName(e) {
|
||||
e.DNSName = n
|
||||
}
|
||||
return e // Bits and Status are not kept by Infoblox (0 and "": not compared)
|
||||
}
|
||||
|
||||
func ibHostName(a extAddr) string {
|
||||
return "rollcall-" + strings.NewReplacer(".", "-", ":", "-").Replace(a.IP)
|
||||
}
|
||||
|
||||
func (ib *infoblox) hostBody(a extAddr, create bool) obj {
|
||||
dns := a.DNSName != "" && ib.dnsView != ""
|
||||
// the name is the DNS name even when Infoblox does not publish it (IPAM only)
|
||||
b := obj{"name": orDefault(a.DNSName, ibHostName(a)), "comment": trunc(a.Desc, 256), "configure_for_dns": dns}
|
||||
if create {
|
||||
addr := "ipv4addr"
|
||||
if ibIs6(a.IP) {
|
||||
addr = "ipv6addr"
|
||||
}
|
||||
b[addr+"s"] = []obj{{addr: a.IP, "configure_for_dhcp": false}}
|
||||
b["extattrs"] = obj{ib.field: obj{"value": a.Machine}}
|
||||
b["network_view"] = ib.networkView
|
||||
if dns {
|
||||
b["view"] = ib.dnsView
|
||||
}
|
||||
} else {
|
||||
b["extattrs+"] = obj{ib.field: obj{"value": a.Machine}} // other attributes stay
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func (ib *infoblox) hostsAt(ctx context.Context, ip string) ([]obj, error) {
|
||||
k := "ipv4addr"
|
||||
if ibIs6(ip) {
|
||||
k = "ipv6addr"
|
||||
}
|
||||
return ib.all(ctx, "record:host", url.Values{k: {ip}, "network_view": {ib.networkView}, "_return_fields": {ibHostFields}})
|
||||
}
|
||||
|
||||
// used: references of whatever uses ip (empty when Infoblox has no network for it).
|
||||
func (ib *infoblox) used(ctx context.Context, ip string) (status string, refs []string, types []string, err error) {
|
||||
t := "ipv4address"
|
||||
if ibIs6(ip) {
|
||||
t = "ipv6address"
|
||||
}
|
||||
var res []obj
|
||||
err = ib.do(ctx, "GET", t, url.Values{"ip_address": {ip}, "network_view": {ib.networkView}, "_return_fields": {"status,types,objects"}}, nil, &res)
|
||||
if err != nil || len(res) == 0 {
|
||||
return "", nil, nil, err
|
||||
}
|
||||
for _, r := range toList(res[0]["objects"]) {
|
||||
refs = append(refs, str(r))
|
||||
}
|
||||
for _, x := range toList(res[0]["types"]) {
|
||||
types = append(types, str(x))
|
||||
}
|
||||
return str(res[0]["status"]), refs, types, nil
|
||||
}
|
||||
|
||||
func (ib *infoblox) Reserve(ctx context.Context, a extAddr) (bool, error) {
|
||||
hosts, err := ib.hostsAt(ctx, a.IP)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
mine := map[string]bool{}
|
||||
var cur obj
|
||||
for _, h := range hosts {
|
||||
if ib.owner(h) == "" {
|
||||
return true, nil // someone else's host
|
||||
}
|
||||
mine[str(h["_ref"])] = true
|
||||
cur = h
|
||||
}
|
||||
status, refs, types, err := ib.used(ctx, a.IP)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
for _, t := range types {
|
||||
switch t {
|
||||
case "NETWORK", "BROADCAST", "DHCP_RANGE", "RESERVED_RANGE":
|
||||
return true, nil
|
||||
}
|
||||
}
|
||||
others := 0
|
||||
for _, r := range refs {
|
||||
if !mine[r] {
|
||||
others++
|
||||
}
|
||||
}
|
||||
if others > 0 || (status == "USED" && len(refs) == 0 && len(mine) == 0) {
|
||||
return true, nil // a lease, a fixed address, a discovered or conflicting use
|
||||
}
|
||||
if cur != nil {
|
||||
e := ib.toExt(cur)
|
||||
a.ID = e.ID
|
||||
if e.DNSName != a.DNSName || e.Machine != a.Machine || e.Desc != a.Desc {
|
||||
return false, ib.Update(ctx, a)
|
||||
}
|
||||
return false, nil
|
||||
}
|
||||
return false, ib.do(ctx, "POST", "record:host", nil, ib.hostBody(a, true), nil)
|
||||
}
|
||||
|
||||
func (ib *infoblox) Owned(ctx context.Context) ([]extAddr, error) {
|
||||
hs, err := ib.all(ctx, "record:host", url.Values{"*" + ib.field + "~": {"."}, "network_view": {ib.networkView}, "_return_fields": {ibHostFields}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []extAddr{}
|
||||
for _, h := range hs {
|
||||
if e := ib.toExt(h); e.Machine != "" && e.IP != "" {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Update renames, re-owns and re-describes a host; switching DNS on or off
|
||||
// re-creates it (a host's DNS view is fixed at creation).
|
||||
func (ib *infoblox) Update(ctx context.Context, a extAddr) error {
|
||||
var cur obj
|
||||
if err := ib.do(ctx, "GET", a.ID, url.Values{"_return_fields": {ibHostFields}}, nil, &cur); err != nil {
|
||||
return err
|
||||
}
|
||||
if ib.owner(cur) == "" {
|
||||
return fmt.Errorf("%s is no longer Rollcall's", a.IP)
|
||||
}
|
||||
if (cur["configure_for_dns"] == true) != (a.DNSName != "" && ib.dnsView != "") {
|
||||
if err := ib.do(ctx, "DELETE", a.ID, nil, nil, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
return ib.do(ctx, "POST", "record:host", nil, ib.hostBody(a, true), nil)
|
||||
}
|
||||
return ib.do(ctx, "PUT", a.ID, nil, ib.hostBody(a, false), nil)
|
||||
}
|
||||
|
||||
func (ib *infoblox) Delete(ctx context.Context, a extAddr) error {
|
||||
var cur obj // re-read: only delete what is still ours
|
||||
if err := ib.do(ctx, "GET", a.ID, url.Values{"_return_fields": {ibHostFields}}, nil, &cur); err != nil {
|
||||
if e, ok := err.(*ibError); ok && (e.Code == 404 || e.Code == 400) {
|
||||
return nil // gone (WAPI answers 400 for a reference that no longer exists)
|
||||
}
|
||||
return err
|
||||
}
|
||||
if ib.owner(cur) == "" {
|
||||
return nil
|
||||
}
|
||||
return ib.do(ctx, "DELETE", a.ID, nil, nil, nil)
|
||||
}
|
||||
|
||||
func (ib *infoblox) Prefixes(ctx context.Context) ([]extPrefix, error) {
|
||||
out := []extPrefix{}
|
||||
for _, t := range []string{"network", "ipv6network"} {
|
||||
ns, err := ib.all(ctx, t, url.Values{"*" + ib.tag + "~": {"."}, "network_view": {ib.networkView}, "_return_fields": {"network,comment,extattrs"}})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
for _, n := range ns {
|
||||
p, err := netip.ParsePrefix(str(n["network"]))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
ea := asObj(n["extattrs"])
|
||||
e := extPrefix{Prefix: p.Masked().String(), Desc: str(n["comment"]), Site: str(asObj(ea["Site"])["value"])}
|
||||
e.VID, _ = strconv.Atoi(fmt.Sprint(asObj(ea["VLAN"])["value"]))
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
@@ -0,0 +1,336 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeWAPI implements the slice of Infoblox WAPI v2.12 the driver uses:
|
||||
// record:host (search, create, read, update with extattrs+, delete), address
|
||||
// usage (ipv4address/ipv6address), networks and paging.
|
||||
type fakeWAPI struct {
|
||||
mu sync.Mutex
|
||||
next int
|
||||
hosts map[string]map[string]any // ref -> host
|
||||
uses map[string][]string // ip -> other objects using it (leases, fixed addresses)
|
||||
types map[string][]string // ip -> address types (DHCP_RANGE, ...)
|
||||
networks []map[string]any
|
||||
pages map[string][]any
|
||||
writes []string
|
||||
}
|
||||
|
||||
func newFakeWAPI() *fakeWAPI {
|
||||
return &fakeWAPI{hosts: map[string]map[string]any{}, uses: map[string][]string{}, types: map[string][]string{}, pages: map[string][]any{}}
|
||||
}
|
||||
|
||||
func (f *fakeWAPI) host(name, ip string, ea map[string]any, dns bool) string {
|
||||
f.next++
|
||||
ref := fmt.Sprintf("record:host/ZG5zLmhvc3Qk%d:%s/default", f.next, name)
|
||||
k := "ipv4addrs"
|
||||
addr := map[string]any{"ipv4addr": ip}
|
||||
if strings.Contains(ip, ":") {
|
||||
k, addr = "ipv6addrs", map[string]any{"ipv6addr": ip}
|
||||
}
|
||||
eas := map[string]any{}
|
||||
for n, v := range ea {
|
||||
eas[n] = map[string]any{"value": v}
|
||||
}
|
||||
f.hosts[ref] = map[string]any{"_ref": ref, "name": name, "comment": "", "configure_for_dns": dns, k: []any{addr}, "extattrs": eas}
|
||||
return ref
|
||||
}
|
||||
|
||||
func hostIPs(h map[string]any) []string {
|
||||
var out []string
|
||||
for _, k := range []string{"ipv4addrs", "ipv6addrs"} {
|
||||
for _, a := range toList(h[k]) {
|
||||
out = append(out, str(asObj(a)["ipv4addr"])+str(asObj(a)["ipv6addr"]))
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (f *fakeWAPI) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
write := func(code int, v any) { w.WriteHeader(code); _ = json.NewEncoder(w).Encode(v) }
|
||||
if u, p, ok := r.BasicAuth(); !ok || u != "rollcall" || p != "s3cret" {
|
||||
write(401, map[string]any{"text": "Authorization Required"})
|
||||
return
|
||||
}
|
||||
path := strings.TrimPrefix(r.URL.Path, "/wapi/v2.12/")
|
||||
q := r.URL.Query()
|
||||
var body map[string]any
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if r.Method != "GET" {
|
||||
f.writes = append(f.writes, r.Method+" "+path)
|
||||
}
|
||||
page := func(res []any) {
|
||||
if id := q.Get("_page_id"); id != "" {
|
||||
res = f.pages[id]
|
||||
}
|
||||
const size = 2 // small pages so paging is exercised
|
||||
out := map[string]any{"result": res}
|
||||
if len(res) > size {
|
||||
id := strconv.Itoa(len(f.pages) + 1)
|
||||
f.pages[id] = res[size:]
|
||||
out = map[string]any{"result": res[:size], "next_page_id": id}
|
||||
}
|
||||
write(200, out)
|
||||
}
|
||||
switch {
|
||||
case path == "record:host" && r.Method == "GET":
|
||||
refs := []string{}
|
||||
for ref := range f.hosts {
|
||||
refs = append(refs, ref)
|
||||
}
|
||||
sort.Strings(refs)
|
||||
res := []any{}
|
||||
for _, ref := range refs {
|
||||
h := f.hosts[ref]
|
||||
ok := true
|
||||
for k, v := range q {
|
||||
switch {
|
||||
case k == "ipv4addr" || k == "ipv6addr":
|
||||
ok = ok && strings.Contains(" "+strings.Join(hostIPs(h), " ")+" ", " "+v[0]+" ")
|
||||
case strings.HasPrefix(k, "*") && strings.HasSuffix(k, "~"):
|
||||
_, has := asObj(h["extattrs"])[strings.TrimSuffix(k[1:], "~")]
|
||||
ok = ok && has
|
||||
}
|
||||
}
|
||||
if ok {
|
||||
res = append(res, h)
|
||||
}
|
||||
}
|
||||
page(res)
|
||||
case path == "record:host" && r.Method == "POST":
|
||||
ea := map[string]any{}
|
||||
for k, v := range asObj(body["extattrs"]) {
|
||||
ea[k] = asObj(v)["value"]
|
||||
}
|
||||
ip := ""
|
||||
if a := toList(body["ipv4addrs"]); len(a) > 0 {
|
||||
ip = str(asObj(a[0])["ipv4addr"])
|
||||
} else if a := toList(body["ipv6addrs"]); len(a) > 0 {
|
||||
ip = str(asObj(a[0])["ipv6addr"])
|
||||
}
|
||||
if body["configure_for_dns"] == true && body["view"] != "default" {
|
||||
write(400, map[string]any{"Error": "AdmConDataNotFoundError", "code": "Client.Ibap.Data.NotFound", "text": "view not found"})
|
||||
return
|
||||
}
|
||||
ref := f.host(str(body["name"]), ip, ea, body["configure_for_dns"] == true)
|
||||
f.hosts[ref]["comment"] = body["comment"]
|
||||
write(201, ref)
|
||||
case path == "ipv4address" || path == "ipv6address":
|
||||
ip := q.Get("ip_address")
|
||||
objs := append([]string{}, f.uses[ip]...)
|
||||
for ref, h := range f.hosts {
|
||||
for _, x := range hostIPs(h) {
|
||||
if x == ip {
|
||||
objs = append(objs, ref)
|
||||
}
|
||||
}
|
||||
}
|
||||
status := "UNUSED"
|
||||
if len(objs) > 0 {
|
||||
status = "USED"
|
||||
}
|
||||
write(200, []any{map[string]any{"ip_address": ip, "status": status, "objects": objs, "types": f.types[ip]}})
|
||||
case path == "network" || path == "ipv6network":
|
||||
res := []any{}
|
||||
for _, n := range f.networks {
|
||||
if strings.Contains(str(n["network"]), ":") == (path == "ipv6network") {
|
||||
res = append(res, n)
|
||||
}
|
||||
}
|
||||
page(res)
|
||||
case strings.HasPrefix(path, "record:host/"):
|
||||
h, ok := f.hosts[path]
|
||||
if !ok {
|
||||
write(400, map[string]any{"Error": "AdmConProtoError: Unknown object", "text": "Unknown object"})
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
write(200, h)
|
||||
case "PUT":
|
||||
for k, v := range body {
|
||||
if k == "extattrs+" {
|
||||
for n, x := range asObj(v) {
|
||||
asObj(h["extattrs"])[n] = x
|
||||
}
|
||||
continue
|
||||
}
|
||||
h[k] = v
|
||||
}
|
||||
write(200, path)
|
||||
case "DELETE":
|
||||
delete(f.hosts, path)
|
||||
write(200, path)
|
||||
}
|
||||
default:
|
||||
write(400, map[string]any{"text": "unknown " + path})
|
||||
}
|
||||
}
|
||||
|
||||
func testInfoblox(t *testing.T, dnsView string) (*App, *fakeWAPI) {
|
||||
t.Helper()
|
||||
f := newFakeWAPI()
|
||||
srv := httptest.NewServer(f)
|
||||
t.Cleanup(srv.Close)
|
||||
pw := filepath.Join(t.TempDir(), "pw")
|
||||
if err := os.WriteFile(pw, []byte("s3cret\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := &Config{IPAM: IPAMConfig{Provider: "infoblox", URL: srv.URL, User: "rollcall", PasswordFile: pw, DNSZone: "int.example", DNSView: dnsView}}
|
||||
p, err := newIPAMProvider(cfg.IPAM)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
a := &App{cfg: cfg, ipam: p, ipamSt: &ipamState{kick: make(chan struct{}, 1)}}
|
||||
a.subnets = []Subnet{{CIDR: "10.213.77.0/24", Purpose: "provisioning"}, {CIDR: "fd00:213:77::/64", Purpose: "provisioning"}}
|
||||
return a, f
|
||||
}
|
||||
|
||||
func TestInfobloxReserve(t *testing.T) {
|
||||
a, f := testInfoblox(t, "")
|
||||
ctx := context.Background()
|
||||
f.host("printer.int.example", "10.213.77.20", nil, true) // someone else's host
|
||||
f.uses["10.213.77.21"] = []string{"lease/ZG5zLmxlYXNl:10.213.77.21/default"} // a lease
|
||||
f.types["10.213.77.22"] = []string{"DHCP_RANGE"} // inside a DHCP range
|
||||
mine := f.host("old.int.example", "10.213.77.23", map[string]any{"rollcall_machine": "m-1", "Owner": "ops"}, true)
|
||||
f.uses["10.213.77.24"] = []string{"fixedaddress/ZG5z:10.213.77.24/default"} // a fixed address
|
||||
for ip, want := range map[string]bool{"10.213.77.20": true, "10.213.77.21": true, "10.213.77.22": true, "10.213.77.23": false, "10.213.77.24": true, "10.213.77.25": false} {
|
||||
taken, err := a.ipam.Reserve(ctx, a.extFor(IPAddr{IP: ip, Subnet: "10.213.77.0/24", MachineID: "m-2", Hostname: "node002", Iface: "eth0"}, &a.subnets[0]))
|
||||
if err != nil {
|
||||
t.Fatal(ip, err)
|
||||
}
|
||||
if taken != want {
|
||||
t.Errorf("%s: taken=%v want %v", ip, taken, want)
|
||||
}
|
||||
}
|
||||
h := f.hosts[mine]
|
||||
if h["name"] != "node002.int.example" || asObj(asObj(h["extattrs"])["rollcall_machine"])["value"] != "m-2" ||
|
||||
asObj(asObj(h["extattrs"])["Owner"])["value"] != "ops" {
|
||||
t.Errorf("re-owned host %v (other extensible attributes must stay)", h)
|
||||
}
|
||||
var made map[string]any
|
||||
for _, x := range f.hosts {
|
||||
if hostIPs(x)[0] == "10.213.77.25" {
|
||||
made = x
|
||||
}
|
||||
}
|
||||
if made == nil || made["name"] != "node002.int.example" || made["configure_for_dns"] != true || made["comment"] != "rollcall m-2 eth0" {
|
||||
t.Fatalf("new host %v", made)
|
||||
}
|
||||
// v6 and a name-less reservation
|
||||
if _, err := a.ipam.Reserve(ctx, a.extFor(IPAddr{IP: "fd00:213:77::50", Subnet: "fd00:213:77::/64", Kind: "reserved", Hostname: "core switch"}, &a.subnets[1])); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var v6 map[string]any
|
||||
for _, x := range f.hosts {
|
||||
if hostIPs(x)[0] == "fd00:213:77::50" {
|
||||
v6 = x
|
||||
}
|
||||
}
|
||||
if v6 == nil || v6["name"] != "rollcall-fd00-213-77--50" || v6["configure_for_dns"] != false {
|
||||
t.Fatalf("v6 reservation %v", v6)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInfobloxReconcile(t *testing.T) {
|
||||
a, f := testInfoblox(t, "")
|
||||
ctx := context.Background()
|
||||
foreign := f.host("printer.int.example", "10.213.77.20", nil, true)
|
||||
f.host("gone.int.example", "10.213.77.30", map[string]any{"rollcall_machine": "m-9"}, true)
|
||||
f.host("wrong.int.example", "10.213.77.50", map[string]any{"rollcall_machine": "m-1"}, true)
|
||||
f.host("a.int.example", "10.213.77.60", map[string]any{"rollcall_machine": "m-5"}, true)
|
||||
f.host("b.int.example", "10.213.77.61", map[string]any{"rollcall_machine": "m-5"}, true) // paging: 5 hosts, pages of 2
|
||||
local := []IPAddr{
|
||||
{IP: "10.213.77.20", Subnet: "10.213.77.0/24", MachineID: "m-2", Hostname: "node002", Kind: "auto"},
|
||||
{IP: "10.213.77.50", Subnet: "10.213.77.0/24", MachineID: "m-1", Hostname: "node001", Kind: "auto"},
|
||||
{IP: "10.213.77.51", Subnet: "10.213.77.0/24", MachineID: "m-4", Hostname: "node004", Kind: "static"},
|
||||
{IP: "10.213.77.60", Subnet: "10.213.77.0/24", MachineID: "m-5", Hostname: "a", Kind: "auto"},
|
||||
{IP: "10.213.77.61", Subnet: "10.213.77.0/24", MachineID: "m-5", Hostname: "b", Kind: "auto"},
|
||||
}
|
||||
for _, x := range f.hosts { // descriptions as Rollcall writes them
|
||||
if m := ib(a).owner(x); m != "" {
|
||||
x["comment"] = "rollcall " + m
|
||||
}
|
||||
}
|
||||
rep := ipamReport{}
|
||||
a.reconcileAddrs(ctx, local, &rep)
|
||||
if rep.Error != "" {
|
||||
t.Fatal(rep.Error)
|
||||
}
|
||||
if fmt.Sprint(rep.Created, rep.Updated, rep.Deleted, len(rep.Conflicts)) != "[10.213.77.51] [10.213.77.50] [10.213.77.30] 1" {
|
||||
t.Errorf("report: %+v", rep)
|
||||
}
|
||||
if f.hosts[foreign]["name"] != "printer.int.example" {
|
||||
t.Error("foreign host touched")
|
||||
}
|
||||
f.writes = nil
|
||||
again := ipamReport{}
|
||||
a.reconcileAddrs(ctx, local, &again)
|
||||
if len(f.writes) != 0 || again.Error != "" {
|
||||
t.Errorf("second pass wrote %v (%+v): bits and status are not kept by Infoblox and must not count", f.writes, again)
|
||||
}
|
||||
// the delete re-checks ownership
|
||||
for ref, x := range f.hosts {
|
||||
if x["name"] == "node004.int.example" {
|
||||
delete(asObj(x["extattrs"]), "rollcall_machine")
|
||||
if err := a.ipam.Delete(ctx, extAddr{ID: ref}); err != nil || f.hosts[ref] == nil {
|
||||
t.Fatalf("deleted a host that is no longer Rollcall's: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := a.ipam.Delete(ctx, extAddr{ID: "record:host/gone:x/default"}); err != nil {
|
||||
t.Errorf("deleting a vanished host: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInfobloxDNSView(t *testing.T) {
|
||||
a, f := testInfoblox(t, "internal")
|
||||
_, err := a.ipam.Reserve(context.Background(), a.extFor(IPAddr{IP: "10.213.77.40", Subnet: "10.213.77.0/24", MachineID: "m-1", Hostname: "n1"}, &a.subnets[0]))
|
||||
if err == nil || !strings.Contains(err.Error(), "view not found") {
|
||||
t.Fatalf("WAPI error not passed on: %v", err)
|
||||
}
|
||||
if len(f.hosts) != 0 {
|
||||
t.Fatal("host made")
|
||||
}
|
||||
}
|
||||
|
||||
func TestInfobloxPrefixes(t *testing.T) {
|
||||
a, f := testInfoblox(t, "")
|
||||
f.networks = []map[string]any{
|
||||
{"network": "10.213.80.0/24", "comment": "new hall", "extattrs": map[string]any{"Site": map[string]any{"value": "DC1"}, "VLAN": map[string]any{"value": "300"}}},
|
||||
{"network": "fd00:213:80::/64", "comment": "new hall v6", "extattrs": map[string]any{}},
|
||||
}
|
||||
ps, err := a.ipam.Prefixes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fmt.Sprint(ps) != "[{10.213.80.0/24 new hall 300 DC1 false } {fd00:213:80::/64 new hall v6 0 false }]" {
|
||||
t.Errorf("%v", ps)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInfobloxAuth(t *testing.T) {
|
||||
a, _ := testInfoblox(t, "")
|
||||
ib(a).pass = "wrong"
|
||||
_, err := a.ipam.Owned(context.Background())
|
||||
if err == nil || !strings.Contains(err.Error(), "401") {
|
||||
t.Fatalf("%v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func ib(a *App) *infoblox { return a.ipam.(*infoblox) }
|
||||
+110
-25
@@ -37,14 +37,20 @@ import (
|
||||
)
|
||||
|
||||
type IPAMConfig struct {
|
||||
Provider string `json:"provider"` // "" = built-in only | netbox
|
||||
Provider string `json:"provider"` // "" = built-in only | netbox | infoblox
|
||||
URL string `json:"url"` // https://netbox.example
|
||||
TokenFile string `json:"token_file"` // API token (NetBox v2 "nbt_..." or v1)
|
||||
User string `json:"user"` // Infoblox WAPI user
|
||||
PasswordFile string `json:"password_file"` // Infoblox WAPI password
|
||||
NetworkView string `json:"network_view"` // Infoblox, default "default"
|
||||
DNSView string `json:"dns_view"` // Infoblox host records' DNS view, default "default"; "-" = IPAM only
|
||||
CAFile string `json:"ca_file"` // CA that signed the server certificate ("" = system roots)
|
||||
DNSZone string `json:"dns_zone"` // dns_name = <host>.<zone> unless the subnet has a domain
|
||||
Tag string `json:"tag"` // ownership tag, default "rollcall"
|
||||
MachineField string `json:"machine_field"` // custom field holding the machine ID, default "rollcall_machine"
|
||||
SyncInterval string `json:"sync_interval"` // reconcile period, default 5m
|
||||
// DeviceSync mirrors machines as NetBox devices (nbdevices.go); nil = off
|
||||
DeviceSync *DeviceSyncConfig `json:"device_sync,omitempty"`
|
||||
}
|
||||
|
||||
// extAddr is one address as the authority holds it.
|
||||
@@ -79,14 +85,31 @@ type ipamProvider interface {
|
||||
}
|
||||
|
||||
type ipamReport struct {
|
||||
Provider string `json:"provider"`
|
||||
At time.Time `json:"at"`
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
Created []string `json:"created"`
|
||||
Updated []string `json:"updated"`
|
||||
Deleted []string `json:"deleted"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Provider string `json:"provider"`
|
||||
At time.Time `json:"at"`
|
||||
DryRun bool `json:"dry_run,omitempty"`
|
||||
Created []string `json:"created"`
|
||||
Updated []string `json:"updated"`
|
||||
Deleted []string `json:"deleted"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Error string `json:"error,omitempty"`
|
||||
Devices *devReport `json:"devices,omitempty"` // ipam.device_sync
|
||||
}
|
||||
|
||||
// fail notes one record's error; the reconcile goes on with the others.
|
||||
func (r *ipamReport) fail(e string) { r.Error = joinErr(r.Error, e) }
|
||||
|
||||
// joinErr keeps the first few errors of a pass readable.
|
||||
func joinErr(all, e string) string {
|
||||
switch n := strings.Count(all, "; ") + 1; {
|
||||
case all == "":
|
||||
return e
|
||||
case n < 5:
|
||||
return all + "; " + e
|
||||
case n == 5:
|
||||
return all + "; ..."
|
||||
}
|
||||
return all
|
||||
}
|
||||
|
||||
type ipamState struct {
|
||||
@@ -101,8 +124,10 @@ func newIPAMProvider(c IPAMConfig) (ipamProvider, error) {
|
||||
return nil, nil
|
||||
case "netbox":
|
||||
return newNetBox(c)
|
||||
case "infoblox":
|
||||
return newInfoblox(c)
|
||||
default:
|
||||
return nil, fmt.Errorf("ipam provider %q: unknown (netbox)", c.Provider)
|
||||
return nil, fmt.Errorf("ipam provider %q: unknown (netbox, infoblox)", c.Provider)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -119,6 +144,20 @@ func orDefault(s, d string) string {
|
||||
|
||||
var dnsLabel = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?$`)
|
||||
|
||||
// validDomain: lower-case dot-separated DNS labels, no trailing dot. Names end
|
||||
// up in dnsmasq lines, cloud-init, PXE menus and DNS updates, so nothing else.
|
||||
func validDomain(s string) bool {
|
||||
if s == "" || len(s) > 253 {
|
||||
return false
|
||||
}
|
||||
for _, l := range strings.Split(s, ".") {
|
||||
if !dnsLabel.MatchString(l) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// extFor maps a Rollcall address to the record the authority should hold.
|
||||
func (a *App) extFor(x IPAddr, sn *Subnet) extAddr {
|
||||
bits := 32
|
||||
@@ -152,8 +191,11 @@ func (a *App) extFor(x IPAddr, sn *Subnet) extAddr {
|
||||
// extDNSName: the machine's main address gets <host>.<zone>; addresses on
|
||||
// other subnets (storage, IB, a tagged VLAN) get <host>-<iface>.<zone>. A
|
||||
// subnet's domain replaces the zone when it lies inside it.
|
||||
func (a *App) extDNSName(x IPAddr, sn *Subnet) string {
|
||||
zone := strings.TrimSuffix(strings.ToLower(a.cfg.IPAM.DNSZone), ".")
|
||||
func (a *App) extDNSName(x IPAddr, sn *Subnet) string { return dnsNameIn(x, sn, a.cfg.IPAM.DNSZone) }
|
||||
|
||||
// dnsNameIn is the naming rule shared by the IPAM authority and the DNS provider.
|
||||
func dnsNameIn(x IPAddr, sn *Subnet, zone string) string {
|
||||
zone = strings.TrimSuffix(strings.ToLower(zone), ".")
|
||||
if sn != nil && sn.Domain != "" { // a subnet domain counts only inside the authority's zone; "pxe.local" stays local
|
||||
d := strings.TrimSuffix(strings.ToLower(sn.Domain), ".")
|
||||
if zone == "" || d == zone || strings.HasSuffix(d, "."+zone) {
|
||||
@@ -165,6 +207,9 @@ func (a *App) extDNSName(x IPAddr, sn *Subnet) string {
|
||||
return ""
|
||||
}
|
||||
if strings.HasSuffix(host, "."+zone) {
|
||||
if !validDomain(host) {
|
||||
return "" // a wildcard or anything that is not a plain name never reaches the DNS
|
||||
}
|
||||
return host
|
||||
}
|
||||
if !dnsLabel.MatchString(host) {
|
||||
@@ -183,6 +228,12 @@ func (a *App) extDNSName(x IPAddr, sn *Subnet) string {
|
||||
// claimIP records an address in the authority (when one is configured) and
|
||||
// then locally; false when it is taken in either.
|
||||
func (a *App) claimIP(ctx context.Context, x IPAddr) (bool, error) {
|
||||
if a.ipam != nil && a.ipamSt != nil {
|
||||
// not while a reconcile runs: it read the address table before this
|
||||
// claim and would delete the record Reserve is about to make
|
||||
a.ipamSt.mu.Lock()
|
||||
defer a.ipamSt.mu.Unlock()
|
||||
}
|
||||
if a.ipam != nil {
|
||||
sn := subnetByRef(a.subnetsSnapshot(), x.Subnet)
|
||||
taken, err := a.ipam.Reserve(ctx, a.extFor(x, sn))
|
||||
@@ -194,6 +245,10 @@ func (a *App) claimIP(ctx context.Context, x IPAddr) (bool, error) {
|
||||
}
|
||||
}
|
||||
ok, err := a.store.ClaimIP(ctx, x)
|
||||
if ok {
|
||||
a.kickDNS()
|
||||
a.kickDHCPRes()
|
||||
}
|
||||
if a.ipam != nil && (err != nil || !ok) {
|
||||
a.kickIPAM() // the record we just made is reconciled away or re-owned
|
||||
}
|
||||
@@ -201,6 +256,8 @@ func (a *App) claimIP(ctx context.Context, x IPAddr) (bool, error) {
|
||||
}
|
||||
|
||||
func (a *App) kickIPAM() {
|
||||
a.kickDNS() // the same address changes move DNS records
|
||||
a.kickDHCPRes()
|
||||
if a.ipamSt == nil {
|
||||
return
|
||||
}
|
||||
@@ -217,6 +274,9 @@ func (a *App) startIPAM() error {
|
||||
}
|
||||
a.ipam = p
|
||||
a.ipamSt = &ipamState{kick: make(chan struct{}, 1)}
|
||||
if _, nb := p.(*netBox); a.cfg.IPAM.DeviceSync != nil && !nb {
|
||||
log.Printf("ipam: device_sync needs the netbox provider; ignored for %s", p.Name())
|
||||
}
|
||||
every := 5 * time.Minute
|
||||
if d, err := time.ParseDuration(a.cfg.IPAM.SyncInterval); err == nil && d >= 10*time.Second {
|
||||
every = d
|
||||
@@ -231,6 +291,9 @@ func (a *App) startIPAM() error {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Minute)
|
||||
rep := a.reconcileIPAM(ctx, false)
|
||||
cancel()
|
||||
if d := rep.Devices; d != nil && (d.Error != "" || len(d.Created)+len(d.Deleted)+len(d.Conflicts) > 0) {
|
||||
log.Printf("ipam %s devices: created %v deleted %v conflicts %v %s", p.Name(), d.Created, d.Deleted, d.Conflicts, d.Error)
|
||||
}
|
||||
if rep.Error != "" {
|
||||
log.Printf("ipam %s: %s", p.Name(), rep.Error)
|
||||
} else if n := len(rep.Created) + len(rep.Updated) + len(rep.Deleted) + len(rep.Conflicts); n > 0 {
|
||||
@@ -264,6 +327,14 @@ func (a *App) reconcileIPAM(ctx context.Context, dry bool) ipamReport {
|
||||
return rep
|
||||
}
|
||||
a.reconcileAddrs(ctx, local, &rep)
|
||||
if nb, ok := a.ipam.(*netBox); ok && a.cfg.IPAM.DeviceSync != nil && !dry {
|
||||
// after the addresses: the sync assigns them to the interfaces
|
||||
if ms, err := a.store.List(ctx); err != nil {
|
||||
rep.Devices = &devReport{Error: err.Error()}
|
||||
} else {
|
||||
rep.Devices = nb.syncDevices(ctx, *a.cfg.IPAM.DeviceSync, ms, local)
|
||||
}
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
@@ -284,13 +355,16 @@ func (a *App) reconcileAddrs(ctx context.Context, local []IPAddr, rep *ipamRepor
|
||||
want := a.extFor(x, subnetByRef(subnets, x.Subnet))
|
||||
if o, ok := have[x.IP]; ok {
|
||||
delete(have, x.IP)
|
||||
if o.DNSName != want.DNSName || o.Machine != want.Machine || o.Status != want.Status || o.Bits != want.Bits || o.Desc != want.Desc {
|
||||
// an authority that does not keep a field answers it empty (Infoblox: bits, status)
|
||||
if o.DNSName != want.DNSName || o.Machine != want.Machine || (o.Status != "" && o.Status != want.Status) ||
|
||||
(o.Bits != 0 && o.Bits != want.Bits) || o.Desc != want.Desc {
|
||||
want.ID = o.ID
|
||||
rep.Updated = append(rep.Updated, x.IP)
|
||||
if !dry {
|
||||
if err := a.ipam.Update(ctx, want); err != nil {
|
||||
rep.Error = fmt.Sprintf("update %s: %v", x.IP, err)
|
||||
return
|
||||
rep.fail(fmt.Sprintf("update %s: %v", x.IP, err))
|
||||
rep.Updated = rep.Updated[:len(rep.Updated)-1]
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -302,8 +376,8 @@ func (a *App) reconcileAddrs(ctx context.Context, local []IPAddr, rep *ipamRepor
|
||||
}
|
||||
taken, err := a.ipam.Reserve(ctx, want)
|
||||
if err != nil {
|
||||
rep.Error = fmt.Sprintf("create %s: %v", x.IP, err)
|
||||
return
|
||||
rep.fail(fmt.Sprintf("create %s: %v", x.IP, err))
|
||||
continue
|
||||
}
|
||||
if taken {
|
||||
rep.Conflicts = append(rep.Conflicts, x.IP+" exists there and is not Rollcall's")
|
||||
@@ -320,8 +394,9 @@ func (a *App) reconcileAddrs(ctx context.Context, local []IPAddr, rep *ipamRepor
|
||||
rep.Deleted = append(rep.Deleted, o.IP)
|
||||
if !dry {
|
||||
if err := a.ipam.Delete(ctx, o); err != nil {
|
||||
rep.Error = fmt.Sprintf("delete %s: %v", o.IP, err)
|
||||
return
|
||||
rep.fail(fmt.Sprintf("delete %s: %v", o.IP, err))
|
||||
rep.Deleted = rep.Deleted[:len(rep.Deleted)-1]
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -415,20 +490,30 @@ func newNetBox(c IPAMConfig) (*netBox, error) {
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("netbox token: %w", err)
|
||||
}
|
||||
hc, err := httpClientCA(c.CAFile, 30*time.Second)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("netbox ca_file: %w", err)
|
||||
}
|
||||
return &netBox{base: strings.TrimRight(c.URL, "/"), token: strings.TrimSpace(string(tok)),
|
||||
tag: c.tag(), field: c.field(), client: hc}, nil
|
||||
}
|
||||
|
||||
// httpClientCA is an HTTP client that trusts caFile (PEM) instead of the
|
||||
// system's CAs when it is given.
|
||||
func httpClientCA(caFile string, timeout time.Duration) (*http.Client, error) {
|
||||
tr := http.DefaultTransport.(*http.Transport).Clone()
|
||||
if c.CAFile != "" {
|
||||
pem, err := os.ReadFile(c.CAFile)
|
||||
if caFile != "" {
|
||||
pem, err := os.ReadFile(caFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("netbox ca_file: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
if !pool.AppendCertsFromPEM(pem) {
|
||||
return nil, fmt.Errorf("netbox ca_file %s: no certificates", c.CAFile)
|
||||
return nil, fmt.Errorf("%s: no certificates", caFile)
|
||||
}
|
||||
tr.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12}
|
||||
}
|
||||
return &netBox{base: strings.TrimRight(c.URL, "/"), token: strings.TrimSpace(string(tok)),
|
||||
tag: c.tag(), field: c.field(), client: &http.Client{Transport: tr, Timeout: 30 * time.Second}}, nil
|
||||
return &http.Client{Transport: tr, Timeout: timeout}, nil
|
||||
}
|
||||
|
||||
func (n *netBox) Name() string { return "netbox" }
|
||||
|
||||
+130
-4
@@ -11,6 +11,7 @@ import (
|
||||
"log"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// bmcCreds is what a power driver needs.
|
||||
@@ -68,7 +69,73 @@ func (a *App) power(ctx context.Context, m *Machine, op, arg string) (string, er
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: %v", errNoBMC, err)
|
||||
}
|
||||
return a.powerVia(ctx, a.rackOf(m), c, op, arg)
|
||||
if c, err = a.pinBMC(ctx, m, c); err != nil {
|
||||
return "", err
|
||||
}
|
||||
out, err := a.powerVia(ctx, a.rackOf(m), c, op, arg)
|
||||
a.noteCertChange(ctx, m, err)
|
||||
return out, err
|
||||
}
|
||||
|
||||
// noteCertChange puts a refused BMC certificate on the machine's events.
|
||||
func (a *App) noteCertChange(ctx context.Context, m *Machine, err error) {
|
||||
if err != nil && strings.Contains(err.Error(), "certificate changed") {
|
||||
_ = a.store.AddEvent(ctx, m.ID, "bmc certificate changed", oneLine(err.Error()))
|
||||
}
|
||||
}
|
||||
|
||||
// pinBMC records the certificate of an HTTPS BMC (Redfish, Proxmox) before
|
||||
// any credentials go to it: trust on first use. From then on every connection
|
||||
// must show the same certificate (bmcHTTPClient). An address change or a BMC
|
||||
// firmware update through Rollcall clears the pin; so does tls_fingerprint=.
|
||||
func (a *App) pinBMC(ctx context.Context, m *Machine, c *bmcCreds) (*bmcCreds, error) {
|
||||
d, _ := driverByName(c.Driver)
|
||||
if (d.Family != "redfish" && d.Family != "api") || c.Extra["tls_fingerprint"] != "" || c.Extra["verify_tls"] == "true" {
|
||||
return c, nil
|
||||
}
|
||||
fp, err := a.powerVia(ctx, a.rackOf(m), c, "tls-fingerprint", "")
|
||||
if err != nil {
|
||||
if strings.Contains(err.Error(), "unknown power operation") { // a rack controller not updated yet
|
||||
return c, nil
|
||||
}
|
||||
return nil, fmt.Errorf("reading the BMC's certificate: %w", err)
|
||||
}
|
||||
fp = strings.TrimSpace(fp)
|
||||
if err := a.setBMCExtra(ctx, m.ID, "tls_fingerprint", fp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = a.store.AddEvent(ctx, m.ID, "bmc certificate pinned", fp)
|
||||
c2 := *c
|
||||
c2.Extra = map[string]string{}
|
||||
for k, v := range c.Extra {
|
||||
c2.Extra[k] = v
|
||||
}
|
||||
c2.Extra["tls_fingerprint"] = fp
|
||||
return &c2, nil
|
||||
}
|
||||
|
||||
// setBMCExtra sets (or with "" removes) one setting on a machine's BMC record.
|
||||
func (a *App) setBMCExtra(ctx context.Context, id, k, v string) error {
|
||||
m, err := a.store.Get(ctx, id)
|
||||
if err != nil || m.BMC == nil {
|
||||
return err
|
||||
}
|
||||
var b obj
|
||||
if err := json.Unmarshal(m.BMC, &b); err != nil {
|
||||
return err
|
||||
}
|
||||
ex, _ := b["extra"].(map[string]any)
|
||||
if ex == nil {
|
||||
ex = map[string]any{}
|
||||
}
|
||||
if v == "" {
|
||||
delete(ex, k)
|
||||
} else {
|
||||
ex[k] = v
|
||||
}
|
||||
b["extra"] = ex
|
||||
nb, _ := json.Marshal(b)
|
||||
return a.store.SetBMC(ctx, id, nb, nil)
|
||||
}
|
||||
|
||||
func (a *App) powerVia(ctx context.Context, rack string, c *bmcCreds, op, arg string) (string, error) {
|
||||
@@ -83,6 +150,27 @@ func (a *App) powerVia(ctx context.Context, rack string, c *bmcCreds, op, arg st
|
||||
Password: c.Password, Extra: c.Extra, Op: op, Arg: arg})
|
||||
}
|
||||
|
||||
// firmwareVia runs a firmware operation (fw-*) where the BMC is reachable.
|
||||
func (a *App) firmwareVia(ctx context.Context, m *Machine, op, arg string) (string, error) {
|
||||
c, err := a.credsFor(m)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%w: %v", errNoBMC, err)
|
||||
}
|
||||
if c, err = a.pinBMC(ctx, m, c); err != nil {
|
||||
return "", err
|
||||
}
|
||||
rack := a.rackOf(m)
|
||||
if rack == a.cfg.RackName || rack == "" {
|
||||
return powerRun(ctx, a.cfg.IPMITool, c, op, arg)
|
||||
}
|
||||
r, ok := a.racksSnapshot()[rack]
|
||||
if !ok || !rackOnline(r, a.cfg.RackName) {
|
||||
return "", fmt.Errorf("rack controller %s is offline", rack)
|
||||
}
|
||||
return a.hub.dispatch(ctx, rack, &rackJob{Kind: "power", Driver: c.Driver, Address: c.Address, Port: c.Port, User: c.User,
|
||||
Password: c.Password, Extra: c.Extra, Op: op, Arg: arg, Timeout: int((fwTimeouts[op] + time.Minute).Seconds())})
|
||||
}
|
||||
|
||||
func (s *Store) SetBMC(ctx context.Context, id string, bmc []byte, secret []byte) error {
|
||||
if secret != nil {
|
||||
_, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2, bmc_secret=$3 WHERE id=$1`, id, bmc, secret)
|
||||
@@ -95,6 +183,27 @@ func (s *Store) SetBMC(ctx context.Context, id string, bmc []byte, secret []byte
|
||||
// probeBMC: after commissioning, a BMC still on plain IPMI is asked whether it
|
||||
// speaks Redfish with the same account; if it does, the vendor's driver takes over.
|
||||
// An operator's explicit choice (driver_set) is left alone.
|
||||
// verifyHintedBMC runs a status call against a BMC from bmc_hints (the agent no
|
||||
// longer gets its password) and records whether it answered.
|
||||
func (a *App) verifyHintedBMC(m *Machine) {
|
||||
var b obj
|
||||
if m.BMC == nil || json.Unmarshal(m.BMC, &b) != nil || str(b["source"]) != "hint" || b["driver_set"] == true {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second)
|
||||
defer cancel()
|
||||
out, err := a.power(ctx, m, "status", "")
|
||||
b["verified_oob"] = err == nil
|
||||
if err != nil {
|
||||
b["errors"] = []string{"hinted BMC did not answer: " + oneLine(err.Error())}
|
||||
} else {
|
||||
delete(b, "errors")
|
||||
}
|
||||
nb, _ := json.Marshal(b)
|
||||
_ = a.store.SetBMC(ctx, m.ID, nb, nil)
|
||||
_ = a.store.AddEvent(ctx, m.ID, "bmc hint", fmt.Sprintf("%s: %s", str(b["address"]), oneLine(orDefault(out, fmt.Sprint(err)))))
|
||||
}
|
||||
|
||||
func (a *App) probeBMC(m *Machine) {
|
||||
if m.BMC == nil || m.bmcSecret == nil {
|
||||
return
|
||||
@@ -112,6 +221,12 @@ func (a *App) probeBMC(m *Machine) {
|
||||
}
|
||||
probe := *c
|
||||
probe.Driver, probe.Port = "redfish", 0
|
||||
pinned, err := a.pinBMC(context.Background(), m, &probe)
|
||||
if err != nil {
|
||||
_ = a.store.AddEvent(context.Background(), m.ID, "bmc probe", "staying on ipmi: "+oneLine(err.Error()))
|
||||
return
|
||||
}
|
||||
probe = *pinned
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 90e9)
|
||||
defer cancel()
|
||||
out, err := a.powerVia(ctx, a.rackOf(m), &probe, "probe", "")
|
||||
@@ -174,9 +289,15 @@ func (a *App) handleBMC(w http.ResponseWriter, r *http.Request) {
|
||||
b["driver"] = "ipmi"
|
||||
nb, _ := json.Marshal(b)
|
||||
_ = a.store.SetBMC(r.Context(), m.ID, nb, nil)
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
if m, err = a.store.Get(r.Context(), m.ID); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
a.probeBMC(m)
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
if m, err = a.store.Get(r.Context(), m.ID); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
_ = json.Unmarshal(m.BMC, &b)
|
||||
writeJSON(w, 200, obj{"id": m.ID, "bmc": b})
|
||||
return
|
||||
@@ -189,7 +310,12 @@ func (a *App) handleBMC(w http.ResponseWriter, r *http.Request) {
|
||||
b["driver"] = *req.Driver
|
||||
}
|
||||
if req.Address != nil {
|
||||
b["address"] = strings.TrimSpace(*req.Address)
|
||||
if a := strings.TrimSpace(*req.Address); a != str(b["address"]) {
|
||||
if ex, _ := b["extra"].(map[string]any); ex != nil {
|
||||
delete(ex, "tls_fingerprint") // another BMC: its certificate is pinned anew
|
||||
}
|
||||
b["address"] = a
|
||||
}
|
||||
}
|
||||
if req.Port != nil {
|
||||
b["port"] = *req.Port
|
||||
|
||||
@@ -136,6 +136,7 @@ func (a *App) handleLocation(w http.ResponseWriter, r *http.Request) {
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "location", l.String())
|
||||
m, _ = a.store.Get(r.Context(), m.ID)
|
||||
a.retag(r.Context(), []*Machine{m}) // rules may look at the rack
|
||||
a.kickIPAM() // the NetBox device follows (ipam.device_sync)
|
||||
writeJSON(w, 200, obj{"id": m.ID, "location": m.Location})
|
||||
}
|
||||
|
||||
@@ -302,6 +303,7 @@ func (a *App) handleLocationImport(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
if !dry {
|
||||
a.retag(r.Context(), nil)
|
||||
a.kickIPAM()
|
||||
}
|
||||
if matched == nil {
|
||||
matched = []obj{}
|
||||
|
||||
+19
-3
@@ -49,6 +49,9 @@ func usage() {
|
||||
rollcall unassign ID
|
||||
rollcall overlay ID [-stage system|runtime] [-extract DIR]
|
||||
rollcall render PROFILE PATH [-stage system|runtime]
|
||||
rollcall reservations [status] | sync [-dry] fixed leases on the site's DHCP server (Kea, OpenWrt) for proxy-mode VLANs
|
||||
rollcall firmware list | import FILE|URL -vendor V -component C -version X [-models M,..] [-sha256 S] | rm PKG
|
||||
rollcall firmware show MACHINE [-refresh] | update MACHINE|tag:T PKG [-apply OnReset|Immediate] [-reboot] [-method M] [-force] [-watch]
|
||||
rollcall os list | sync [NAME...] [-force] | import NAME FILE [-family F] [-version V] | promote NAME VERSION | rm NAME VERSION
|
||||
rollcall deploy ID OS [-version V] [-disk /dev/sdX] [-hostname H] [-upgrade] [-user-data FILE] [-boot] [-watch]
|
||||
rollcall release ID [-boot] [-watch]
|
||||
@@ -60,12 +63,13 @@ func usage() {
|
||||
rollcall location racks | rack NAME -building B -room R [-height 42]
|
||||
rollcall list [-tag T] [-building B] [-room R] [-rack K]
|
||||
rollcall fabric list | add NAME [-kind ethernet|infiniband] | rm NAME
|
||||
rollcall vlan list | add FABRIC VID [-dhcp] [-rack R] [-secondary R] [-mtu N] [-name N] | rm FABRIC VID
|
||||
rollcall vlan list | add FABRIC VID [-dhcp | -proxy-dhcp] [-rack R] [-secondary R] [-mtu N] [-name N] | rm FABRIC VID
|
||||
rollcall subnet list | show CIDR | add CIDR -fabric F [-vid N] [-purpose P] [-gateway G] [-dns a,b] [-domain D] | rm CIDR [-force]
|
||||
rollcall range add CIDR dynamic|reserved|static START END | rm ID
|
||||
rollcall ip list [-subnet CIDR] | reserve IP [-name N] | release IP
|
||||
rollcall admin-cert request NAME | list | revoke NAME|SERIAL admin client certificate (browser/CLI without the token)
|
||||
rollcall ipam [status] | sync [-dry] | prefixes | import external IPAM (NetBox): state, reconcile, delegated prefixes
|
||||
rollcall dns [status] | sync [-dry] DNS records written directly (PowerDNS API / RFC 2136)
|
||||
rollcall ipam [status] | sync [-dry] | prefixes | import external IPAM (NetBox, Infoblox): state, reconcile, delegated prefixes
|
||||
rollcall layout list | show NAME | add NAME FILE.json | rm NAME
|
||||
rollcall net show ID | set ID [-layout L] [-rack R] [-static IFACE=IP] [-unstatic IP]
|
||||
rollcall controllers | controller rm NAME | controller join-code [-name N] [-ttl 60] [-addr HOST:443] | dhcp CONTROLLER
|
||||
@@ -97,6 +101,13 @@ func main() {
|
||||
cmd, args := os.Args[1], os.Args[2:]
|
||||
switch cmd {
|
||||
case "rack":
|
||||
if len(args) > 0 && args[0] == "renew" {
|
||||
if err := rackRenew(args[1:]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
return
|
||||
}
|
||||
if len(args) > 0 && args[0] == "join" {
|
||||
if err := rackJoin(args[1:]); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
@@ -141,11 +152,16 @@ func main() {
|
||||
case "list", "show", "events", "power", "bootdev", "recommission", "delete",
|
||||
"images", "profile", "assign", "unassign", "overlay", "render",
|
||||
"deploy", "release", "jobs", "job", "tag", "tags", "location",
|
||||
"bmc", "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp", "ipam", "admin-cert":
|
||||
"bmc", "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp", "ipam", "dns", "reservations", "admin-cert":
|
||||
if err := runClient(cmd, args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case "firmware":
|
||||
if err := firmwareCmd(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
os.Exit(1)
|
||||
}
|
||||
case "os":
|
||||
if err := osCmd(args); err != nil {
|
||||
fmt.Fprintln(os.Stderr, "error:", err)
|
||||
|
||||
@@ -0,0 +1,465 @@
|
||||
package main
|
||||
|
||||
// NetBox device sync (ipam.device_sync): each machine becomes a dcim device
|
||||
// with its serial, interfaces (MACs as dcim mac-addresses) and location, and
|
||||
// Rollcall's IP addresses in NetBox are assigned to those interfaces (the
|
||||
// management address becomes the device's primary IP).
|
||||
//
|
||||
// Sites, locations and racks are NetBox's: Rollcall matches the machine's
|
||||
// building, room and rack to them by name and never creates them (a machine
|
||||
// whose building matches no site goes to device_sync.site). Manufacturers,
|
||||
// device types and the role are created when missing, since a device needs
|
||||
// them. Devices carry the tag and the rollcall_machine custom field like
|
||||
// addresses; devices without them are never changed or deleted, and one
|
||||
// holding a machine's name is a conflict.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type DeviceSyncConfig struct {
|
||||
Site string `json:"site"` // site slug for machines whose building matches no site (required)
|
||||
Role string `json:"role"` // device role slug, default "server" (created when missing)
|
||||
}
|
||||
|
||||
type devReport struct {
|
||||
Created []string `json:"created"`
|
||||
Updated []string `json:"updated"`
|
||||
Deleted []string `json:"deleted"`
|
||||
Conflicts []string `json:"conflicts"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
func (r *devReport) fail(e string) { r.Error = joinErr(r.Error, e) }
|
||||
|
||||
var nbSlugRe = regexp.MustCompile(`[^a-z0-9]+`)
|
||||
|
||||
func nbSlug(s string) string {
|
||||
s = strings.Trim(nbSlugRe.ReplaceAllString(strings.ToLower(s), "-"), "-")
|
||||
if len(s) > 100 {
|
||||
s = s[:100]
|
||||
}
|
||||
return orDefault(s, "unknown")
|
||||
}
|
||||
|
||||
// all reads every object of a list endpoint.
|
||||
func (n *netBox) all(ctx context.Context, path string, q url.Values) ([]obj, error) {
|
||||
if q == nil {
|
||||
q = url.Values{}
|
||||
}
|
||||
q.Set("limit", "500")
|
||||
next := path + "?" + q.Encode()
|
||||
var out []obj
|
||||
for next != "" {
|
||||
var p nbPage[obj]
|
||||
if err := n.do(ctx, "GET", next, nil, &p); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, p.Results...)
|
||||
next = ""
|
||||
if p.Next != nil {
|
||||
next = *p.Next
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func nbID(o obj) int64 {
|
||||
f, _ := o["id"].(float64)
|
||||
return int64(f)
|
||||
}
|
||||
|
||||
// nbRef is the id of a nested object field ({"id": 3, ...}), 0 when unset.
|
||||
func nbRef(o obj, field string) int64 {
|
||||
if m, ok := o[field].(map[string]any); ok {
|
||||
return nbID(m)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
// mine is the machine a NetBox object belongs to, "" when it is not Rollcall's.
|
||||
func (n *netBox) mine(o obj) string { return n.owner(nbIPFromObj(o)) }
|
||||
|
||||
// ensure finds an object by a query or creates it.
|
||||
func (n *netBox) ensure(ctx context.Context, path string, q url.Values, create obj) (int64, error) {
|
||||
got, err := n.all(ctx, path, q)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if len(got) > 0 {
|
||||
return nbID(got[0]), nil
|
||||
}
|
||||
var made obj
|
||||
if err := n.do(ctx, "POST", path, create, &made); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
return nbID(made), nil
|
||||
}
|
||||
|
||||
func nbStatus(state string) string {
|
||||
switch {
|
||||
case state == "deployed" || state == "booted" || state == "stateless" || state == "deploying" || state == "releasing":
|
||||
return "active"
|
||||
case strings.HasSuffix(state, "-failed"):
|
||||
return "failed"
|
||||
case state == "new":
|
||||
return "planned"
|
||||
}
|
||||
return "staged" // commissioned, ready to deploy
|
||||
}
|
||||
|
||||
func nbIfaceType(n invNIC) string {
|
||||
switch n.Speed {
|
||||
case 1000:
|
||||
return "1000base-t"
|
||||
case 10000:
|
||||
return "10gbase-x-sfpp"
|
||||
case 25000:
|
||||
return "25gbase-x-sfp28"
|
||||
case 40000:
|
||||
return "40gbase-x-qsfpp"
|
||||
case 100000:
|
||||
return "100gbase-x-qsfp28"
|
||||
case 200000:
|
||||
return "200gbase-x-qsfp56"
|
||||
case 400000:
|
||||
return "400gbase-x-qsfpdd"
|
||||
}
|
||||
return "other"
|
||||
}
|
||||
|
||||
// syncDevices brings NetBox's devices in line with the machines.
|
||||
func (n *netBox) syncDevices(ctx context.Context, cfg DeviceSyncConfig, machines []*Machine, ips []IPAddr) *devReport {
|
||||
rep := &devReport{Created: []string{}, Updated: []string{}, Deleted: []string{}, Conflicts: []string{}}
|
||||
if cfg.Site == "" {
|
||||
rep.Error = "ipam.device_sync.site (a site slug) is required"
|
||||
return rep
|
||||
}
|
||||
sites, err := n.all(ctx, "dcim/sites/", nil)
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
siteBy := map[string]int64{}
|
||||
for _, s := range sites {
|
||||
siteBy[strings.ToLower(str(s["slug"]))] = nbID(s)
|
||||
siteBy[strings.ToLower(str(s["name"]))] = nbID(s)
|
||||
}
|
||||
defSite, ok := siteBy[strings.ToLower(cfg.Site)]
|
||||
if !ok {
|
||||
rep.Error = fmt.Sprintf("NetBox has no site %q (ipam.device_sync.site)", cfg.Site)
|
||||
return rep
|
||||
}
|
||||
type siteName struct {
|
||||
site int64
|
||||
name string
|
||||
}
|
||||
locBy, rackBy := map[siteName]int64{}, map[siteName]int64{}
|
||||
if locs, err := n.all(ctx, "dcim/locations/", nil); err == nil {
|
||||
for _, l := range locs {
|
||||
locBy[siteName{nbRef(l, "site"), strings.ToLower(str(l["name"]))}] = nbID(l)
|
||||
}
|
||||
}
|
||||
if racks, err := n.all(ctx, "dcim/racks/", nil); err == nil {
|
||||
for _, r := range racks {
|
||||
rackBy[siteName{nbRef(r, "site"), strings.ToLower(str(r["name"]))}] = nbID(r)
|
||||
}
|
||||
}
|
||||
tags := []obj{{"name": n.tag}}
|
||||
role := orDefault(cfg.Role, "server")
|
||||
roleID, err := n.ensure(ctx, "dcim/device-roles/", url.Values{"slug": {nbSlug(role)}},
|
||||
obj{"name": strings.ToUpper(role[:1]) + role[1:], "slug": nbSlug(role), "color": "2196f3", "tags": tags})
|
||||
if err != nil {
|
||||
rep.Error = "device role: " + err.Error()
|
||||
return rep
|
||||
}
|
||||
owned := map[string]obj{}
|
||||
devs, err := n.all(ctx, "dcim/devices/", url.Values{"tag": {n.tag}})
|
||||
if err != nil {
|
||||
rep.Error = err.Error()
|
||||
return rep
|
||||
}
|
||||
for _, d := range devs {
|
||||
if m := n.mine(d); m != "" {
|
||||
owned[m] = d
|
||||
}
|
||||
}
|
||||
byMachine := map[string][]IPAddr{}
|
||||
for _, x := range ips {
|
||||
if x.MachineID != "" {
|
||||
byMachine[x.MachineID] = append(byMachine[x.MachineID], x)
|
||||
}
|
||||
}
|
||||
nbIPs := map[string]obj{} // Rollcall's addresses in NetBox, by IP
|
||||
if all, err := n.all(ctx, "ipam/ip-addresses/", url.Values{"tag": {n.tag}}); err == nil {
|
||||
for _, a := range all {
|
||||
if p, ok := parseAddr(str(a["address"])); ok {
|
||||
nbIPs[p.String()] = a
|
||||
}
|
||||
}
|
||||
}
|
||||
types := map[string]int64{} // manufacturer|model -> device type id
|
||||
sort.Slice(machines, func(i, j int) bool { return machines[i].ID < machines[j].ID })
|
||||
for _, m := range machines {
|
||||
if m.State == "new" {
|
||||
continue
|
||||
}
|
||||
name := orDefault(deref(m.NodeName), m.ID)
|
||||
mfr, model := orDefault(deref(m.Manufacturer), "Unknown"), orDefault(deref(m.Product), "Unknown")
|
||||
tk := mfr + "|" + model
|
||||
if _, ok := types[tk]; !ok {
|
||||
mid, err := n.ensure(ctx, "dcim/manufacturers/", url.Values{"slug": {nbSlug(mfr)}}, obj{"name": mfr, "slug": nbSlug(mfr), "tags": tags})
|
||||
if err != nil {
|
||||
rep.fail(fmt.Sprintf("%s: manufacturer: %v", name, err))
|
||||
continue
|
||||
}
|
||||
tid, err := n.ensure(ctx, "dcim/device-types/", url.Values{"manufacturer_id": {strconv.FormatInt(mid, 10)}, "slug": {nbSlug(model)}},
|
||||
obj{"manufacturer": mid, "model": model, "slug": nbSlug(model), "u_height": 1, "tags": tags})
|
||||
if err != nil {
|
||||
rep.fail(fmt.Sprintf("%s: device type: %v", name, err))
|
||||
continue
|
||||
}
|
||||
types[tk] = tid
|
||||
}
|
||||
site := defSite
|
||||
if id, ok := siteBy[strings.ToLower(m.Location.Building)]; ok && m.Location.Building != "" {
|
||||
site = id
|
||||
}
|
||||
body := obj{"name": name, "device_type": types[tk], "role": roleID, "site": site, "status": nbStatus(m.State),
|
||||
"serial": trunc(dmiValue(deref(m.Serial)), 50), "tags": tags, "custom_fields": obj{n.field: m.ID},
|
||||
"location": nil, "rack": nil, "position": nil, "face": nil}
|
||||
if id, ok := locBy[siteName{site, strings.ToLower(m.Location.Room)}]; ok && m.Location.Room != "" {
|
||||
body["location"] = id
|
||||
}
|
||||
if id, ok := rackBy[siteName{site, strings.ToLower(m.Location.Rack)}]; ok && m.Location.Rack != "" {
|
||||
body["rack"] = id
|
||||
if m.Location.Unit > 0 {
|
||||
body["position"], body["face"] = m.Location.Unit, "front"
|
||||
}
|
||||
}
|
||||
if m.Location.AssetTag != "" {
|
||||
body["asset_tag"] = m.Location.AssetTag
|
||||
}
|
||||
var dev obj
|
||||
if d, ok := owned[m.ID]; ok {
|
||||
delete(owned, m.ID)
|
||||
if deviceDiffers(d, body) {
|
||||
if err := n.do(ctx, "PATCH", fmt.Sprintf("dcim/devices/%d/", nbID(d)), body, &dev); err != nil {
|
||||
rep.fail(fmt.Sprintf("%s: %v", name, err))
|
||||
continue
|
||||
}
|
||||
rep.Updated = append(rep.Updated, name)
|
||||
} else {
|
||||
dev = d
|
||||
}
|
||||
} else {
|
||||
if same, err := n.all(ctx, "dcim/devices/", url.Values{"name": {name}, "site_id": {strconv.FormatInt(site, 10)}}); err != nil {
|
||||
rep.fail(fmt.Sprintf("%s: %v", name, err))
|
||||
continue
|
||||
} else if len(same) > 0 {
|
||||
rep.Conflicts = append(rep.Conflicts, name+": NetBox has a device of that name (not Rollcall's)")
|
||||
continue
|
||||
}
|
||||
if err := n.do(ctx, "POST", "dcim/devices/", body, &dev); err != nil {
|
||||
rep.fail(fmt.Sprintf("%s: %v", name, err))
|
||||
continue
|
||||
}
|
||||
rep.Created = append(rep.Created, name)
|
||||
}
|
||||
if err := n.syncInterfaces(ctx, dev, m, byMachine[m.ID], nbIPs, tags); err != nil {
|
||||
rep.fail(fmt.Sprintf("%s interfaces: %v", name, err))
|
||||
}
|
||||
}
|
||||
gone := make([]string, 0, len(owned))
|
||||
for k := range owned {
|
||||
gone = append(gone, k)
|
||||
}
|
||||
sort.Strings(gone)
|
||||
for _, k := range gone { // machines no longer in Rollcall
|
||||
d := owned[k]
|
||||
if err := n.do(ctx, "DELETE", fmt.Sprintf("dcim/devices/%d/", nbID(d)), nil, nil); err != nil {
|
||||
rep.fail(fmt.Sprintf("delete %s: %v", str(d["name"]), err))
|
||||
continue
|
||||
}
|
||||
rep.Deleted = append(rep.Deleted, str(d["name"]))
|
||||
}
|
||||
return rep
|
||||
}
|
||||
|
||||
// deviceDiffers compares what Rollcall sets with what NetBox holds.
|
||||
func deviceDiffers(d, want obj) bool {
|
||||
ref := func(f string) int64 { return nbRef(d, f) }
|
||||
num := func(v any) int64 {
|
||||
switch x := v.(type) {
|
||||
case int64:
|
||||
return x
|
||||
case int:
|
||||
return int64(x)
|
||||
case float64:
|
||||
return int64(x)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
if str(d["name"]) != str(want["name"]) || str(d["serial"]) != str(want["serial"]) || str(asObj(d["status"])["value"]) != str(want["status"]) {
|
||||
return true
|
||||
}
|
||||
for _, f := range []string{"device_type", "role", "site", "location", "rack"} {
|
||||
if ref(f) != num(want[f]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
if num(d["position"]) != num(want["position"]) {
|
||||
return true
|
||||
}
|
||||
return str(d["asset_tag"]) != str(want["asset_tag"])
|
||||
}
|
||||
|
||||
// syncInterfaces keeps a device's interfaces, MACs and assigned addresses.
|
||||
func (n *netBox) syncInterfaces(ctx context.Context, dev obj, m *Machine, ips []IPAddr, nbIPs map[string]obj, tags []obj) error {
|
||||
devID := nbID(dev)
|
||||
have := map[string]obj{}
|
||||
got, err := n.all(ctx, "dcim/interfaces/", url.Values{"device_id": {strconv.FormatInt(devID, 10)}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, i := range got {
|
||||
have[str(i["name"])] = i
|
||||
}
|
||||
ifaceID := map[string]int64{}
|
||||
nics := machineNICs(m)
|
||||
for _, nic := range nics {
|
||||
cur, ok := have[nic.Name]
|
||||
delete(have, nic.Name)
|
||||
if !ok {
|
||||
if err := n.do(ctx, "POST", "dcim/interfaces/", obj{"device": devID, "name": nic.Name, "type": nbIfaceType(nic), "tags": tags}, &cur); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
ifaceID[nic.Name] = nbID(cur)
|
||||
if nic.MAC == "" {
|
||||
continue
|
||||
}
|
||||
if pm := asObj(cur["primary_mac_address"]); strings.EqualFold(str(pm["mac_address"]), nic.MAC) {
|
||||
continue
|
||||
}
|
||||
macs, err := n.all(ctx, "dcim/mac-addresses/", url.Values{"interface_id": {strconv.FormatInt(nbID(cur), 10)}})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var macID int64
|
||||
for _, x := range macs {
|
||||
if strings.EqualFold(str(x["mac_address"]), nic.MAC) {
|
||||
macID = nbID(x)
|
||||
}
|
||||
}
|
||||
if macID == 0 {
|
||||
var made obj
|
||||
if err := n.do(ctx, "POST", "dcim/mac-addresses/", obj{"mac_address": nic.MAC, "assigned_object_type": "dcim.interface", "assigned_object_id": nbID(cur), "tags": tags}, &made); err != nil {
|
||||
return err
|
||||
}
|
||||
macID = nbID(made)
|
||||
}
|
||||
if err := n.do(ctx, "PATCH", fmt.Sprintf("dcim/interfaces/%d/", nbID(cur)), obj{"primary_mac_address": macID}, nil); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for name, i := range have { // interfaces Rollcall made that the machine no longer has
|
||||
if n.tagged(i) {
|
||||
if err := n.do(ctx, "DELETE", fmt.Sprintf("dcim/interfaces/%d/", nbID(i)), nil, nil); err != nil {
|
||||
return fmt.Errorf("delete interface %s: %w", name, err)
|
||||
}
|
||||
}
|
||||
}
|
||||
// addresses on their interfaces; the primary IP of each family is the
|
||||
// management address, else the first address on the management (or
|
||||
// boot) interface
|
||||
var boot string
|
||||
if b := bootNIC(nics, m.MACs); b != nil {
|
||||
boot = b.Name
|
||||
}
|
||||
mgmt := deref(m.MgmtIP)
|
||||
type cand struct {
|
||||
id, iface int64
|
||||
fam int
|
||||
}
|
||||
var cands []cand
|
||||
primary := map[int]int64{}
|
||||
mgmtIface := ifaceID[boot]
|
||||
for _, x := range ips {
|
||||
a, ok := nbIPs[x.IP]
|
||||
if !ok || n.mine(a) != m.ID {
|
||||
continue
|
||||
}
|
||||
iid, ok := ifaceID[orDefault(x.Iface, boot)]
|
||||
if !ok && x.IP == mgmt {
|
||||
iid, ok = ifaceID[boot]
|
||||
}
|
||||
if !ok {
|
||||
continue // a bond or VLAN interface: not in the inventory
|
||||
}
|
||||
if str(a["assigned_object_type"]) != "dcim.interface" || int64(num(a["assigned_object_id"])) != iid {
|
||||
if err := n.do(ctx, "PATCH", fmt.Sprintf("ipam/ip-addresses/%d/", nbID(a)), obj{"assigned_object_type": "dcim.interface", "assigned_object_id": iid}, nil); err != nil {
|
||||
return fmt.Errorf("assign %s: %w", x.IP, err)
|
||||
}
|
||||
}
|
||||
fam := 4
|
||||
if strings.Contains(x.IP, ":") {
|
||||
fam = 6
|
||||
}
|
||||
if x.IP == mgmt {
|
||||
primary[fam], mgmtIface = nbID(a), iid
|
||||
}
|
||||
cands = append(cands, cand{nbID(a), iid, fam})
|
||||
}
|
||||
for _, c := range cands {
|
||||
if _, set := primary[c.fam]; !set && c.iface == mgmtIface {
|
||||
primary[c.fam] = c.id
|
||||
}
|
||||
}
|
||||
body := obj{}
|
||||
if id, ok := primary[4]; ok && nbRef(dev, "primary_ip4") != id {
|
||||
body["primary_ip4"] = id
|
||||
}
|
||||
if id, ok := primary[6]; ok && nbRef(dev, "primary_ip6") != id {
|
||||
body["primary_ip6"] = id
|
||||
}
|
||||
if len(body) > 0 {
|
||||
if err := n.do(ctx, "PATCH", fmt.Sprintf("dcim/devices/%d/", devID), body, nil); err != nil {
|
||||
return fmt.Errorf("primary ip: %w", err)
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// tagged: an object Rollcall created (it carries the tag).
|
||||
func (n *netBox) tagged(o obj) bool {
|
||||
for _, t := range toList(o["tags"]) {
|
||||
if str(asObj(t)["name"]) == n.tag {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func nbIPFromObj(a obj) nbIP {
|
||||
x := nbIP{CustomFields: asObj(a["custom_fields"])}
|
||||
for _, t := range toList(a["tags"]) {
|
||||
x.Tags = append(x.Tags, struct{ Name string }{str(asObj(t)["name"])})
|
||||
}
|
||||
return x
|
||||
}
|
||||
|
||||
// dmiValue drops the placeholders firmware leaves in DMI fields.
|
||||
func dmiValue(s string) string {
|
||||
switch strings.ToLower(strings.TrimSpace(s)) {
|
||||
case "", "not specified", "to be filled by o.e.m.", "default string", "system serial number", "0123456789", "none", "n/a":
|
||||
return ""
|
||||
}
|
||||
return strings.TrimSpace(s)
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// fakeDCIM keeps NetBox collections as plain objects; references are stored
|
||||
// as ids and served nested ({"id": n}), as NetBox does.
|
||||
type fakeDCIM struct {
|
||||
mu sync.Mutex
|
||||
next int64
|
||||
objs map[string]map[int64]map[string]any // collection path -> id -> object
|
||||
reqs []string
|
||||
}
|
||||
|
||||
var nbNested = []string{"site", "location", "rack", "role", "device_type", "manufacturer", "device", "primary_ip4", "primary_ip6", "primary_mac_address"}
|
||||
|
||||
func newFakeDCIM() *fakeDCIM { return &fakeDCIM{next: 1, objs: map[string]map[int64]map[string]any{}} }
|
||||
|
||||
func (f *fakeDCIM) put(coll string, o map[string]any) int64 {
|
||||
if f.objs[coll] == nil {
|
||||
f.objs[coll] = map[int64]map[string]any{}
|
||||
}
|
||||
id := f.next
|
||||
f.next++
|
||||
o["id"] = float64(id)
|
||||
if s, ok := o["status"].(string); ok {
|
||||
o["status"] = map[string]any{"value": s}
|
||||
}
|
||||
f.objs[coll][id] = o
|
||||
return id
|
||||
}
|
||||
|
||||
func (f *fakeDCIM) out(coll string, o map[string]any) map[string]any {
|
||||
c := map[string]any{}
|
||||
for k, v := range o {
|
||||
c[k] = v
|
||||
}
|
||||
for _, k := range nbNested {
|
||||
if id, ok := tnum(c[k]); ok && id > 0 {
|
||||
n := map[string]any{"id": float64(id)}
|
||||
if k == "primary_mac_address" {
|
||||
n["mac_address"] = f.objs["dcim/mac-addresses/"][id]["mac_address"]
|
||||
}
|
||||
c[k] = n
|
||||
}
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
func tnum(v any) (int64, bool) {
|
||||
switch x := v.(type) {
|
||||
case float64:
|
||||
return int64(x), true
|
||||
case int64:
|
||||
return x, true
|
||||
case int:
|
||||
return int64(x), true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func (f *fakeDCIM) match(o map[string]any, k, want string) bool {
|
||||
switch k {
|
||||
case "limit":
|
||||
return true
|
||||
case "tag":
|
||||
for _, t := range toList(o["tags"]) {
|
||||
if asObj(t)["name"] == want {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
case "interface_id":
|
||||
k = "assigned_object_id"
|
||||
}
|
||||
if strings.HasSuffix(k, "_id") && k != "assigned_object_id" {
|
||||
k = strings.TrimSuffix(k, "_id")
|
||||
}
|
||||
if id, ok := tnum(o[k]); ok {
|
||||
return fmt.Sprint(id) == want
|
||||
}
|
||||
return fmt.Sprint(o[k]) == want
|
||||
}
|
||||
|
||||
func (f *fakeDCIM) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.reqs = append(f.reqs, r.Method+" "+r.URL.Path)
|
||||
write := func(code int, v any) { w.WriteHeader(code); _ = json.NewEncoder(w).Encode(v) }
|
||||
p := strings.TrimPrefix(r.URL.Path, "/api/")
|
||||
var coll string
|
||||
var id int64
|
||||
if i := strings.LastIndex(strings.TrimSuffix(p, "/"), "/"); i > 0 {
|
||||
if _, err := fmt.Sscanf(p[i+1:], "%d/", &id); err == nil {
|
||||
coll = p[:i+1]
|
||||
}
|
||||
}
|
||||
var body map[string]any
|
||||
if r.Body != nil {
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
}
|
||||
if coll == "" {
|
||||
coll = p
|
||||
switch r.Method {
|
||||
case "GET":
|
||||
ids := []int64{}
|
||||
for i := range f.objs[coll] {
|
||||
ids = append(ids, i)
|
||||
}
|
||||
sort.Slice(ids, func(i, j int) bool { return ids[i] < ids[j] })
|
||||
res := []any{}
|
||||
next:
|
||||
for _, i := range ids {
|
||||
o := f.objs[coll][i]
|
||||
for k, v := range r.URL.Query() {
|
||||
if !f.match(o, k, v[0]) {
|
||||
continue next
|
||||
}
|
||||
}
|
||||
res = append(res, f.out(coll, o))
|
||||
}
|
||||
write(200, map[string]any{"next": nil, "results": res})
|
||||
case "POST":
|
||||
if coll == "dcim/devices/" {
|
||||
for _, o := range f.objs[coll] {
|
||||
if o["name"] == body["name"] && fmt.Sprint(o["site"]) == fmt.Sprint(body["site"]) {
|
||||
write(400, map[string]any{"__all__": []string{"Device name must be unique per site."}})
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
id := f.put(coll, body)
|
||||
write(201, f.out(coll, f.objs[coll][id]))
|
||||
}
|
||||
return
|
||||
}
|
||||
o, ok := f.objs[coll][id]
|
||||
if !ok {
|
||||
write(404, map[string]any{"detail": "not found"})
|
||||
return
|
||||
}
|
||||
switch r.Method {
|
||||
case "PATCH":
|
||||
for k, v := range body {
|
||||
o[k] = v
|
||||
}
|
||||
if s, ok := o["status"].(string); ok {
|
||||
o["status"] = map[string]any{"value": s}
|
||||
}
|
||||
write(200, f.out(coll, o))
|
||||
case "DELETE":
|
||||
delete(f.objs[coll], id)
|
||||
if coll == "dcim/devices/" { // interfaces go with the device
|
||||
for i, x := range f.objs["dcim/interfaces/"] {
|
||||
if n, _ := tnum(x["device"]); n == id {
|
||||
delete(f.objs["dcim/interfaces/"], i)
|
||||
}
|
||||
}
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
default:
|
||||
write(200, f.out(coll, o))
|
||||
}
|
||||
}
|
||||
|
||||
func (f *fakeDCIM) find(coll, k, v string) map[string]any {
|
||||
for _, o := range f.objs[coll] {
|
||||
if fmt.Sprint(o[k]) == v {
|
||||
return o
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (f *fakeDCIM) count(coll string) int { return len(f.objs[coll]) }
|
||||
|
||||
func testDCIM(t *testing.T) (*netBox, *fakeDCIM) {
|
||||
t.Helper()
|
||||
f := newFakeDCIM()
|
||||
srv := httptest.NewServer(f)
|
||||
t.Cleanup(srv.Close)
|
||||
tok := filepath.Join(t.TempDir(), "token")
|
||||
if err := os.WriteFile(tok, []byte("nbt_k.secret\n"), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p, err := newIPAMProvider(IPAMConfig{Provider: "netbox", URL: srv.URL, TokenFile: tok})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ev := f.put("dcim/sites/", map[string]any{"name": "Ev", "slug": "ev"})
|
||||
dc := f.put("dcim/sites/", map[string]any{"name": "DC1", "slug": "dc1"})
|
||||
f.put("dcim/locations/", map[string]any{"name": "Hall A", "site": float64(dc)})
|
||||
f.put("dcim/racks/", map[string]any{"name": "R12", "site": float64(dc)})
|
||||
f.put("dcim/racks/", map[string]any{"name": "R12", "site": float64(ev)}) // same name, other site
|
||||
return p.(*netBox), f
|
||||
}
|
||||
|
||||
func dcimMachine(id, name, state string, loc Location, nics ...[2]string) *Machine {
|
||||
var net []obj
|
||||
for _, n := range nics {
|
||||
net = append(net, obj{"name": n[0], "mac": n[1], "speed": "25000Mb/s"})
|
||||
}
|
||||
inv, _ := json.Marshal(obj{"network": net})
|
||||
s := func(v string) *string { return &v }
|
||||
m := &Machine{ID: id, State: state, Serial: s("SN-" + id), Manufacturer: s("Dell Inc."), Product: s("PowerEdge R760"),
|
||||
Inventory: inv, Location: loc}
|
||||
if name != "" {
|
||||
m.NodeName = s(name)
|
||||
}
|
||||
if len(nics) > 0 {
|
||||
m.MACs = []string{nics[0][1]}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
func TestNetBoxDeviceSync(t *testing.T) {
|
||||
nb, f := testDCIM(t)
|
||||
ctx := context.Background()
|
||||
cfg := DeviceSyncConfig{Site: "ev"}
|
||||
tagged := []any{map[string]any{"name": "rollcall"}}
|
||||
ipID := f.put("ipam/ip-addresses/", map[string]any{"address": "10.213.77.50/24", "tags": tagged,
|
||||
"custom_fields": map[string]any{"rollcall_machine": "m-a"}})
|
||||
ip6ID := f.put("ipam/ip-addresses/", map[string]any{"address": "fd00:213:77::50/64", "tags": tagged,
|
||||
"custom_fields": map[string]any{"rollcall_machine": "m-a"}})
|
||||
foreignIP := f.put("ipam/ip-addresses/", map[string]any{"address": "10.213.77.51/24", "tags": []any{},
|
||||
"custom_fields": map[string]any{}})
|
||||
// a device someone else made with a machine's name
|
||||
f.put("dcim/devices/", map[string]any{"name": "taken", "site": float64(1), "tags": []any{}, "custom_fields": map[string]any{}})
|
||||
|
||||
a := dcimMachine("m-a", "node001", "deployed", Location{Building: "DC1", Room: "hall a", Rack: "R12", Unit: 10, AssetTag: "AT-1"},
|
||||
[2]string{"eno1", "aa:bb:cc:00:00:01"}, [2]string{"eno2", "aa:bb:cc:00:00:02"})
|
||||
b := dcimMachine("m-b", "", "ready", Location{Building: "nowhere", Rack: "R12"}, [2]string{"eth0", "aa:bb:cc:00:00:03"})
|
||||
c := dcimMachine("m-c", "taken", "commission-failed", Location{})
|
||||
n := dcimMachine("m-n", "", "new", Location{})
|
||||
ips := []IPAddr{{IP: "10.213.77.50", MachineID: "m-a", Iface: "eno1"}, {IP: "fd00:213:77::50", MachineID: "m-a", Iface: "eno1"},
|
||||
{IP: "10.213.77.51", MachineID: "m-b"}}
|
||||
mgmt := "10.213.77.50"
|
||||
a.MgmtIP = &mgmt
|
||||
ns := "Not Specified"
|
||||
b.Serial = &ns
|
||||
|
||||
rep := nb.syncDevices(ctx, cfg, []*Machine{a, b, c, n}, ips)
|
||||
if rep.Error != "" {
|
||||
t.Fatal(rep.Error)
|
||||
}
|
||||
if fmt.Sprint(rep.Created) != "[node001 m-b]" || len(rep.Conflicts) != 1 || !strings.Contains(rep.Conflicts[0], "taken") {
|
||||
t.Fatalf("report %+v", rep)
|
||||
}
|
||||
d := f.find("dcim/devices/", "name", "node001")
|
||||
dc := f.find("dcim/sites/", "slug", "dc1")["id"]
|
||||
hall := f.find("dcim/locations/", "name", "Hall A")["id"]
|
||||
if d["site"] != dc || d["location"] != hall || d["position"] != float64(10) || d["asset_tag"] != "AT-1" ||
|
||||
d["serial"] != "SN-m-a" || asObj(d["status"])["value"] != "active" || asObj(d["custom_fields"])["rollcall_machine"] != "m-a" {
|
||||
t.Fatalf("device %v", d)
|
||||
}
|
||||
if r := f.objs["dcim/racks/"]; r[int64(num(d["rack"]))]["site"] != dc {
|
||||
t.Fatalf("rack from another site: %v", d["rack"])
|
||||
}
|
||||
db := f.find("dcim/devices/", "name", "m-b")
|
||||
if num(db["site"]) != 1 || num(db["rack"]) != 5 || asObj(db["status"])["value"] != "staged" {
|
||||
t.Fatalf("default-site device %v", db) // unknown building: the default site, and its own R12
|
||||
}
|
||||
if f.find("dcim/devices/", "name", "m-n") != nil {
|
||||
t.Fatal("a new machine (not commissioned) was synced")
|
||||
}
|
||||
if f.count("dcim/manufacturers/") != 1 || f.count("dcim/device-types/") != 1 || f.count("dcim/device-roles/") != 1 {
|
||||
t.Fatal("types not shared")
|
||||
}
|
||||
// interfaces with primary MACs, the address on eno1 and as primary IP
|
||||
eno1 := f.find("dcim/interfaces/", "name", "eno1")
|
||||
if eno1["type"] != "25gbase-x-sfp28" {
|
||||
t.Fatalf("iface %v", eno1)
|
||||
}
|
||||
mac := f.objs["dcim/mac-addresses/"][int64(num(eno1["primary_mac_address"]))]
|
||||
if mac["mac_address"] != "aa:bb:cc:00:00:01" {
|
||||
t.Fatalf("primary mac %v", eno1["primary_mac_address"])
|
||||
}
|
||||
ip := f.objs["ipam/ip-addresses/"][ipID]
|
||||
if ip["assigned_object_type"] != "dcim.interface" || ip["assigned_object_id"] != eno1["id"] || d["primary_ip4"] != float64(ipID) {
|
||||
t.Fatalf("ip %v primary %v", ip, d["primary_ip4"])
|
||||
}
|
||||
if d["primary_ip6"] != float64(ip6ID) {
|
||||
t.Fatalf("primary ip6 %v (the v6 address on the management interface)", d["primary_ip6"])
|
||||
}
|
||||
if db["serial"] != "" {
|
||||
t.Fatalf("DMI placeholder as serial: %v", db["serial"])
|
||||
}
|
||||
if f.objs["ipam/ip-addresses/"][foreignIP]["assigned_object_id"] != nil {
|
||||
t.Fatal("assigned an address that is not Rollcall's")
|
||||
}
|
||||
|
||||
// a second pass changes nothing
|
||||
f.reqs = nil
|
||||
rep = nb.syncDevices(ctx, cfg, []*Machine{a, b, c, n}, ips)
|
||||
for _, r := range f.reqs {
|
||||
if !strings.HasPrefix(r, "GET ") {
|
||||
t.Fatalf("second pass wrote: %v (%+v)", f.reqs, rep)
|
||||
}
|
||||
}
|
||||
|
||||
// moves, a lost NIC, a removed machine
|
||||
a.State = "releasing"
|
||||
a.Location.Unit = 20
|
||||
a.Inventory, _ = json.Marshal(obj{"network": []obj{{"name": "eno1", "mac": "aa:bb:cc:00:00:01", "speed": "25000Mb/s"}}})
|
||||
rep = nb.syncDevices(ctx, cfg, []*Machine{a, c}, ips)
|
||||
if fmt.Sprint(rep.Updated) != "[node001]" || fmt.Sprint(rep.Deleted) != "[m-b]" || rep.Error != "" {
|
||||
t.Fatalf("report %+v", rep)
|
||||
}
|
||||
if d["position"] != float64(20) || f.find("dcim/interfaces/", "name", "eno2") != nil || f.find("dcim/devices/", "name", "taken") == nil {
|
||||
t.Fatalf("after update: %v", d)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNetBoxDeviceSyncSite(t *testing.T) {
|
||||
nb, _ := testDCIM(t)
|
||||
if rep := nb.syncDevices(context.Background(), DeviceSyncConfig{}, nil, nil); !strings.Contains(rep.Error, "required") {
|
||||
t.Fatal(rep.Error)
|
||||
}
|
||||
if rep := nb.syncDevices(context.Background(), DeviceSyncConfig{Site: "mars"}, nil, nil); !strings.Contains(rep.Error, "no site") {
|
||||
t.Fatal(rep.Error)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNBSlug(t *testing.T) {
|
||||
for in, want := range map[string]string{"Dell Inc.": "dell-inc", "PowerEdge R760": "poweredge-r760", " ": "unknown", "xFusion/2288H V7": "xfusion-2288h-v7"} {
|
||||
if got := nbSlug(in); got != want {
|
||||
t.Errorf("%q: %q", in, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
+178
-8
@@ -8,6 +8,7 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"os"
|
||||
"strconv"
|
||||
"strings"
|
||||
@@ -71,6 +72,12 @@ func (a *App) handleVLANs(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 400, errors.New("fabric and vid (0 untagged, 1-4094, or an IB pkey) are required"))
|
||||
return
|
||||
}
|
||||
if v.DHCPMode == "proxy" {
|
||||
v.DHCP = true
|
||||
} else if v.DHCPMode != "" && v.DHCPMode != "serve" {
|
||||
fail(w, 400, fmt.Errorf("dhcp_mode %q: serve or proxy", v.DHCPMode))
|
||||
return
|
||||
}
|
||||
if v.DHCP && v.PrimaryRack == "" {
|
||||
v.PrimaryRack = a.cfg.RackName
|
||||
}
|
||||
@@ -128,8 +135,17 @@ func (a *App) handleSubnets(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
}
|
||||
if sn.Gateway != "" && !sn.contains(sn.Gateway) {
|
||||
fail(w, 400, fmt.Errorf("gateway %s is outside %s", sn.Gateway, sn.CIDR))
|
||||
if sn.Gateway != "" {
|
||||
gw, err := netip.ParseAddr(sn.Gateway)
|
||||
if err != nil || !sn.contains(gw.String()) {
|
||||
fail(w, 400, fmt.Errorf("gateway %q is not an address in %s", sn.Gateway, sn.CIDR))
|
||||
return
|
||||
}
|
||||
sn.Gateway = gw.String()
|
||||
}
|
||||
sn.Domain = strings.ToLower(strings.TrimSuffix(sn.Domain, "."))
|
||||
if sn.Domain != "" && !validDomain(sn.Domain) {
|
||||
fail(w, 400, fmt.Errorf("domain %q is not a DNS name", sn.Domain))
|
||||
return
|
||||
}
|
||||
switch sn.Purpose {
|
||||
@@ -259,6 +275,16 @@ func (a *App) handleIPs(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
ip, ok := parseAddr(x.IP)
|
||||
if !ok || strings.Contains(x.IP, "/") {
|
||||
fail(w, 400, fmt.Errorf("%q is not an address", x.IP))
|
||||
return
|
||||
}
|
||||
x.IP = ip.String() // one spelling: the table's key, NetBox's and DNS's
|
||||
if strings.IndexFunc(x.Hostname, func(c rune) bool { return c < ' ' || c == 0x7f }) >= 0 || len(x.Hostname) > 200 {
|
||||
fail(w, 400, errors.New("hostname: one line of text"))
|
||||
return
|
||||
}
|
||||
sn := subnetFor(a.subnetsSnapshot(), x.IP)
|
||||
if sn == nil {
|
||||
fail(w, 404, fmt.Errorf("%s is in no subnet", x.IP))
|
||||
@@ -511,9 +537,9 @@ func (c *client) netCmd(cmd string, args []string) error {
|
||||
if err := c.do("GET", "/api/v1/vlans", nil, &vs); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("%-14s %5s %-14s %5s %-5s %-14s %s\n", "FABRIC", "VID", "NAME", "MTU", "DHCP", "RACK", "SECONDARY")
|
||||
fmt.Printf("%-14s %5s %-14s %5s %-6s %-14s %s\n", "FABRIC", "VID", "NAME", "MTU", "DHCP", "RACK", "SECONDARY")
|
||||
for _, v := range vs {
|
||||
fmt.Printf("%-14s %5d %-14s %5d %-5v %-14s %s\n", v.Fabric, v.VID, v.Name, v.MTU, v.DHCP, v.PrimaryRack, v.SecondaryRack)
|
||||
fmt.Printf("%-14s %5d %-14s %5d %-6s %-14s %s\n", v.Fabric, v.VID, v.Name, v.MTU, orDash(v.DHCPMode), v.PrimaryRack, v.SecondaryRack)
|
||||
}
|
||||
case "add", "set":
|
||||
need(3)
|
||||
@@ -521,15 +547,20 @@ func (c *client) netCmd(cmd string, args []string) error {
|
||||
name := fs.String("name", "", "name")
|
||||
mtu := fs.Int("mtu", 1500, "MTU")
|
||||
dhcp := fs.Bool("dhcp", false, "serve DHCP/PXE on this VLAN")
|
||||
proxy := fs.Bool("proxy-dhcp", false, "another DHCP server hands out addresses here; only answer PXE (proxy DHCP)")
|
||||
rack := fs.String("rack", "", "rack controller serving it (default: the region's embedded one)")
|
||||
sec := fs.String("secondary", "", "second rack controller (the dynamic range is split)")
|
||||
_ = fs.Parse(args[3:])
|
||||
vid, _ := strconv.Atoi(args[2])
|
||||
var v VLAN
|
||||
if err := c.do("POST", "/api/v1/vlans", VLAN{Fabric: args[1], VID: vid, Name: *name, MTU: *mtu, DHCP: *dhcp, PrimaryRack: *rack, SecondaryRack: *sec}, &v); err != nil {
|
||||
mode := ""
|
||||
if *proxy {
|
||||
mode = "proxy"
|
||||
}
|
||||
if err := c.do("POST", "/api/v1/vlans", VLAN{Fabric: args[1], VID: vid, Name: *name, MTU: *mtu, DHCP: *dhcp || *proxy, DHCPMode: mode, PrimaryRack: *rack, SecondaryRack: *sec}, &v); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("vlan %s/%d: dhcp=%v rack=%s\n", v.Fabric, v.VID, v.DHCP, v.PrimaryRack)
|
||||
fmt.Printf("vlan %s/%d: dhcp=%s rack=%s\n", v.Fabric, v.VID, orDash(v.DHCPMode), v.PrimaryRack)
|
||||
case "rm":
|
||||
need(3)
|
||||
return c.do("DELETE", "/api/v1/vlans/"+args[1]+"/"+args[2], nil, nil)
|
||||
@@ -792,6 +823,68 @@ func (c *client) netCmd(cmd string, args []string) error {
|
||||
os.Stdout.Write(data)
|
||||
case "ipam":
|
||||
return c.ipamCmd(sub, args)
|
||||
case "dns":
|
||||
return c.dnsCmd(sub, args)
|
||||
case "reservations":
|
||||
return c.reservationsCmd(sub, args)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *client) reservationsCmd(sub string, args []string) error {
|
||||
show := func(rep dhcpResReport) {
|
||||
for _, x := range []struct {
|
||||
what string
|
||||
list []string
|
||||
}{{"put", rep.Put}, {"delete", rep.Deleted}, {"conflict", rep.Conflicts}, {"skipped", rep.Skipped}} {
|
||||
for _, v := range x.list {
|
||||
fmt.Printf("%-9s %s\n", x.what, v)
|
||||
}
|
||||
}
|
||||
dry := ""
|
||||
if rep.DryRun {
|
||||
dry = " (dry run, nothing changed)"
|
||||
}
|
||||
fmt.Printf("%s: %d put, %d deleted, %d conflicts%s\n", orDash(rep.Provider), len(rep.Put), len(rep.Deleted), len(rep.Conflicts), dry)
|
||||
if rep.Error != "" {
|
||||
fmt.Println("error:", rep.Error)
|
||||
}
|
||||
}
|
||||
switch sub {
|
||||
case "", "status":
|
||||
var st struct {
|
||||
Provider string `json:"provider"`
|
||||
URL string `json:"url"`
|
||||
Last dhcpResReport `json:"last_sync"`
|
||||
}
|
||||
if err := c.do("GET", "/api/v1/dhcp-reservations", nil, &st); err != nil {
|
||||
return err
|
||||
}
|
||||
if st.Provider == "" {
|
||||
fmt.Println("no DHCP reservation provider (dhcp_reservations.provider in config.json: kea or openwrt)")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("provider %s at %s\n", st.Provider, st.URL)
|
||||
if !st.Last.At.IsZero() {
|
||||
fmt.Printf("last sync %s\n", st.Last.At.Local().Format("2006-01-02 15:04:05"))
|
||||
show(st.Last)
|
||||
}
|
||||
case "sync":
|
||||
fs := newFlags("reservations sync")
|
||||
dry := fs.Bool("dry", false, "show what would change")
|
||||
_ = fs.Parse(args[min(1, len(args)):])
|
||||
q := ""
|
||||
if *dry {
|
||||
q = "?dry=1"
|
||||
}
|
||||
var rep dhcpResReport
|
||||
err := c.do("POST", "/api/v1/dhcp-reservations/sync"+q, nil, &rep)
|
||||
if rep.Provider != "" || rep.Error != "" {
|
||||
show(rep)
|
||||
}
|
||||
return err
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -815,6 +908,20 @@ func (c *client) ipamCmd(sub string, args []string) error {
|
||||
if rep.Error != "" {
|
||||
fmt.Println("error:", rep.Error)
|
||||
}
|
||||
if d := rep.Devices; d != nil {
|
||||
for _, x := range []struct {
|
||||
k string
|
||||
v []string
|
||||
}{{"created", d.Created}, {"updated", d.Updated}, {"deleted", d.Deleted}, {"conflict", d.Conflicts}} {
|
||||
for _, n := range x.v {
|
||||
fmt.Printf("%-9s device %s\n", x.k, n)
|
||||
}
|
||||
}
|
||||
fmt.Printf("devices: %d created, %d updated, %d deleted, %d conflicts\n", len(d.Created), len(d.Updated), len(d.Deleted), len(d.Conflicts))
|
||||
if d.Error != "" {
|
||||
fmt.Println("error:", d.Error)
|
||||
}
|
||||
}
|
||||
}
|
||||
switch sub {
|
||||
case "", "status":
|
||||
@@ -830,10 +937,14 @@ func (c *client) ipamCmd(sub string, args []string) error {
|
||||
return err
|
||||
}
|
||||
if st.Provider == "" {
|
||||
fmt.Println("built-in IPAM only (set ipam.provider in config.json for NetBox)")
|
||||
fmt.Println("built-in IPAM only (set ipam.provider in config.json: netbox or infoblox)")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("provider %s %s\nzone %s\nowner tag %q, custom field %q\n", st.Provider, st.URL, orDash(st.Zone), st.Tag, st.Field)
|
||||
owner := fmt.Sprintf("tag %q, custom field %q", st.Tag, st.Field)
|
||||
if st.Provider == "infoblox" {
|
||||
owner = fmt.Sprintf("extensible attribute %q (delegated networks: %q)", st.Field, st.Tag)
|
||||
}
|
||||
fmt.Printf("provider %s %s\nzone %s\nowner %s\n", st.Provider, st.URL, orDash(st.Zone), owner)
|
||||
if st.Last.At.IsZero() {
|
||||
fmt.Println("last sync never")
|
||||
return nil
|
||||
@@ -973,3 +1084,62 @@ func printNet(d obj) {
|
||||
b, _ := json.MarshalIndent(d["network_config"], " ", " ")
|
||||
fmt.Printf(" network-config:\n %s\n", b)
|
||||
}
|
||||
|
||||
func (c *client) dnsCmd(sub string, args []string) error {
|
||||
show := func(rep dnsReport) {
|
||||
for _, x := range rep.Upserted {
|
||||
fmt.Printf("%-9s %s\n", "write", x)
|
||||
}
|
||||
for _, x := range rep.Deleted {
|
||||
fmt.Printf("%-9s %s\n", "delete", x)
|
||||
}
|
||||
for _, x := range rep.Conflicts {
|
||||
fmt.Printf("%-9s %s\n", "conflict", x)
|
||||
}
|
||||
dry := ""
|
||||
if rep.DryRun {
|
||||
dry = " (dry run, nothing changed)"
|
||||
}
|
||||
fmt.Printf("%s %s: %d written, %d deleted, %d conflicts%s\n", orDash(rep.Provider), rep.Zone, len(rep.Upserted), len(rep.Deleted), len(rep.Conflicts), dry)
|
||||
if rep.Error != "" {
|
||||
fmt.Println("error:", rep.Error)
|
||||
}
|
||||
}
|
||||
switch sub {
|
||||
case "", "status":
|
||||
var st struct {
|
||||
Provider string `json:"provider"`
|
||||
Zone string `json:"zone"`
|
||||
Last dnsReport `json:"last_sync"`
|
||||
}
|
||||
if err := c.do("GET", "/api/v1/dns", nil, &st); err != nil {
|
||||
return err
|
||||
}
|
||||
if st.Provider == "" {
|
||||
fmt.Println("no DNS provider (dns.provider in config.json: powerdns or rfc2136)")
|
||||
return nil
|
||||
}
|
||||
fmt.Printf("provider %s, zone %s\n", st.Provider, st.Zone)
|
||||
if !st.Last.At.IsZero() {
|
||||
fmt.Printf("last sync %s\n", st.Last.At.Local().Format("2006-01-02 15:04:05"))
|
||||
show(st.Last)
|
||||
}
|
||||
case "sync":
|
||||
fs := newFlags("dns sync")
|
||||
dry := fs.Bool("dry", false, "show what would change")
|
||||
_ = fs.Parse(args[min(1, len(args)):])
|
||||
q := ""
|
||||
if *dry {
|
||||
q = "?dry=1"
|
||||
}
|
||||
var rep dnsReport
|
||||
err := c.do("POST", "/api/v1/dns/sync"+q, nil, &rep)
|
||||
if rep.Provider != "" || rep.Error != "" {
|
||||
show(rep)
|
||||
}
|
||||
return err
|
||||
default:
|
||||
usage()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
+102
-17
@@ -33,7 +33,8 @@ type VLAN struct {
|
||||
VID int `json:"vid"` // 0 = untagged; an InfiniBand partition key on IB fabrics
|
||||
Name string `json:"name"`
|
||||
MTU int `json:"mtu"`
|
||||
DHCP bool `json:"dhcp"` // rollcall serves DHCP/PXE on it
|
||||
DHCP bool `json:"dhcp"` // a rack controller answers DHCP/PXE on it
|
||||
DHCPMode string `json:"dhcp_mode"` // serve (addresses + PXE) | proxy (PXE only; another server hands out addresses)
|
||||
PrimaryRack string `json:"primary_rack"` // rack controller that serves it
|
||||
SecondaryRack string `json:"secondary_rack"` // second one; the dynamic range is split between them
|
||||
}
|
||||
@@ -55,6 +56,7 @@ type Subnet struct {
|
||||
VID int `json:"vid"`
|
||||
MTU int `json:"mtu"`
|
||||
DHCP bool `json:"dhcp"`
|
||||
DHCPMode string `json:"dhcp_mode"` // serve | proxy (from the VLAN)
|
||||
PrimaryRack string `json:"primary_rack"`
|
||||
SecondaryRack string `json:"secondary_rack"`
|
||||
Purpose string `json:"purpose"` // provisioning | mgmt | bmc | storage | ib | other
|
||||
@@ -140,7 +142,7 @@ func (s *Store) DeleteFabric(ctx context.Context, name string) error {
|
||||
}
|
||||
|
||||
func (s *Store) VLANs(ctx context.Context) ([]VLAN, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT id, fabric, vid, coalesce(name,''), mtu, dhcp, coalesce(primary_rack,''), coalesce(secondary_rack,'') FROM vlans ORDER BY fabric, vid`)
|
||||
rows, err := s.pool.Query(ctx, `SELECT id, fabric, vid, coalesce(name,''), mtu, dhcp, dhcp_mode, coalesce(primary_rack,''), coalesce(secondary_rack,'') FROM vlans ORDER BY fabric, vid`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -148,23 +150,35 @@ func (s *Store) VLANs(ctx context.Context) ([]VLAN, error) {
|
||||
out := []VLAN{}
|
||||
for rows.Next() {
|
||||
var v VLAN
|
||||
if err := rows.Scan(&v.ID, &v.Fabric, &v.VID, &v.Name, &v.MTU, &v.DHCP, &v.PrimaryRack, &v.SecondaryRack); err != nil {
|
||||
if err := rows.Scan(&v.ID, &v.Fabric, &v.VID, &v.Name, &v.MTU, &v.DHCP, &v.DHCPMode, &v.PrimaryRack, &v.SecondaryRack); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
v.DHCPMode = dhcpMode(v.DHCP, v.DHCPMode)
|
||||
out = append(out, v)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// dhcpMode normalises: no DHCP -> "", DHCP without a mode -> serve.
|
||||
func dhcpMode(dhcp bool, mode string) string {
|
||||
switch {
|
||||
case !dhcp:
|
||||
return ""
|
||||
case mode == "proxy":
|
||||
return "proxy"
|
||||
}
|
||||
return "serve"
|
||||
}
|
||||
|
||||
// PutVLAN upserts by (fabric, vid) and returns the id.
|
||||
func (s *Store) PutVLAN(ctx context.Context, v VLAN) (int64, error) {
|
||||
if v.MTU <= 0 {
|
||||
v.MTU = 1500
|
||||
}
|
||||
var id int64
|
||||
err := s.pool.QueryRow(ctx, `INSERT INTO vlans(fabric, vid, name, mtu, dhcp, primary_rack, secondary_rack) VALUES ($1,$2,$3,$4,$5,nullif($6,''),nullif($7,''))
|
||||
ON CONFLICT (fabric, vid) DO UPDATE SET name=$3, mtu=$4, dhcp=$5, primary_rack=nullif($6,''), secondary_rack=nullif($7,'') RETURNING id`,
|
||||
v.Fabric, v.VID, v.Name, v.MTU, v.DHCP, v.PrimaryRack, v.SecondaryRack).Scan(&id)
|
||||
err := s.pool.QueryRow(ctx, `INSERT INTO vlans(fabric, vid, name, mtu, dhcp, primary_rack, secondary_rack, dhcp_mode) VALUES ($1,$2,$3,$4,$5,nullif($6,''),nullif($7,''),$8)
|
||||
ON CONFLICT (fabric, vid) DO UPDATE SET name=$3, mtu=$4, dhcp=$5, primary_rack=nullif($6,''), secondary_rack=nullif($7,''), dhcp_mode=$8 RETURNING id`,
|
||||
v.Fabric, v.VID, v.Name, v.MTU, v.DHCP, v.PrimaryRack, v.SecondaryRack, dhcpMode(v.DHCP, v.DHCPMode)).Scan(&id)
|
||||
return id, err
|
||||
}
|
||||
|
||||
@@ -175,7 +189,7 @@ func (s *Store) DeleteVLAN(ctx context.Context, fabric string, vid int) error {
|
||||
|
||||
func (s *Store) Subnets(ctx context.Context) ([]Subnet, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT s.cidr, coalesce(s.name,''), coalesce(s.vlan_id,0), coalesce(v.fabric,''), coalesce(v.vid,0), coalesce(v.mtu,1500),
|
||||
coalesce(v.dhcp,false), coalesce(v.primary_rack,''), coalesce(v.secondary_rack,''), s.purpose, coalesce(s.gateway,''), s.dns,
|
||||
coalesce(v.dhcp,false), coalesce(v.dhcp_mode,''), coalesce(v.primary_rack,''), coalesce(v.secondary_rack,''), s.purpose, coalesce(s.gateway,''), s.dns,
|
||||
coalesce(s.domain,''), coalesce(s.comment,''), (SELECT count(*) FROM ip_addresses a WHERE a.subnet=s.cidr)
|
||||
FROM subnets s LEFT JOIN vlans v ON v.id = s.vlan_id ORDER BY s.cidr`)
|
||||
if err != nil {
|
||||
@@ -186,11 +200,12 @@ func (s *Store) Subnets(ctx context.Context) ([]Subnet, error) {
|
||||
for rows.Next() {
|
||||
var sn Subnet
|
||||
var dns []byte
|
||||
if err := rows.Scan(&sn.CIDR, &sn.Name, &sn.VLANID, &sn.Fabric, &sn.VID, &sn.MTU, &sn.DHCP, &sn.PrimaryRack, &sn.SecondaryRack,
|
||||
if err := rows.Scan(&sn.CIDR, &sn.Name, &sn.VLANID, &sn.Fabric, &sn.VID, &sn.MTU, &sn.DHCP, &sn.DHCPMode, &sn.PrimaryRack, &sn.SecondaryRack,
|
||||
&sn.Purpose, &sn.Gateway, &dns, &sn.Domain, &sn.Comment, &sn.Used); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = json.Unmarshal(dns, &sn.DNS)
|
||||
sn.DHCPMode = dhcpMode(sn.DHCP, sn.DHCPMode)
|
||||
if sn.DNS == nil {
|
||||
sn.DNS = []string{}
|
||||
}
|
||||
@@ -568,6 +583,8 @@ type invNIC struct {
|
||||
|
||||
var speedRe = regexp.MustCompile(`(\d+)`)
|
||||
|
||||
var ifnameRe = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,14}$`)
|
||||
|
||||
func machineNICs(m *Machine) []invNIC {
|
||||
var inv obj
|
||||
_ = json.Unmarshal(m.Inventory, &inv)
|
||||
@@ -581,7 +598,7 @@ func machineNICs(m *Machine) []invNIC {
|
||||
for _, ip := range toList(n["ipv4"]) {
|
||||
nic.IPv4 = append(nic.IPv4, str(ip))
|
||||
}
|
||||
if nic.Name != "" {
|
||||
if ifnameRe.MatchString(nic.Name) { // the machine reports it; it lands in udev rules and dnsmasq files
|
||||
out = append(out, nic)
|
||||
}
|
||||
}
|
||||
@@ -666,8 +683,8 @@ func selectNICs(nics []invNIC, boot *invNIC, sel string) []invNIC {
|
||||
}
|
||||
}
|
||||
sort.Slice(ib, func(i, j int) bool { return ib[i].Name < ib[j].Name })
|
||||
k, _ := strconv.Atoi(arg)
|
||||
if k < len(ib) {
|
||||
k, err := strconv.Atoi(arg)
|
||||
if err == nil && k >= 0 && k < len(ib) {
|
||||
return []invNIC{ib[k]}
|
||||
}
|
||||
default:
|
||||
@@ -719,6 +736,11 @@ type NetPlan struct {
|
||||
// VLAN (an IPv6 prefix next to the IPv4 one makes the machine dual-stack).
|
||||
func defaultLayout(bootSubnet *Subnet, subnets []Subnet) *NetLayout {
|
||||
links := []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}}
|
||||
if bootSubnet != nil && bootSubnet.DHCPMode == "proxy" && !hasStaticRange(*bootSubnet) {
|
||||
// the site's DHCP server owns the addresses; without a static range set aside
|
||||
// for Rollcall, machines keep getting theirs from it
|
||||
links = []LayoutLink{{Subnet: "boot", Mode: "dhcp"}}
|
||||
}
|
||||
if bootSubnet == nil {
|
||||
links = []LayoutLink{{Subnet: "boot", Mode: "dhcp"}}
|
||||
} else if bootSubnet.VLANID != 0 {
|
||||
@@ -1193,6 +1215,9 @@ func (a *App) netPlan(ctx context.Context, m *Machine, allocate bool) (*NetPlan,
|
||||
|
||||
// ---------------------------------------------------------------- dnsmasq
|
||||
|
||||
// uefiBoot is what UEFI machines load: signed shim, then signed GRUB (grub.go).
|
||||
const uefiBoot = "bootx64.efi"
|
||||
|
||||
const dnsmasqBase = `# generated by rollcall for rack controller %s; do not edit (rollcall subnet/vlan commands change it)
|
||||
bind-dynamic
|
||||
%sdomain-needed
|
||||
@@ -1201,11 +1226,6 @@ expand-hosts
|
||||
log-dhcp
|
||||
enable-tftp
|
||||
tftp-root=%s
|
||||
dhcp-match=set:efi64,option:client-arch,7
|
||||
dhcp-match=set:efi64,option:client-arch,9
|
||||
dhcp-match=set:bios,option:client-arch,0
|
||||
dhcp-boot=tag:bios,pxelinux.0
|
||||
dhcp-boot=tag:efi64,efi64/syslinux.efi
|
||||
dhcp-hostsfile=%s/hosts.dhcp
|
||||
addn-hosts=%s/hosts
|
||||
`
|
||||
@@ -1259,7 +1279,7 @@ func dnsmasqConfig(rc RackController, subnets []Subnet, dnsDir string) (string,
|
||||
shared := false
|
||||
domains := []string{}
|
||||
for _, sn := range served {
|
||||
shared = shared || sn.SecondaryRack != ""
|
||||
shared = shared || sn.SecondaryRack != "" || sn.DHCPMode == "proxy"
|
||||
if sn.Domain != "" && !hasTag(domains, sn.Domain) {
|
||||
domains = append(domains, sn.Domain)
|
||||
}
|
||||
@@ -1297,10 +1317,66 @@ func dnsmasqConfig(rc RackController, subnets []Subnet, dnsDir string) (string,
|
||||
for _, d := range domains {
|
||||
fmt.Fprintf(&b, "local=/%s/\n", d)
|
||||
}
|
||||
// architecture tags for the boot files of served subnets; left out when every
|
||||
// subnet is proxied (pxe-service picks by architecture itself, and these shared
|
||||
// tags are part of the cyclic tag list dnsmasq 2.91 builds in proxy mode)
|
||||
for _, sn := range served {
|
||||
if sn.DHCPMode != "proxy" && familyOf(sn.CIDR) == 4 {
|
||||
b.WriteString("dhcp-match=set:efi64,option:client-arch,7\ndhcp-match=set:efi64,option:client-arch,9\ndhcp-match=set:bios,option:client-arch,0\n")
|
||||
break
|
||||
}
|
||||
}
|
||||
pxeDone := map[string]bool{}
|
||||
for i, sn := range served {
|
||||
tag := fmt.Sprintf("s%d", i)
|
||||
v6 := familyOf(sn.CIDR) == 6
|
||||
if sn.DHCPMode == "proxy" {
|
||||
// another DHCP server hands out addresses (and router, DNS); this rack only
|
||||
// answers PXE clients with where to boot from. Proxy DHCP is IPv4 (PXE is).
|
||||
if v6 {
|
||||
continue
|
||||
}
|
||||
_, n, _ := net.ParseCIDR(sn.CIDR)
|
||||
fmt.Fprintf(&b, "\n# %s · %s vlan %d · %s · proxy DHCP (addresses from the site's DHCP server)\n", sn.CIDR, orDash(sn.Fabric), sn.VID, sn.Purpose)
|
||||
// No set: tag on a proxy range. dnsmasq (2.91, still in 2.93) appends a request's
|
||||
// tags to the context's persistent tag list in its proxy path without resetting it:
|
||||
// the next request's stack-allocated interface tag ends up pointing at itself and
|
||||
// dnsmasq loops forever (option_filter, rfc2131.c proxy branch). The PXE services
|
||||
// are limited by the interface tag instead, which is rebuilt for every request.
|
||||
fmt.Fprintf(&b, "dhcp-range=%s,proxy,%s\n", n.IP.String(), sn.netmask())
|
||||
var on []string
|
||||
for _, ri := range rc.Interfaces {
|
||||
for _, ad := range ri.Addrs {
|
||||
if sn.contains(ad) && !hasTag(on, ri.Name) {
|
||||
on = append(on, ri.Name)
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(on) == 0 {
|
||||
on = []string{""} // relayed: untagged
|
||||
}
|
||||
for _, i := range on {
|
||||
if pxeDone[i] {
|
||||
continue
|
||||
}
|
||||
pxeDone[i] = true
|
||||
t := ""
|
||||
if i != "" {
|
||||
t = "tag:" + i + ","
|
||||
}
|
||||
fmt.Fprintf(&b, "pxe-service=%sx86PC,\"Rollcall\",pxelinux\n", t)
|
||||
fmt.Fprintf(&b, "pxe-service=%sBC_EFI,\"Rollcall\",%s\n", t, uefiBoot)
|
||||
fmt.Fprintf(&b, "pxe-service=%sX86-64_EFI,\"Rollcall\",%s\n", t, uefiBoot)
|
||||
}
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(&b, "\n# %s · %s vlan %d · %s\n", sn.CIDR, orDash(sn.Fabric), sn.VID, sn.Purpose)
|
||||
if !v6 {
|
||||
// boot files per served subnet: dhcp-boot matching a proxy subnet's clients as
|
||||
// well as its pxe-service sends dnsmasq 2.91 into a busy loop (lab, 2026-10-29)
|
||||
fmt.Fprintf(&b, "dhcp-boot=tag:%s,tag:bios,pxelinux.0\n", tag)
|
||||
fmt.Fprintf(&b, "dhcp-boot=tag:%s,tag:efi64,%s\n", tag, uefiBoot)
|
||||
}
|
||||
if sn.Domain != "" {
|
||||
fmt.Fprintf(&b, "domain=%s,%s\n", sn.Domain, sn.CIDR)
|
||||
}
|
||||
@@ -1378,3 +1454,12 @@ func dnsmasqConfig(rc RackController, subnets []Subnet, dnsDir string) (string,
|
||||
}
|
||||
|
||||
var errSubnetUnknown = errors.New("subnet not found")
|
||||
|
||||
func hasStaticRange(sn Subnet) bool {
|
||||
for _, r := range sn.Ranges {
|
||||
if r.Kind == "static" {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -289,3 +289,69 @@ func TestStaleJobs(t *testing.T) {
|
||||
t.Errorf("stale jobs: %+v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyDHCP(t *testing.T) {
|
||||
sn := labSubnet()
|
||||
sn.DHCPMode = "proxy"
|
||||
sn.Ranges = nil
|
||||
rc := RackController{Name: "r1", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"10.0.0.2/24"}}}}
|
||||
conf, ok := dnsmasqConfig(rc, []Subnet{sn}, "/d")
|
||||
if !ok {
|
||||
t.Fatal("no config")
|
||||
}
|
||||
for _, want := range []string{"interface=eth1", "dhcp-range=10.0.0.0,proxy,255.255.255.0", `pxe-service=tag:eth1,x86PC,"Rollcall",pxelinux`, `pxe-service=tag:eth1,X86-64_EFI,"Rollcall",bootx64.efi`} {
|
||||
if !strings.Contains(conf, want) {
|
||||
t.Errorf("proxy config lacks %s:\n%s", want, conf)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"option:router", "option:dns-server", "dhcp-authoritative", ",12h", "set:s0,10.0.0.0,proxy"} {
|
||||
if strings.Contains(conf, bad) {
|
||||
t.Errorf("proxy config has %q (the site's server owns that):\n%s", bad, conf)
|
||||
}
|
||||
}
|
||||
// machines keep DHCP from the site's server unless a static range is set aside
|
||||
if l := defaultLayout(&sn, []Subnet{sn}); l.Interfaces[0].Links[0].Mode != "dhcp" {
|
||||
t.Errorf("proxy subnet without a static range: %+v", l.Interfaces[0].Links)
|
||||
}
|
||||
sn.Ranges = []IPRange{{Kind: "static", Start: "10.0.0.50", End: "10.0.0.99"}}
|
||||
if l := defaultLayout(&sn, []Subnet{sn}); l.Interfaces[0].Links[0].Mode != "auto" {
|
||||
t.Errorf("proxy subnet with a static range: %+v", l.Interfaces[0].Links)
|
||||
}
|
||||
if dhcpMode(true, "") != "serve" || dhcpMode(false, "proxy") != "" || dhcpMode(true, "proxy") != "proxy" {
|
||||
t.Error("dhcpMode")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDhcpBootOnlyForServedSubnets(t *testing.T) {
|
||||
serve := labSubnet()
|
||||
proxy := labSubnet()
|
||||
proxy.CIDR, proxy.Gateway, proxy.DHCPMode, proxy.Ranges = "10.9.0.0/24", "", "proxy", nil
|
||||
rc := RackController{Name: "r1", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"10.0.0.2/24"}}, {Name: "eth2", Addrs: []string{"10.9.0.2/24"}}}}
|
||||
conf, _ := dnsmasqConfig(rc, []Subnet{serve, proxy}, "/d")
|
||||
if !strings.Contains(conf, "dhcp-boot=tag:s0,tag:bios,pxelinux.0") || !strings.Contains(conf, "dhcp-boot=tag:s0,tag:efi64,bootx64.efi") {
|
||||
t.Errorf("served subnet lacks its boot files:\n%s", conf)
|
||||
}
|
||||
for _, l := range strings.Split(conf, "\n") {
|
||||
if strings.HasPrefix(l, "dhcp-boot=") && !strings.HasPrefix(l, "dhcp-boot=tag:s0,") {
|
||||
t.Errorf("dhcp-boot not limited to the served subnet: %s", l)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(conf, "pxe-service=tag:eth2,x86PC") || strings.Contains(conf, "set:s1,10.9.0.0,proxy") {
|
||||
t.Errorf("proxy subnet lacks pxe-service:\n%s", conf)
|
||||
}
|
||||
}
|
||||
|
||||
func TestProxyOnlyConfigHasNoArchTags(t *testing.T) {
|
||||
sn := labSubnet()
|
||||
sn.DHCPMode, sn.Ranges = "proxy", nil
|
||||
rc := RackController{Name: "r1", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"10.0.0.2/24"}}}}
|
||||
conf, _ := dnsmasqConfig(rc, []Subnet{sn}, "/d")
|
||||
if strings.Contains(conf, "dhcp-match=") || strings.Contains(conf, "dhcp-boot=") {
|
||||
t.Errorf("proxy-only config has arch tags or dhcp-boot:\n%s", conf)
|
||||
}
|
||||
serve := labSubnet()
|
||||
conf, _ = dnsmasqConfig(rc, []Subnet{serve}, "/d")
|
||||
if !strings.Contains(conf, "dhcp-match=set:bios,option:client-arch,0") {
|
||||
t.Errorf("served subnet lacks arch tags:\n%s", conf)
|
||||
}
|
||||
}
|
||||
+84
-272
@@ -12,7 +12,11 @@ package main
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"crypto/subtle"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
@@ -34,17 +38,10 @@ type powerDriver struct {
|
||||
DefaultPort int `json:"default_port"`
|
||||
}
|
||||
|
||||
var powerDrivers = []powerDriver{
|
||||
{"ipmi", "IPMI 2.0 over LAN (ipmitool lanplus); any vendor", "ipmi", 623},
|
||||
{"redfish", "DMTF Redfish, any vendor", "redfish", 443},
|
||||
{"idrac", "Dell iDRAC 8/9 (Redfish)", "redfish", 443},
|
||||
{"ilo", "HPE iLO 4 (2.30+), 5, 6 (Redfish)", "redfish", 443},
|
||||
{"supermicro", "Supermicro X11 and later (Redfish)", "redfish", 443},
|
||||
{"xcc", "Lenovo XClarity Controller (Redfish)", "redfish", 443},
|
||||
{"openbmc", "OpenBMC (Redfish)", "redfish", 443},
|
||||
{"proxmox", "Proxmox VE virtual machine (API token: user = token id, password = secret; extra node, vmid)", "api", 8006},
|
||||
{"manual", "no power control: someone presses the button", "none", 0},
|
||||
}
|
||||
var powerDrivers = append(append([]powerDriver{{"ipmi", "IPMI 2.0 over LAN (ipmitool lanplus); any vendor", "ipmi", 623}},
|
||||
redfishDrivers()...),
|
||||
powerDriver{"proxmox", "Proxmox VE virtual machine (API token: user = token id, password = secret; extra node, vmid)", "api", 8006},
|
||||
powerDriver{"manual", "no power control: someone presses the button", "none", 0})
|
||||
|
||||
func driverByName(n string) (powerDriver, bool) {
|
||||
if n == "" {
|
||||
@@ -64,6 +61,16 @@ var errManualPower = errors.New("this machine has manual power control: do it by
|
||||
|
||||
// powerRun executes one operation with a driver.
|
||||
func powerRun(ctx context.Context, ipmitool string, c *bmcCreds, op, arg string) (string, error) {
|
||||
if op == "tls-fingerprint" {
|
||||
ctx, cancel := context.WithTimeout(ctx, 20*time.Second)
|
||||
defer cancel()
|
||||
return bmcTLSFingerprint(ctx, c)
|
||||
}
|
||||
if t, ok := fwTimeouts[op]; ok { // firmware: Redfish UpdateService (redfish_fw.go)
|
||||
ctx, cancel := context.WithTimeout(ctx, t)
|
||||
defer cancel()
|
||||
return firmwareRun(ctx, c, op, arg)
|
||||
}
|
||||
if !powerOps[op] {
|
||||
return "", fmt.Errorf("unknown power operation %q", op)
|
||||
}
|
||||
@@ -136,26 +143,80 @@ func ipmiExec(ctx context.Context, tool string, c *bmcCreds, args ...string) (st
|
||||
return strings.TrimSpace(out.String()), err
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Redfish
|
||||
// ---------------------------------------------------------------- BMC HTTP (Redfish, Proxmox)
|
||||
|
||||
type rfClient struct {
|
||||
base, user, pass string
|
||||
hc *http.Client
|
||||
etag string
|
||||
// bmcHTTPClient: BMCs ship self-signed certificates, so a BMC is trusted by
|
||||
// the fingerprint of the certificate it showed first (extra tls_fingerprint,
|
||||
// pinned by the region before credentials are ever sent; see pinBMC).
|
||||
// verify_tls=true checks it against the system's CAs instead.
|
||||
func bmcHTTPClient(c *bmcCreds) *http.Client {
|
||||
tc := &tls.Config{InsecureSkipVerify: c.Extra["verify_tls"] != "true"} //nolint:gosec // pinned below
|
||||
if pin := strings.ToLower(c.Extra["tls_fingerprint"]); pin != "" && c.Extra["verify_tls"] != "true" {
|
||||
tc.VerifyConnection = func(cs tls.ConnectionState) error {
|
||||
if len(cs.PeerCertificates) == 0 {
|
||||
return errors.New("the BMC sent no certificate")
|
||||
}
|
||||
got := certSHA256(cs.PeerCertificates[0])
|
||||
if subtle.ConstantTimeCompare([]byte(got), []byte(pin)) != 1 {
|
||||
return fmt.Errorf("the BMC's certificate changed (pinned %s, now %s): not sending credentials; if the BMC was replaced or got new firmware, clear tls_fingerprint to trust the new one", short(pin), short(got))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
tr := &http.Transport{TLSClientConfig: tc, DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext}
|
||||
return &http.Client{Timeout: 45 * time.Second, Transport: tr}
|
||||
}
|
||||
|
||||
func bmcHTTPClient(c *bmcCreds) *http.Client {
|
||||
// BMCs ship self-signed certificates; checking them is opt-in (extra verify_tls=true)
|
||||
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Extra["verify_tls"] != "true"}, //nolint:gosec
|
||||
DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext}
|
||||
return &http.Client{Timeout: 45 * time.Second, Transport: tr}
|
||||
func certSHA256(c *x509.Certificate) string {
|
||||
s := sha256.Sum256(c.Raw)
|
||||
return hex.EncodeToString(s[:])
|
||||
}
|
||||
|
||||
func short(fp string) string {
|
||||
if len(fp) > 16 {
|
||||
return fp[:16] + "…"
|
||||
}
|
||||
return fp
|
||||
}
|
||||
|
||||
// bmcTLSFingerprint connects to the BMC's HTTPS port and returns its
|
||||
// certificate's SHA-256 (op tls-fingerprint): a handshake only, no request,
|
||||
// no credentials.
|
||||
func bmcTLSFingerprint(ctx context.Context, c *bmcCreds) (string, error) {
|
||||
port := 443
|
||||
if d, _ := driverByName(c.Driver); d.Family == "api" {
|
||||
port = 8006
|
||||
}
|
||||
u, err := url.Parse(bmcBase(c, "https", port))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
host := u.Host
|
||||
if u.Port() == "" {
|
||||
host = net.JoinHostPort(u.Hostname(), "443")
|
||||
}
|
||||
d := &tls.Dialer{NetDialer: &net.Dialer{Timeout: 10 * time.Second}, Config: &tls.Config{InsecureSkipVerify: true}} //nolint:gosec // reading the certificate is the point
|
||||
conn, err := d.DialContext(ctx, "tcp", host)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer conn.Close()
|
||||
cs := conn.(*tls.Conn).ConnectionState()
|
||||
if len(cs.PeerCertificates) == 0 {
|
||||
return "", errors.New("the BMC sent no certificate")
|
||||
}
|
||||
return certSHA256(cs.PeerCertificates[0]), nil
|
||||
}
|
||||
|
||||
// bmcBase builds https://host[:port] from the record; IPMI's port is not Redfish's.
|
||||
func bmcBase(c *bmcCreds, scheme string, defPort int) string {
|
||||
a := strings.TrimRight(c.Address, "/")
|
||||
if strings.Contains(a, "://") {
|
||||
return a
|
||||
if strings.Contains(a, "://") { // an operator's https://host:port: scheme and authority only
|
||||
u, err := url.Parse(a)
|
||||
if err != nil || (u.Scheme != "https" && u.Scheme != "http") || u.Host == "" || u.User != nil {
|
||||
return "https://invalid.invalid"
|
||||
}
|
||||
return u.Scheme + "://" + u.Host
|
||||
}
|
||||
port := c.Port
|
||||
if port == 623 || port == 0 {
|
||||
@@ -171,255 +232,6 @@ func bmcBase(c *bmcCreds, scheme string, defPort int) string {
|
||||
return fmt.Sprintf("%s://%s:%d", scheme, host, port)
|
||||
}
|
||||
|
||||
func newRF(c *bmcCreds) *rfClient {
|
||||
return &rfClient{base: bmcBase(c, "https", 443), user: c.User, pass: c.Password, hc: bmcHTTPClient(c)}
|
||||
}
|
||||
|
||||
func (r *rfClient) do(ctx context.Context, method, path string, body, out any) (int, error) {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, r.base+path, rd)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if r.user != "" {
|
||||
req.SetBasicAuth(r.user, r.pass)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if method == "PATCH" && r.etag != "" {
|
||||
req.Header.Set("If-Match", r.etag) // some services (Supermicro, newer iDRAC) insist
|
||||
}
|
||||
resp, err := r.hc.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
if method == "GET" {
|
||||
if et := resp.Header.Get("ETag"); et != "" {
|
||||
r.etag = et
|
||||
}
|
||||
}
|
||||
if resp.StatusCode >= 300 {
|
||||
return resp.StatusCode, fmt.Errorf("%s %s: %s %s", method, path, resp.Status, redfishMessage(data))
|
||||
}
|
||||
if out != nil && len(data) > 0 {
|
||||
if err := json.Unmarshal(data, out); err != nil {
|
||||
return resp.StatusCode, fmt.Errorf("%s %s: %w", method, path, err)
|
||||
}
|
||||
}
|
||||
return resp.StatusCode, nil
|
||||
}
|
||||
|
||||
// redfishMessage pulls the human message out of a Redfish error body.
|
||||
func redfishMessage(data []byte) string {
|
||||
var e struct {
|
||||
Error struct {
|
||||
Message string `json:"message"`
|
||||
Ext []struct {
|
||||
Message string `json:"Message"`
|
||||
} `json:"@Message.ExtendedInfo"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(data, &e) == nil {
|
||||
if len(e.Error.Ext) > 0 && e.Error.Ext[0].Message != "" {
|
||||
return e.Error.Ext[0].Message
|
||||
}
|
||||
if e.Error.Message != "" {
|
||||
return e.Error.Message
|
||||
}
|
||||
}
|
||||
return oneLine(string(data))
|
||||
}
|
||||
|
||||
// redfishSystemPath: extra system=…, else the vendor's usual one if it exists, else the first member.
|
||||
func (r *rfClient) system(ctx context.Context, c *bmcCreds) (string, obj, error) {
|
||||
var cands []string
|
||||
if s := c.Extra["system"]; s != "" {
|
||||
cands = append(cands, s)
|
||||
}
|
||||
switch c.Driver {
|
||||
case "idrac":
|
||||
cands = append(cands, "/redfish/v1/Systems/System.Embedded.1")
|
||||
case "ilo", "xcc", "supermicro":
|
||||
cands = append(cands, "/redfish/v1/Systems/1")
|
||||
}
|
||||
for _, p := range cands {
|
||||
var sys obj
|
||||
if _, err := r.do(ctx, "GET", p, nil, &sys); err == nil {
|
||||
return p, sys, nil
|
||||
} else if c.Extra["system"] != "" {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
var coll struct {
|
||||
Members []struct {
|
||||
ID string `json:"@odata.id"`
|
||||
} `json:"Members"`
|
||||
}
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/Systems", nil, &coll); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if len(coll.Members) == 0 {
|
||||
return "", nil, errors.New("the BMC lists no systems")
|
||||
}
|
||||
p := coll.Members[0].ID
|
||||
var sys obj
|
||||
_, err := r.do(ctx, "GET", p, nil, &sys)
|
||||
return p, sys, err
|
||||
}
|
||||
|
||||
// resetTypes are the ResetType values the system accepts.
|
||||
func (r *rfClient) resetTypes(ctx context.Context, sys obj) []string {
|
||||
act, _ := dig(sys, "Actions", "#ComputerSystem.Reset").(obj)
|
||||
var out []string
|
||||
for _, v := range toList(act["ResetType@Redfish.AllowableValues"]) {
|
||||
out = append(out, str(v))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if info := str(act["@Redfish.ActionInfo"]); info != "" {
|
||||
var ai struct {
|
||||
Parameters []struct {
|
||||
Name string `json:"Name"`
|
||||
AllowableValues []string `json:"AllowableValues"`
|
||||
} `json:"Parameters"`
|
||||
}
|
||||
if _, err := r.do(ctx, "GET", info, nil, &ai); err == nil {
|
||||
for _, p := range ai.Parameters {
|
||||
if p.Name == "ResetType" {
|
||||
out = p.AllowableValues
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pickReset(allowed []string, prefs ...string) string {
|
||||
if len(allowed) == 0 {
|
||||
return prefs[0]
|
||||
}
|
||||
for _, p := range prefs {
|
||||
if hasTag(allowed, p) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
var redfishBootTargets = map[string]string{"pxe": "Pxe", "disk": "Hdd", "bios": "BiosSetup", "cdrom": "Cd", "none": "None", "safe": "None"}
|
||||
|
||||
func redfishOp(ctx context.Context, c *bmcCreds, op, arg string) (string, error) {
|
||||
r := newRF(c)
|
||||
path, sys, err := r.system(ctx, c)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
state := strings.ToLower(str(sys["PowerState"])) // On | Off | PoweringOn | PoweringOff
|
||||
on := state == "on" || state == "poweringon"
|
||||
reset := func(prefs ...string) (string, error) {
|
||||
t := pickReset(r.resetTypes(ctx, sys), prefs...)
|
||||
if t == "" {
|
||||
return "", fmt.Errorf("the BMC accepts none of %v", prefs)
|
||||
}
|
||||
target := str(dig(sys, "Actions", "#ComputerSystem.Reset", "target"))
|
||||
if target == "" {
|
||||
target = path + "/Actions/ComputerSystem.Reset"
|
||||
}
|
||||
if _, err := r.do(ctx, "POST", target, obj{"ResetType": t}, nil); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "Chassis Power Control: " + t + " (redfish)", nil
|
||||
}
|
||||
switch op {
|
||||
case "status":
|
||||
if on {
|
||||
return "Chassis Power is on", nil
|
||||
}
|
||||
return "Chassis Power is off", nil
|
||||
case "on":
|
||||
if on {
|
||||
return "Chassis Power is on already", nil
|
||||
}
|
||||
return reset("On", "PushPowerButton")
|
||||
case "off":
|
||||
if !on {
|
||||
return "Chassis Power is off already", nil
|
||||
}
|
||||
return reset("ForceOff", "PushPowerButton")
|
||||
case "soft":
|
||||
if !on {
|
||||
return "Chassis Power is off already", nil
|
||||
}
|
||||
return reset("GracefulShutdown", "PushPowerButton")
|
||||
case "cycle":
|
||||
if !on {
|
||||
return reset("On", "PushPowerButton")
|
||||
}
|
||||
return reset("PowerCycle", "ForceRestart", "GracefulRestart")
|
||||
case "reset":
|
||||
if !on {
|
||||
return reset("On", "PushPowerButton")
|
||||
}
|
||||
return reset("ForceRestart", "PowerCycle", "GracefulRestart")
|
||||
case "bootdev":
|
||||
boot := obj{"BootSourceOverrideTarget": redfishBootTargets[arg], "BootSourceOverrideEnabled": "Once"}
|
||||
if arg == "none" || arg == "safe" {
|
||||
boot = obj{"BootSourceOverrideEnabled": "Disabled"}
|
||||
}
|
||||
if _, err := r.do(ctx, "PATCH", path, obj{"Boot": boot}, nil); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "Set Boot Device to " + arg + " (redfish " + str(boot["BootSourceOverrideTarget"]) + ", once)", nil
|
||||
}
|
||||
return "", fmt.Errorf("redfish: %s is not supported", op)
|
||||
}
|
||||
|
||||
// redfishProbe asks whether the BMC speaks Redfish with these credentials and
|
||||
// suggests the driver for its vendor. The answer is JSON.
|
||||
func redfishProbe(ctx context.Context, c *bmcCreds) (string, error) {
|
||||
r := newRF(c)
|
||||
var root obj
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/", nil, &root); err != nil {
|
||||
return "", fmt.Errorf("no Redfish service: %w", err)
|
||||
}
|
||||
res := obj{"redfish": str(root["RedfishVersion"]), "vendor": str(root["Vendor"])}
|
||||
if res["vendor"] == "" {
|
||||
for k := range asObj(root["Oem"]) {
|
||||
res["vendor"] = k
|
||||
}
|
||||
}
|
||||
path, sys, err := r.system(ctx, &bmcCreds{Driver: "redfish", Extra: c.Extra, User: c.User, Password: c.Password, Address: c.Address, Port: c.Port})
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Redfish %s answers, but not to these credentials: %w", res["redfish"], err)
|
||||
}
|
||||
res["system"], res["manufacturer"], res["model"], res["power"] = path, str(sys["Manufacturer"]), str(sys["Model"]), str(sys["PowerState"])
|
||||
v := strings.ToLower(str(res["vendor"]) + " " + str(res["manufacturer"]))
|
||||
switch {
|
||||
case strings.Contains(v, "dell"):
|
||||
res["driver"] = "idrac"
|
||||
case strings.Contains(v, "hpe"), strings.Contains(v, "hewlett"), strings.Contains(v, "hp "):
|
||||
res["driver"] = "ilo"
|
||||
case strings.Contains(v, "supermicro"):
|
||||
res["driver"] = "supermicro"
|
||||
case strings.Contains(v, "lenovo"):
|
||||
res["driver"] = "xcc"
|
||||
case strings.Contains(v, "openbmc"):
|
||||
res["driver"] = "openbmc"
|
||||
default:
|
||||
res["driver"] = "redfish"
|
||||
}
|
||||
b, _ := json.Marshal(res)
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func asObj(v any) obj {
|
||||
o, _ := v.(obj)
|
||||
return o
|
||||
|
||||
+88
-19
@@ -23,6 +23,7 @@ import (
|
||||
"net/http"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"sort"
|
||||
"strconv"
|
||||
@@ -39,8 +40,9 @@ type RackIface struct {
|
||||
|
||||
type RackController struct {
|
||||
Name string `json:"name"`
|
||||
URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1
|
||||
Kind string `json:"kind"` // embedded | remote
|
||||
URL string `json:"url"` // how machines reach it, e.g. http://10.213.78.1
|
||||
TLSURL string `json:"tls_url,omitempty"` // the same host over HTTPS, when it serves it (agents)
|
||||
Kind string `json:"kind"` // embedded | remote
|
||||
Version string `json:"version"`
|
||||
Arch string `json:"arch,omitempty"` // GOARCH, for self-update
|
||||
Interfaces []RackIface `json:"interfaces"`
|
||||
@@ -75,7 +77,9 @@ type rackJob struct {
|
||||
Extra map[string]string `json:"extra"`
|
||||
Op string `json:"op"`
|
||||
Arg string `json:"arg"`
|
||||
Timeout int `json:"timeout,omitempty"` // seconds the region waits (default 90)
|
||||
done chan rackResult
|
||||
rack string // who took it: only that rack may answer
|
||||
}
|
||||
|
||||
type rackResult struct {
|
||||
@@ -87,7 +91,7 @@ type rackResult struct {
|
||||
// ---------------------------------------------------------------- store
|
||||
|
||||
func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
|
||||
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
|
||||
rows, err := s.pool.Query(ctx, `SELECT name, coalesce(url,''), coalesce(tls_url,''), kind, coalesce(version,''), coalesce(arch,''), interfaces, dhcp_interfaces, coalesce(tftp_root,''), last_seen, registered FROM rack_controllers ORDER BY name`)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@@ -96,7 +100,7 @@ func (s *Store) RackControllers(ctx context.Context) ([]RackController, error) {
|
||||
for rows.Next() {
|
||||
var r RackController
|
||||
var ifs, dif []byte
|
||||
if err := rows.Scan(&r.Name, &r.URL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
|
||||
if err := rows.Scan(&r.Name, &r.URL, &r.TLSURL, &r.Kind, &r.Version, &r.Arch, &ifs, &dif, &r.TFTPRoot, &r.LastSeen, &r.Registered); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
_ = json.Unmarshal(ifs, &r.Interfaces)
|
||||
@@ -115,9 +119,9 @@ func (s *Store) PutRackController(ctx context.Context, r RackController) error {
|
||||
}
|
||||
ifs, _ := json.Marshal(r.Interfaces)
|
||||
dif, _ := json.Marshal(r.DHCPInterfaces)
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''))
|
||||
ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,'')`,
|
||||
r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch)
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO rack_controllers(name, url, kind, version, interfaces, dhcp_interfaces, tftp_root, last_seen, arch, tls_url) VALUES ($1,$2,$3,$4,$5,$6,$7,now(),nullif($8,''),nullif($9,''))
|
||||
ON CONFLICT (name) DO UPDATE SET url=$2, kind=$3, version=$4, interfaces=$5, dhcp_interfaces=$6, tftp_root=$7, last_seen=now(), arch=nullif($8,''), tls_url=nullif($9,'')`,
|
||||
r.Name, r.URL, r.Kind, r.Version, ifs, dif, r.TFTPRoot, r.Arch, r.TLSURL)
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -137,8 +141,13 @@ func (s *Store) SetRackController(ctx context.Context, id, rack string) error {
|
||||
}
|
||||
|
||||
func (s *Store) PutPXE(ctx context.Context, mac, machineID, rack, content string) error {
|
||||
_, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4)
|
||||
ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now()`, mac, machineID, rack, content)
|
||||
// a MAC another machine owns is not taken over: a box claiming a deployed
|
||||
// machine's MAC would otherwise replace that machine's boot file
|
||||
tag, err := s.pool.Exec(ctx, `INSERT INTO pxe_files(mac, machine_id, rack, content) VALUES ($1,$2,$3,$4)
|
||||
ON CONFLICT (mac) DO UPDATE SET machine_id=$2, rack=$3, content=$4, updated=now() WHERE pxe_files.machine_id=$2`, mac, machineID, rack, content)
|
||||
if err == nil && tag.RowsAffected() == 0 {
|
||||
return fmt.Errorf("MAC %s belongs to another machine; delete that machine first", mac)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
@@ -164,6 +173,8 @@ func (s *Store) PXEFiles(ctx context.Context, rack string) (map[string]string, e
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
var pxeNameRe = regexp.MustCompile(`^01(-[0-9a-f]{2}){6,20}$`)
|
||||
|
||||
func pxeFileName(mac string) string { return "01-" + strings.ReplaceAll(normMAC(mac), ":", "-") }
|
||||
|
||||
// ---------------------------------------------------------------- hub: state generations and the command queue
|
||||
@@ -233,7 +244,11 @@ func (h *rackHub) dispatch(ctx context.Context, rack string, j *rackJob) (string
|
||||
delete(h.wake, rack)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
t := time.NewTimer(90 * time.Second)
|
||||
wait := 90 * time.Second
|
||||
if j.Timeout > 0 {
|
||||
wait = time.Duration(j.Timeout) * time.Second
|
||||
}
|
||||
t := time.NewTimer(wait)
|
||||
defer t.Stop()
|
||||
select {
|
||||
case r := <-j.done:
|
||||
@@ -264,6 +279,7 @@ func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*
|
||||
if q := h.work[rack]; len(q) > 0 {
|
||||
h.work[rack] = nil
|
||||
for _, j := range q {
|
||||
j.rack = rack
|
||||
h.flight[j.ID] = j
|
||||
}
|
||||
h.mu.Unlock()
|
||||
@@ -291,10 +307,14 @@ func (h *rackHub) take(ctx context.Context, rack string, wait time.Duration) []*
|
||||
}
|
||||
}
|
||||
|
||||
func (h *rackHub) finish(r rackResult) bool {
|
||||
func (h *rackHub) finish(rack string, r rackResult) bool {
|
||||
h.mu.Lock()
|
||||
j, ok := h.flight[r.ID]
|
||||
delete(h.flight, r.ID)
|
||||
if ok && j.rack != rack {
|
||||
ok = false
|
||||
} else {
|
||||
delete(h.flight, r.ID)
|
||||
}
|
||||
h.mu.Unlock()
|
||||
if ok {
|
||||
j.done <- r
|
||||
@@ -390,10 +410,14 @@ func (a *App) urlFor(m *Machine) string { return a.rackURL(a.rackOf(m)) }
|
||||
|
||||
// appendFor is commission_append with the server URL of the machine's rack.
|
||||
func (a *App) appendFor(base string) string {
|
||||
if base == a.cfg.ServerURL {
|
||||
return a.cfg.CommissionAppend
|
||||
s := a.cfg.CommissionAppend
|
||||
if base != a.cfg.ServerURL {
|
||||
s = strings.ReplaceAll(s, a.cfg.ServerURL, base)
|
||||
}
|
||||
return strings.ReplaceAll(a.cfg.CommissionAppend, a.cfg.ServerURL, base)
|
||||
if t := a.agentTLSArgs(base); len(t) > 0 && !strings.Contains(s, "rollcall.tls=") {
|
||||
s += " " + strings.Join(t, " ")
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// rackAddresses are the rack controllers' own addresses, never handed to machines.
|
||||
@@ -479,6 +503,8 @@ func (a *App) rackState(ctx context.Context, name string) (*rackState, error) {
|
||||
// hostsFiles: static DHCP leases for the machines this rack serves, and names
|
||||
// for every machine: the addresses on the interface carrying the management
|
||||
// address get the plain name (A and AAAA), others name-<interface>, IB name-ib.
|
||||
var nonHostChars = regexp.MustCompile(`[^a-z0-9-]+`)
|
||||
|
||||
func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) {
|
||||
ms, err := a.store.List(ctx)
|
||||
if err != nil {
|
||||
@@ -507,6 +533,13 @@ func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (st
|
||||
fmt.Fprintf(&hosts, "%s %s\n", ip, name)
|
||||
}
|
||||
}
|
||||
// proxy subnets: the site's DHCP server hands out the addresses, so no dhcp-host
|
||||
// lines for them (they would do nothing, and the "known" tag they give a client
|
||||
// helps dnsmasq 2.91 build a cyclic tag list in its PXE proxy code: busy loop)
|
||||
proxied := func(ip string) bool {
|
||||
sn := subnetFor(subnets, ip)
|
||||
return sn != nil && sn.DHCPMode == "proxy"
|
||||
}
|
||||
sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) })
|
||||
for _, m := range ms {
|
||||
if m.NodeName == nil || len(m.MACs) == 0 {
|
||||
@@ -522,7 +555,7 @@ func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (st
|
||||
}
|
||||
if mgmt != "" {
|
||||
line(mgmt, name)
|
||||
if mine && familyOf(mgmt+"/32") == 4 {
|
||||
if mine && familyOf(mgmt+"/32") == 4 && !proxied(mgmt) {
|
||||
fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], mgmt, name)
|
||||
}
|
||||
}
|
||||
@@ -534,11 +567,13 @@ func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (st
|
||||
switch {
|
||||
case x.Iface == mgmtIface && mgmtIface != "":
|
||||
line(x.IP, name)
|
||||
if v6 && mine {
|
||||
if v6 && mine && !proxied(x.IP) {
|
||||
fmt.Fprintf(&dhcp, "%s,[%s],%s\n", m.MACs[0], x.IP, name)
|
||||
}
|
||||
case x.Iface != "":
|
||||
line(x.IP, name+"-"+strings.NewReplacer(".", "-", ":", "-").Replace(x.Iface))
|
||||
if sfx := strings.Trim(nonHostChars.ReplaceAllString(strings.ToLower(x.Iface), "-"), "-"); sfx != "" {
|
||||
line(x.IP, name+"-"+sfx)
|
||||
}
|
||||
}
|
||||
}
|
||||
if ibip != "" {
|
||||
@@ -579,6 +614,10 @@ func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err e
|
||||
note("default", ch, e)
|
||||
}
|
||||
for name, body := range st.PXE {
|
||||
if !pxeNameRe.MatchString(name) {
|
||||
note(name, false, fmt.Errorf("not a pxelinux file name"))
|
||||
continue
|
||||
}
|
||||
ch, e := writeIfChanged(filepath.Join(dir, name), body, 0o644)
|
||||
note(name, ch, e)
|
||||
}
|
||||
@@ -589,6 +628,30 @@ func applyRackState(st *rackState, tftp, dnsDir string) (changed []string, err e
|
||||
}
|
||||
}
|
||||
}
|
||||
// the same menus for UEFI (shim + GRUB, grub.go), where the install laid that out
|
||||
if gdir := filepath.Join(tftp, "grub"); dirExists(gdir) {
|
||||
ch, e := writeIfChanged(filepath.Join(gdir, "grub.cfg"), grubDispatch, 0o644)
|
||||
note("grub.cfg", ch, e)
|
||||
if st.DefaultPXE != "" {
|
||||
ch, e := writeIfChanged(filepath.Join(gdir, "grub.cfg-default"), pxelinuxToGrub(st.DefaultPXE), 0o644)
|
||||
note("grub.cfg-default", ch, e)
|
||||
}
|
||||
want := map[string]bool{}
|
||||
for name, body := range st.PXE {
|
||||
if g := grubName(name); g != "" {
|
||||
want[g] = true
|
||||
ch, e := writeIfChanged(filepath.Join(gdir, g), pxelinuxToGrub(body), 0o644)
|
||||
note(g, ch, e)
|
||||
}
|
||||
}
|
||||
if olds, e := filepath.Glob(filepath.Join(gdir, "grub.cfg-*:*")); e == nil {
|
||||
for _, p := range olds {
|
||||
if !want[filepath.Base(p)] && !strings.HasSuffix(p, ".tmp") {
|
||||
note("-"+filepath.Base(p), true, os.Remove(p))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if dnsDir != "" {
|
||||
ch, e := writeIfChanged(filepath.Join(dnsDir, "hosts.dhcp"), st.HostsDHCP, 0o644)
|
||||
note("hosts.dhcp", ch, e)
|
||||
@@ -788,6 +851,7 @@ func (a *App) handleRackRegister(w http.ResponseWriter, r *http.Request, rack st
|
||||
fail(w, 400, errors.New("url is required"))
|
||||
return
|
||||
}
|
||||
rc.TLSURL = validTLSURL(rc) // its own host only
|
||||
if err := a.store.PutRackController(r.Context(), rc); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
@@ -830,7 +894,7 @@ func (a *App) handleRackResult(w http.ResponseWriter, r *http.Request, rack stri
|
||||
fail(w, 400, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, 200, obj{"accepted": a.hub.finish(res)})
|
||||
writeJSON(w, 200, obj{"accepted": a.hub.finish(rack, res)})
|
||||
}
|
||||
|
||||
func (a *App) handleControllers(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -929,3 +993,8 @@ func localAddrs() []string {
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func dirExists(p string) bool {
|
||||
st, err := os.Stat(p)
|
||||
return err == nil && st.IsDir()
|
||||
}
|
||||
+253
-20
@@ -13,6 +13,7 @@ import (
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"crypto/x509/pkix"
|
||||
@@ -53,19 +54,34 @@ type RackConfig struct {
|
||||
KeyFile string `json:"key_file,omitempty"` // and key; with them the secret is not needed
|
||||
AutoUpdate *bool `json:"auto_update,omitempty"` // follow the region's version (default true)
|
||||
UpdateDir string `json:"update_dir,omitempty"` // default /var/lib/rollcall/update
|
||||
// CertStateDir: where a renewed certificate goes (cert and key in one file,
|
||||
// rack.pem). The files from rack join are root's; this directory is the
|
||||
// service user's. Default /var/lib/rollcall/rack-cert.
|
||||
CertStateDir string `json:"cert_state_dir,omitempty"`
|
||||
// TLSListen: HTTPS for the agents (node API, images), with a server
|
||||
// certificate from the region for the URL's host. Default <url host>:443;
|
||||
// "off" disables it.
|
||||
TLSListen string `json:"tls_listen,omitempty"`
|
||||
}
|
||||
|
||||
func (c RackConfig) renewedPath() string {
|
||||
return filepath.Join(orDefault(c.CertStateDir, "/var/lib/rollcall/rack-cert"), "rack.pem")
|
||||
}
|
||||
|
||||
type rackd struct {
|
||||
cfg RackConfig
|
||||
region *url.URL
|
||||
hc *http.Client // long polls
|
||||
fast *http.Client
|
||||
big *http.Client // image downloads, no timeout
|
||||
gen int64
|
||||
tlsc *tls.Config
|
||||
certMu sync.Mutex
|
||||
upd rackUpdater
|
||||
cert *tls.Certificate
|
||||
srvMu sync.Mutex
|
||||
srvCert *tls.Certificate // server certificate for agents (HTTPS)
|
||||
tlsURL string
|
||||
cfg RackConfig
|
||||
region *url.URL
|
||||
hc *http.Client // long polls
|
||||
fast *http.Client
|
||||
big *http.Client // image downloads, no timeout
|
||||
gen int64
|
||||
tlsc *tls.Config
|
||||
certMu sync.Mutex
|
||||
upd rackUpdater
|
||||
cert *tls.Certificate
|
||||
|
||||
headMu sync.Mutex
|
||||
heads map[string]headInfo
|
||||
@@ -136,7 +152,11 @@ func (r *rackd) clients() error {
|
||||
}
|
||||
pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0])
|
||||
r.cert = &pair
|
||||
if renewed := loadRenewed(r.cfg.renewedPath(), pool, r.cfg.Name, pair.Leaf); renewed != nil {
|
||||
r.cert = renewed
|
||||
}
|
||||
r.tlsc.RootCAs = pool
|
||||
r.tlsc.VerifyConnection = regionLeaf
|
||||
r.tlsc.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) {
|
||||
r.certMu.Lock()
|
||||
defer r.certMu.Unlock()
|
||||
@@ -184,11 +204,9 @@ func (r *rackd) renew() error {
|
||||
return err
|
||||
}
|
||||
pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0])
|
||||
if err := os.WriteFile(r.cfg.KeyFile, key, 0o640); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := os.WriteFile(r.cfg.CertFile, []byte(resp.Cert), 0o644); err != nil {
|
||||
return err
|
||||
// one file, renamed into place: a crash never leaves a key without its certificate
|
||||
if err := writeAtomic(r.cfg.renewedPath(), append([]byte(resp.Cert), key...), 0o600); err != nil {
|
||||
return fmt.Errorf("keep the renewed certificate (the region already switched to it; until this works the previous one is accepted for 7 days): %w; create the directory: install -d -m 0700 -o rollcall -g rollcall %s", err, filepath.Dir(r.cfg.renewedPath()))
|
||||
}
|
||||
r.certMu.Lock()
|
||||
r.cert = &pair
|
||||
@@ -197,6 +215,84 @@ func (r *rackd) renew() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// loadRenewed is the certificate renew() kept, when it belongs to this rack,
|
||||
// comes from the CA and outlives the one from rack join.
|
||||
func loadRenewed(p string, pool *x509.CertPool, name string, current *x509.Certificate) *tls.Certificate {
|
||||
b, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
pair, err := tls.X509KeyPair(b, b)
|
||||
if err != nil {
|
||||
log.Printf("%s: %v (ignored)", p, err)
|
||||
return nil
|
||||
}
|
||||
leaf, err := x509.ParseCertificate(pair.Certificate[0])
|
||||
if err != nil || leaf.Subject.CommonName != "rack:"+name {
|
||||
return nil
|
||||
}
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil {
|
||||
log.Printf("%s: %v (ignored)", p, err)
|
||||
return nil
|
||||
}
|
||||
if current != nil && !leaf.NotAfter.After(current.NotAfter) {
|
||||
return nil
|
||||
}
|
||||
pair.Leaf = leaf
|
||||
return &pair
|
||||
}
|
||||
|
||||
func writeAtomic(p string, data []byte, mode os.FileMode) error {
|
||||
f, err := os.CreateTemp(filepath.Dir(p), ".tmp-*")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
if _, err := f.Write(data); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Chmod(mode); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Sync(); err != nil {
|
||||
f.Close()
|
||||
return err
|
||||
}
|
||||
if err := f.Close(); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.Rename(f.Name(), p)
|
||||
}
|
||||
|
||||
// rackRenew renews the certificate now (rollcall rack renew), as the service user.
|
||||
func rackRenew(args []string) error {
|
||||
fs := newFlags("rack renew")
|
||||
cfgPath := fs.String("config", "/etc/rollcall/rack.json", "rack controller configuration")
|
||||
_ = fs.Parse(args)
|
||||
cfg, err := loadRackConfig(*cfgPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ru, err := url.Parse(cfg.Region)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}}
|
||||
if err := r.clients(); err != nil {
|
||||
return err
|
||||
}
|
||||
if r.cert == nil {
|
||||
return errors.New("this rack controller has no certificate (rollcall rack join)")
|
||||
}
|
||||
if err := r.renew(); err != nil {
|
||||
return err
|
||||
}
|
||||
fmt.Printf("renewed: %s, valid until %s; restart rollcall-rack to use it\n", cfg.renewedPath(), r.cert.Leaf.NotAfter.Format("2006-01-02"))
|
||||
return nil
|
||||
}
|
||||
|
||||
func newCSR(cn string) (keyPEM []byte, csrPEM string, err error) {
|
||||
key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
if err != nil {
|
||||
@@ -272,6 +368,28 @@ func rackJoin(args []string) error {
|
||||
if err := json.Unmarshal(data, &out); err != nil {
|
||||
return err
|
||||
}
|
||||
// what is written here is trusted for good: the CA must be the one the code
|
||||
// names, and the certificate ours from it
|
||||
caCert, err := parseCertPEM([]byte(out.CA))
|
||||
if err != nil {
|
||||
return fmt.Errorf("the region sent no CA: %w", err)
|
||||
}
|
||||
if fp := sha256.Sum256(caCert.Raw); string(fp[:16]) != string(code.Fingerprint[:]) || !caCert.IsCA {
|
||||
return errors.New("the region sent a CA that is not the one in the join code; not trusting it")
|
||||
}
|
||||
pair, err := tls.X509KeyPair([]byte(out.Cert), key)
|
||||
if err != nil {
|
||||
return fmt.Errorf("the certificate does not match our key: %w", err)
|
||||
}
|
||||
leaf, err := x509.ParseCertificate(pair.Certificate[0])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(caCert)
|
||||
if _, err := leaf.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil {
|
||||
return fmt.Errorf("the certificate is not from the code's CA: %w", err)
|
||||
}
|
||||
cfg.Name = out.Name
|
||||
if err := os.MkdirAll(*dir, 0o750); err != nil {
|
||||
return err
|
||||
@@ -334,7 +452,7 @@ func runRack(args []string) error {
|
||||
}
|
||||
go r.loop()
|
||||
mux := http.NewServeMux()
|
||||
for _, p := range []string{"/rollcall/", "/images/", "/os/"} {
|
||||
for _, p := range []string{"/rollcall/", "/images/", "/os/", "/firmware/"} { // firmware: BMCs pull packages from here
|
||||
mux.HandleFunc("GET "+p, r.serveCached)
|
||||
mux.HandleFunc("HEAD "+p, r.serveCached)
|
||||
}
|
||||
@@ -358,7 +476,13 @@ func runRack(args []string) error {
|
||||
mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, _ *http.Request) {
|
||||
writeJSON(w, 200, obj{"rack": cfg.Name, "gen": r.gen, "version": version})
|
||||
})
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second}
|
||||
mux.HandleFunc("GET /ca.pem", func(w http.ResponseWriter, req *http.Request) { // agents check it by its fingerprint
|
||||
http.ServeFile(w, req, cfg.CAFile)
|
||||
})
|
||||
if r.cert != nil && cfg.TLSListen != "off" {
|
||||
go r.serveTLS(logging(mux))
|
||||
}
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: logging(mux), ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
|
||||
log.Printf("rack controller %s: region %s, serving %s on %s (tftp %s, cache %s)", cfg.Name, cfg.Region, cfg.URL, cfg.Listen, cfg.TFTPRoot, cfg.CacheDir)
|
||||
return srv.ListenAndServe()
|
||||
}
|
||||
@@ -407,7 +531,10 @@ func (r *rackd) call(c *http.Client, method, p string, body, out any) (int, erro
|
||||
}
|
||||
|
||||
func (r *rackd) register() error {
|
||||
_, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, Version: version, Arch: runtime.GOARCH,
|
||||
r.srvMu.Lock()
|
||||
tlsURL := r.tlsURL
|
||||
r.srvMu.Unlock()
|
||||
_, err := r.call(r.fast, "POST", "/api/v1/rack/register", RackController{URL: r.cfg.URL, TLSURL: tlsURL, Version: version, Arch: runtime.GOARCH,
|
||||
Interfaces: localInterfaces(), DHCPInterfaces: r.cfg.DHCPInterfaces, TFTPRoot: r.cfg.TFTPRoot}, nil)
|
||||
return err
|
||||
}
|
||||
@@ -522,8 +649,8 @@ func (r *rackd) head(rel string) (headInfo, error) {
|
||||
// fresh says whether the cached copy matches the region's file. OS images live
|
||||
// under versioned paths and never change; the rest is checked by size and time.
|
||||
func (r *rackd) fresh(rel string, fi os.FileInfo) bool {
|
||||
if strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/") {
|
||||
return true
|
||||
if (strings.HasPrefix(rel, "/os/") && !strings.Contains(rel, "/current/")) || strings.HasPrefix(rel, "/firmware/") {
|
||||
return true // packages live under their id and never change
|
||||
}
|
||||
h, err := r.head(rel)
|
||||
if err != nil {
|
||||
@@ -640,3 +767,109 @@ func (r *rackd) prefetch(files []string) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- HTTPS for agents
|
||||
|
||||
func (c RackConfig) serverCertPath() string {
|
||||
return filepath.Join(orDefault(c.CertStateDir, "/var/lib/rollcall/rack-cert"), "server.pem")
|
||||
}
|
||||
|
||||
// serverCert keeps a server certificate for the URL's host, from the region.
|
||||
func (r *rackd) serverCert() (*tls.Certificate, error) {
|
||||
host, err := rackHost(r.cfg.URL)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
p := r.cfg.serverCertPath()
|
||||
if b, err := os.ReadFile(p); err == nil {
|
||||
if pair, err := tls.X509KeyPair(b, b); err == nil {
|
||||
leaf, _ := x509.ParseCertificate(pair.Certificate[0])
|
||||
if leaf != nil && time.Until(leaf.NotAfter) > renewBefore && leaf.VerifyHostname(host) == nil {
|
||||
pair.Leaf = leaf
|
||||
return &pair, nil
|
||||
}
|
||||
}
|
||||
}
|
||||
key, csr, err := newCSR(rackServerCN + r.cfg.Name)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var resp struct {
|
||||
Cert string `json:"cert"`
|
||||
}
|
||||
if _, err := r.call(r.fast, "POST", "/api/v1/rack/server-cert", enrollRequest{CSR: csr}, &resp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pair, err := tls.X509KeyPair([]byte(resp.Cert), key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0])
|
||||
if err := writeAtomic(p, append([]byte(resp.Cert), key...), 0o600); err != nil {
|
||||
log.Printf("keep the server certificate: %v (it is used, but asked for again at the next start)", err)
|
||||
}
|
||||
return &pair, nil
|
||||
}
|
||||
|
||||
// serveTLS serves the agents' endpoints over HTTPS once the region issued a
|
||||
// certificate, re-registers with the HTTPS URL and renews it when it ages.
|
||||
func (r *rackd) serveTLS(h http.Handler) {
|
||||
host, err := rackHost(r.cfg.URL)
|
||||
if err != nil {
|
||||
log.Printf("https for agents: %v", err)
|
||||
return
|
||||
}
|
||||
addr := r.cfg.TLSListen
|
||||
if addr == "" {
|
||||
addr = net.JoinHostPort(host, "443")
|
||||
}
|
||||
for {
|
||||
c, err := r.serverCert()
|
||||
if err == nil {
|
||||
r.srvMu.Lock()
|
||||
r.srvCert = c
|
||||
r.srvMu.Unlock()
|
||||
break
|
||||
}
|
||||
log.Printf("https for agents: no server certificate yet: %v", err)
|
||||
time.Sleep(30 * time.Second)
|
||||
}
|
||||
srv := &http.Server{Addr: addr, Handler: h, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute,
|
||||
TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, GetCertificate: func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
r.srvMu.Lock()
|
||||
defer r.srvMu.Unlock()
|
||||
return r.srvCert, nil
|
||||
}}}
|
||||
ln, err := net.Listen("tcp", addr)
|
||||
if err != nil {
|
||||
log.Printf("https for agents on %s: %v", addr, err)
|
||||
return
|
||||
}
|
||||
_, port, _ := net.SplitHostPort(addr)
|
||||
u := "https://" + hostPort(host, "")
|
||||
if port != "443" {
|
||||
u = "https://" + hostPort(host, port)
|
||||
}
|
||||
r.srvMu.Lock()
|
||||
r.tlsURL = u
|
||||
r.srvMu.Unlock()
|
||||
if err := r.register(); err != nil {
|
||||
log.Printf("register with %s: %v", u, err)
|
||||
}
|
||||
log.Printf("serving agents over HTTPS on %s (%s)", addr, u)
|
||||
go func() {
|
||||
for range time.Tick(12 * time.Hour) {
|
||||
r.srvMu.Lock()
|
||||
leaf := r.srvCert.Leaf
|
||||
r.srvMu.Unlock()
|
||||
if leaf != nil && time.Until(leaf.NotAfter) < renewBefore {
|
||||
if c, err := r.serverCert(); err == nil {
|
||||
r.srvMu.Lock()
|
||||
r.srvCert = c
|
||||
r.srvMu.Unlock()
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
log.Printf("https for agents: %v", srv.ServeTLS(ln, "", ""))
|
||||
}
|
||||
@@ -7,10 +7,18 @@ package main
|
||||
// expected version, and leaves a "ready" marker. A root path unit
|
||||
// (rollcall-rack-update.path) installs it, restarts the rack and puts the
|
||||
// previous binary back if the rack does not register again within a minute.
|
||||
//
|
||||
// The rack process is not trusted with root: it faces the provisioning
|
||||
// networks. The region signs each binary with its CA key, and the root unit
|
||||
// installs nothing that does not verify against the CA certificate written at
|
||||
// join (/etc/rollcall/rack/ca.pem, root's).
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/rand"
|
||||
"crypto/sha256"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
@@ -29,6 +37,7 @@ type rackUpdate struct {
|
||||
Version string `json:"version"`
|
||||
Arch string `json:"arch"`
|
||||
SHA256 string `json:"sha256"`
|
||||
Sig string `json:"sig"` // base64 DER ECDSA over the binary's SHA-256, by the region CA
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- region
|
||||
@@ -37,14 +46,36 @@ type binCache struct {
|
||||
mu sync.Mutex
|
||||
key map[string]string // path -> "size/mtime"
|
||||
sums map[string]string // path -> sha256
|
||||
sigs map[string]string // sha256 -> signature (kept: a new one each time would change the state)
|
||||
}
|
||||
|
||||
var rackBins = &binCache{key: map[string]string{}, sums: map[string]string{}}
|
||||
var rackBins = &binCache{key: map[string]string{}, sums: map[string]string{}, sigs: map[string]string{}}
|
||||
|
||||
// binarySig signs a binary's SHA-256 with the region CA key.
|
||||
func (a *App) binarySig(sum string) string {
|
||||
digest, err := hex.DecodeString(sum)
|
||||
if err != nil || a.ca == nil {
|
||||
return ""
|
||||
}
|
||||
rackBins.mu.Lock()
|
||||
defer rackBins.mu.Unlock()
|
||||
if s, ok := rackBins.sigs[sum]; ok {
|
||||
return s
|
||||
}
|
||||
der, err := ecdsa.SignASN1(rand.Reader, a.ca.Key, digest)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
rackBins.sigs[sum] = base64.StdEncoding.EncodeToString(der)
|
||||
return rackBins.sigs[sum]
|
||||
}
|
||||
|
||||
// rackBinary: the binary a rack of this architecture should run: the region's
|
||||
// own for the same architecture, else rack_binaries/rollcall-linux-<arch>.
|
||||
func (a *App) rackBinary(arch string) (path, sum string, ok bool) {
|
||||
if arch == "" {
|
||||
switch arch { // a file name part: never a path (the rack chooses it)
|
||||
case "amd64", "arm64", "386", "arm", "riscv64", "ppc64le", "s390x", "loong64":
|
||||
default:
|
||||
return "", "", false
|
||||
}
|
||||
if arch == runtime.GOARCH && runtime.GOOS == "linux" {
|
||||
@@ -87,7 +118,7 @@ func (a *App) rackUpdateFor(rc RackController) *rackUpdate {
|
||||
return nil
|
||||
}
|
||||
if _, sum, ok := a.rackBinary(rc.Arch); ok {
|
||||
return &rackUpdate{Version: version, Arch: rc.Arch, SHA256: sum}
|
||||
return &rackUpdate{Version: version, Arch: rc.Arch, SHA256: sum, Sig: a.binarySig(sum)}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -135,6 +166,10 @@ func (r *rackd) maybeUpdate(up *rackUpdate) {
|
||||
if up == nil || up.Version == "" || up.Version == version || !r.autoUpdate() || up.Arch != runtime.GOARCH {
|
||||
return
|
||||
}
|
||||
if _, err := hex.DecodeString(up.SHA256); err != nil || len(up.SHA256) != 64 || up.Sig == "" {
|
||||
log.Printf("update to %s: the region sent no valid checksum and signature", up.Version)
|
||||
return
|
||||
}
|
||||
r.upd.mu.Lock()
|
||||
if t, ok := r.upd.tried[up.Version]; ok && time.Since(t) < time.Hour {
|
||||
r.upd.mu.Unlock()
|
||||
@@ -205,6 +240,13 @@ func (r *rackd) stageUpdate(up *rackUpdate) error {
|
||||
if err := os.WriteFile(filepath.Join(dir, "rollcall.new.sha256"), []byte(up.SHA256+" rollcall.new\n"), 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
sig, err := base64.StdEncoding.DecodeString(up.Sig)
|
||||
if err != nil {
|
||||
return fmt.Errorf("signature: %w", err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "rollcall.new.sig"), sig, 0o644); err != nil {
|
||||
return err
|
||||
}
|
||||
ready := filepath.Join(dir, "ready")
|
||||
if err := os.WriteFile(ready+".tmp", []byte(up.Version+"\n"), 0o644); err != nil {
|
||||
return err
|
||||
|
||||
@@ -0,0 +1,828 @@
|
||||
package main
|
||||
|
||||
// Redfish power driver: one implementation of the DMTF standard, and what each
|
||||
// vendor documents differently in redfishProfiles (with the sources).
|
||||
//
|
||||
// For every vendor:
|
||||
// - ResetType values and boot targets come from @Redfish.AllowableValues
|
||||
// (or the action's ActionInfo); nothing is assumed.
|
||||
// - A one-time boot override sends target and enable in one PATCH, the boot
|
||||
// mode as the profile says (keep / omit / try), and only writable fields.
|
||||
// - PATCH carries If-Match with the resource's ETag (header or
|
||||
// @odata.etag). On 412 the strong form and then none are tried; on 428
|
||||
// the ETag is fetched first.
|
||||
// - A service that keeps pending settings elsewhere (AMI's .../SD,
|
||||
// @Redfish.Settings) gets the same PATCH there.
|
||||
// - "Busy" answers are retried for up to a minute: the server is in POST,
|
||||
// 409 right after a power change, 503, "Retry in N seconds".
|
||||
// - 202 answers are followed to their task monitor.
|
||||
// - A power cycle the system does not offer is ForceOff, wait for Off, On.
|
||||
//
|
||||
// Field notes behind these rules: OpenStack Ironic/sushy, MAAS and bmclib.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
type redfishProfile struct {
|
||||
Driver string // the power driver name
|
||||
Title string // for people
|
||||
Vendors []string // lower-case substrings of ServiceRoot Vendor or the system's Manufacturer
|
||||
OemKeys []string // ServiceRoot Oem keys that identify it
|
||||
Systems []string // its usual ComputerSystem, tried before the collection
|
||||
// BootMode: BootSourceOverrideMode in a boot override.
|
||||
// keep: always send the current mode (left out, the BMC may switch UEFI to Legacy)
|
||||
// omit: never send it unless the operator sets boot_mode (a change starts a BIOS job)
|
||||
// try: send the current mode; again without it if the BMC refuses the body
|
||||
BootMode string
|
||||
// Clear: bodies that clear a boot override, tried in order (default: Enabled Disabled)
|
||||
Clear []obj
|
||||
// Auth: "session" logs in through the SessionService (X-Auth-Token); default basic.
|
||||
Auth string
|
||||
// Firmware (UpdateService, redfish_fw.go). FWPlan builds the vendor's
|
||||
// documented request for a component; without one the standard methods are
|
||||
// tried in FWMethods order (default multipart, simple, push). FWUnsupported
|
||||
// says why Rollcall cannot update this vendor's firmware. FWFormats are the
|
||||
// package files it takes, checked at import.
|
||||
FWPlan func(fwPlanCtx) (*fwRequest, error)
|
||||
FWMethods []string
|
||||
FWUnsupported string
|
||||
FWClaims bool // the plan takes HttpPushUriTargetsBusy; fw-finish gives it back
|
||||
FWFormats []string
|
||||
FWNotes string
|
||||
FWSources []string
|
||||
Sources []string
|
||||
}
|
||||
|
||||
// redfishProfiles, most specific first: the AMI-based ones (Gigabyte, ASRock
|
||||
// Rack) also carry Oem.Ami, so they come before the generic AMI entry.
|
||||
var redfishProfiles = []redfishProfile{
|
||||
{Driver: "idrac", Title: "Dell iDRAC 8/9/10", Vendors: []string{"dell"}, OemKeys: []string{"Dell"},
|
||||
Systems: []string{"/redfish/v1/Systems/System.Embedded.1"}, BootMode: "omit",
|
||||
Sources: []string{
|
||||
"https://dl.dell.com/topicspdf/idrac9-lifecycle-controller-v33-series_api-guide2_en-us.pdf",
|
||||
"https://github.com/dell/iDRAC-Redfish-Scripting/blob/master/Redfish%20Python/SetNextOneTimeBootDeviceREDFISH.py",
|
||||
}},
|
||||
{Driver: "ilo", Title: "HPE iLO 4 (2.30+), 5, 6", Vendors: []string{"hpe", "hewlett"}, OemKeys: []string{"Hpe", "Hp"},
|
||||
Systems: []string{"/redfish/v1/Systems/1"}, BootMode: "try",
|
||||
Sources: []string{
|
||||
"https://servermanagementportal.ext.hpe.com/docs/redfishservices/ilos/ilo5/ilo5_307/ilo5_computersystem_resourcedefns307",
|
||||
"https://servermanagementportal.ext.hpe.com/docs/references_and_material/blogposts/etc/masterserverstates/masterserverstates.md",
|
||||
"https://servermanagementportal.ext.hpe.com/docs/concepts/errorresponses",
|
||||
}},
|
||||
{Driver: "xcc", Title: "Lenovo XClarity Controller (XCC, XCC2)", Vendors: []string{"lenovo"},
|
||||
Systems: []string{"/redfish/v1/Systems/1"}, BootMode: "try",
|
||||
Sources: []string{
|
||||
"https://pubs.lenovo.com/xcc-restapi/update_next_onetime_bootconfig_patch",
|
||||
"https://pubs.lenovo.com/xcc-restapi/server_reset_operations_post",
|
||||
}},
|
||||
{Driver: "supermicro", Title: "Supermicro X11 and later, H12 and later", Vendors: []string{"supermicro"}, OemKeys: []string{"Supermicro"},
|
||||
Systems: []string{"/redfish/v1/Systems/1"}, BootMode: "keep",
|
||||
Sources: []string{
|
||||
"https://www.supermicro.com/manuals/other/RedfishUserGuide.pdf",
|
||||
"https://storyboard.openstack.org/#!/story/2008252",
|
||||
}},
|
||||
{Driver: "xfusion", Title: "xFusion / Huawei FusionServer iBMC", Vendors: []string{"xfusion", "huawei"}, OemKeys: []string{"xFusion", "Huawei"},
|
||||
Systems: []string{"/redfish/v1/Systems/1"}, BootMode: "try", Auth: "session",
|
||||
Sources: []string{
|
||||
"https://www.xfusion.com/wp-content/uploads/2025/08/FusionServer-iBMC-User-Guide.pdf",
|
||||
"https://github.com/Open-xFusion/Server_Plugin_Ansible/blob/HEAD/ibmc_ansible/ibmc_redfish_api/api_manage_boot_device.py",
|
||||
"https://github.com/Huawei/Huawei-iBMC-Cmdlets/blob/HEAD/code/common/Types.ps1",
|
||||
}},
|
||||
{Driver: "kaytus", Title: "KAYTUS / Inspur (no public Redfish guide: check on hardware)", Vendors: []string{"kaytus", "inspur"}, OemKeys: []string{"Public"},
|
||||
Systems: []string{"/redfish/v1/Systems/1"}, BootMode: "try",
|
||||
Sources: []string{
|
||||
"https://github.com/ilkermanap/pyredfish/blob/HEAD/pyredfish/vendors/kaytus.py",
|
||||
"https://github.com/erelbi/kaytus-bmc-sdk",
|
||||
"https://www.commoncriteriaportal.org/files/epfiles/Kaytus%20ST%20v27a.pdf",
|
||||
}},
|
||||
{Driver: "irmc", Title: "Fujitsu / Fsas iRMC S5, S6", Vendors: []string{"fujitsu", "ts_fujitsu", "fsas"}, OemKeys: []string{"ts_fujitsu"},
|
||||
Systems: []string{"/redfish/v1/Systems/0"}, BootMode: "omit",
|
||||
// Enabled "Disabled" is refused on every firmware; 1.20 wants Continuous with None
|
||||
Clear: []obj{{"BootSourceOverrideTarget": "None"}, {"BootSourceOverrideTarget": "None", "BootSourceOverrideEnabled": "Continuous"}},
|
||||
Sources: []string{
|
||||
"https://github.com/fujitsu/ansible-irmc-integration/blob/HEAD/plugins/modules/irmc_setnextboot.py",
|
||||
"https://github.com/fujitsu/ansible-irmc-integration/blob/HEAD/plugins/modules/irmc_powerstate.py",
|
||||
}},
|
||||
{Driver: "cimc", Title: "Cisco UCS C-series CIMC (standalone)", Vendors: []string{"cisco"}, OemKeys: []string{"Cisco"},
|
||||
BootMode: "try", // the system is /Systems/<serial>: found through the collection
|
||||
Sources: []string{"https://www.cisco.com/c/en/us/td/docs/unified_computing/ucs/c/sw/api/4_3/b-cisco-ucs-c-series-servers-rest-api-programmer-s-guide-release-4-3.pdf"}},
|
||||
{Driver: "gigabyte", Title: "GIGABYTE servers (MegaRAC SP-X BMC)", Vendors: []string{"giga-byte", "gigabyte"},
|
||||
Systems: []string{"/redfish/v1/Systems/Self"}, BootMode: "keep",
|
||||
Sources: []string{"https://download.gigabyte.com/FileList/Manual/server_manual_redfish_v1.11.0.pdf"}},
|
||||
{Driver: "asrockrack", Title: "ASRock Rack (MegaRAC BMC)", Vendors: []string{"asrockrack", "asrock rack", "asrock"},
|
||||
Systems: []string{"/redfish/v1/Systems/Self"}, BootMode: "keep",
|
||||
Sources: []string{"https://bugs.launchpad.net/ironic/+bug/2073518", "https://opendev.org/openstack/ironic/commit/54977a14"}},
|
||||
{Driver: "openbmc", Title: "OpenBMC (bmcweb)", Vendors: []string{"openbmc"},
|
||||
Systems: []string{"/redfish/v1/Systems/system"}, BootMode: "try",
|
||||
Sources: []string{"https://github.com/openbmc/bmcweb/blob/master/redfish-core/lib/systems.hpp"}},
|
||||
{Driver: "ami", Title: "AMI MegaRAC SP-X (many ODM boards)", Vendors: []string{"american megatrends"}, OemKeys: []string{"Ami"},
|
||||
Systems: []string{"/redfish/v1/Systems/Self"}, BootMode: "keep",
|
||||
Sources: []string{"https://download-2.msi.com/archive/mnu_exe/server/MSI-Redfish-API-v1.0-UG-250801.pdf"}},
|
||||
{Driver: "redfish", Title: "DMTF Redfish, any vendor", BootMode: "try",
|
||||
Sources: []string{"https://www.dmtf.org/standards/redfish"}},
|
||||
}
|
||||
|
||||
func redfishProfileFor(driver string) redfishProfile {
|
||||
for _, p := range redfishProfiles {
|
||||
if p.Driver == driver {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return redfishProfiles[len(redfishProfiles)-1]
|
||||
}
|
||||
|
||||
// matchRedfishProfile picks the profile for what a service says about itself.
|
||||
func matchRedfishProfile(vendor, manufacturer string, oemKeys []string) redfishProfile {
|
||||
v := strings.ToLower(vendor + " " + manufacturer)
|
||||
for _, p := range redfishProfiles {
|
||||
for _, s := range p.Vendors {
|
||||
if strings.Contains(v, s) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
for _, k := range p.OemKeys {
|
||||
if hasTag(oemKeys, k) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
}
|
||||
return redfishProfiles[len(redfishProfiles)-1]
|
||||
}
|
||||
|
||||
func redfishDrivers() []powerDriver {
|
||||
var out []powerDriver
|
||||
for _, p := range redfishProfiles {
|
||||
if p.Driver == "redfish" { // the generic one first
|
||||
out = append([]powerDriver{{p.Driver, p.Title, "redfish", 443}}, out...)
|
||||
continue
|
||||
}
|
||||
out = append(out, powerDriver{p.Driver, p.Title + " (Redfish)", "redfish", 443})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- client
|
||||
|
||||
type rfClient struct {
|
||||
base, user, pass string
|
||||
hc *http.Client
|
||||
etags map[string]string // resource -> ETag from its last GET
|
||||
token, session string // X-Auth-Token and the session to delete, when logged in
|
||||
pushURIs map[string]bool // upload URIs the UpdateService advertised (Lenovo: /mfwupdate)
|
||||
}
|
||||
|
||||
// login opens a session (profile or extra auth=session); logout closes it.
|
||||
func (r *rfClient) login(ctx context.Context) error {
|
||||
resp, err := r.call(ctx, "POST", "/redfish/v1/SessionService/Sessions", obj{"UserName": r.user, "Password": r.pass}, "")
|
||||
if err != nil {
|
||||
return fmt.Errorf("session login: %w", err)
|
||||
}
|
||||
if r.token = resp.hdr.Get("X-Auth-Token"); r.token == "" {
|
||||
return errors.New("session login: no X-Auth-Token in the answer")
|
||||
}
|
||||
r.session = resp.hdr.Get("Location")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *rfClient) logout() {
|
||||
if r.session == "" {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
|
||||
defer cancel()
|
||||
_, _ = r.call(ctx, "DELETE", r.session, nil, "") // sessions are few: never leave one behind
|
||||
r.token, r.session = "", ""
|
||||
}
|
||||
|
||||
// open logs in when the profile or the operator wants sessions.
|
||||
func (r *rfClient) open(ctx context.Context, c *bmcCreds, prof redfishProfile) error {
|
||||
if a := c.Extra["auth"]; a == "session" || (a == "" && prof.Auth == "session") {
|
||||
return r.login(ctx)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// rfSleep waits between retries; tests make it instant.
|
||||
var rfSleep = func(ctx context.Context, d time.Duration) error {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(d):
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
func newRF(c *bmcCreds) *rfClient {
|
||||
return &rfClient{base: bmcBase(c, "https", 443), user: c.User, pass: c.Password, hc: bmcHTTPClient(c), etags: map[string]string{}}
|
||||
}
|
||||
|
||||
// resolve turns a path the BMC handed us (@odata.id, an action target, a task
|
||||
// monitor) into one on this BMC. Vendors return relative paths, full URLs and
|
||||
// trailing slashes; anything that would leave the BMC or the Redfish tree (and
|
||||
// take the credentials with it) is refused.
|
||||
func (r *rfClient) resolve(p string) (string, error) {
|
||||
if strings.Contains(p, "://") {
|
||||
u, err := url.Parse(p)
|
||||
b, _ := url.Parse(r.base)
|
||||
if err != nil || b == nil || !strings.EqualFold(u.Host, b.Host) || u.User != nil {
|
||||
return "", fmt.Errorf("the BMC pointed at %q, which is not this BMC", p)
|
||||
}
|
||||
p = u.RequestURI()
|
||||
}
|
||||
if r.pushURIs[p] && strings.HasPrefix(p, "/") && !strings.Contains(p, "..") {
|
||||
return p, nil // the BMC's own upload URI, on this BMC (outside /redfish/ on some)
|
||||
}
|
||||
if !strings.HasPrefix(p, "/redfish/") || strings.Contains(p, "/../") || strings.HasSuffix(p, "/..") {
|
||||
return "", fmt.Errorf("the BMC pointed at %q, outside /redfish/", p)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
|
||||
type rfResp struct {
|
||||
code int
|
||||
hdr http.Header
|
||||
body []byte
|
||||
}
|
||||
|
||||
// rfError is a refused request, with the registry message IDs it carried.
|
||||
type rfError struct {
|
||||
method, path string
|
||||
code int
|
||||
ids []string
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *rfError) Error() string {
|
||||
return fmt.Sprintf("%s %s: %d %s", e.method, e.path, e.code, e.msg)
|
||||
}
|
||||
|
||||
// has: one of the message IDs ends in key ("Ami.1.0.OperationSupportedInFutureStateURI").
|
||||
func (e *rfError) has(key string) bool {
|
||||
for _, id := range e.ids {
|
||||
if strings.HasSuffix(id, "."+key) || id == key {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return strings.Contains(e.msg, key)
|
||||
}
|
||||
|
||||
// succeeded: the error body says the request worked (only Success messages).
|
||||
func (e *rfError) succeeded() bool {
|
||||
if len(e.ids) == 0 {
|
||||
return false
|
||||
}
|
||||
for _, id := range e.ids {
|
||||
if !strings.HasSuffix(id, ".Success") && !strings.HasSuffix(id, "GeneralError") {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return e.has("Success") || strings.Contains(strings.ToLower(e.msg), "completed successfully")
|
||||
}
|
||||
|
||||
var retryInRe = regexp.MustCompile(`(?i)retry in (\d+) seconds?`)
|
||||
|
||||
// busy: the service will take the same request a little later.
|
||||
func (e *rfError) busy() (time.Duration, bool) {
|
||||
if m := retryInRe.FindStringSubmatch(e.msg); m != nil {
|
||||
n, _ := strconv.Atoi(m[1])
|
||||
return time.Duration(min(max(n, 1), 30)) * time.Second, true
|
||||
}
|
||||
switch {
|
||||
case e.has("UnableToModifyDuringSystemPOST"): // HPE: boot settings only when off or past POST
|
||||
return 10 * time.Second, true
|
||||
case e.code == http.StatusConflict, e.code == http.StatusServiceUnavailable: // right after a power change
|
||||
return 5 * time.Second, true
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
func asRFError(err error) *rfError {
|
||||
var e *rfError
|
||||
if errors.As(err, &e) {
|
||||
return e
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// call sends one request; ifMatch "" sends none.
|
||||
func (r *rfClient) call(ctx context.Context, method, path string, body any, ifMatch string) (*rfResp, error) {
|
||||
var rd io.Reader
|
||||
if body != nil {
|
||||
b, _ := json.Marshal(body)
|
||||
rd = bytes.NewReader(b)
|
||||
}
|
||||
path, err := r.resolve(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, method, r.base+path, rd)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if r.token != "" {
|
||||
req.Header.Set("X-Auth-Token", r.token)
|
||||
} else if r.user != "" && !(method == "POST" && strings.HasSuffix(path, "/SessionService/Sessions")) {
|
||||
req.SetBasicAuth(r.user, r.pass)
|
||||
}
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if body != nil {
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
}
|
||||
if ifMatch != "" {
|
||||
req.Header.Set("If-Match", ifMatch)
|
||||
}
|
||||
resp, err := r.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20))
|
||||
out := &rfResp{code: resp.StatusCode, hdr: resp.Header, body: data}
|
||||
if resp.StatusCode >= 300 {
|
||||
ids, msg := redfishMessage(data)
|
||||
e := &rfError{method: method, path: path, code: resp.StatusCode, ids: ids, msg: resp.Status + " " + msg}
|
||||
if resp.StatusCode == http.StatusBadRequest && method != "GET" && e.succeeded() { // KAYTUS: 400 with "completed successfully"
|
||||
return out, nil
|
||||
}
|
||||
return out, e
|
||||
}
|
||||
if method == "GET" {
|
||||
et := resp.Header.Get("ETag")
|
||||
if et == "" {
|
||||
var o struct {
|
||||
ETag json.RawMessage `json:"@odata.etag"` // iRMC: a bare number
|
||||
}
|
||||
_ = json.Unmarshal(data, &o)
|
||||
et = strings.Trim(string(o.ETag), `"`)
|
||||
if len(o.ETag) > 0 && o.ETag[0] == '"' {
|
||||
var s string
|
||||
_ = json.Unmarshal(o.ETag, &s)
|
||||
et = s
|
||||
}
|
||||
}
|
||||
if et != "" {
|
||||
r.etags[path] = et
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// do is call plus decoding; GETs record the resource's ETag.
|
||||
func (r *rfClient) do(ctx context.Context, method, path string, body, out any) (int, error) {
|
||||
resp, err := r.call(ctx, method, path, body, "")
|
||||
if err != nil {
|
||||
if resp != nil {
|
||||
return resp.code, err
|
||||
}
|
||||
return 0, err
|
||||
}
|
||||
if out != nil && len(resp.body) > 0 {
|
||||
if err := json.Unmarshal(resp.body, out); err != nil {
|
||||
return resp.code, fmt.Errorf("%s %s: %w", method, path, err)
|
||||
}
|
||||
}
|
||||
return resp.code, nil
|
||||
}
|
||||
|
||||
// redfishMessage pulls the message IDs and the human message out of an error body.
|
||||
func redfishMessage(data []byte) ([]string, string) {
|
||||
var e struct {
|
||||
Error struct {
|
||||
Code string `json:"code"`
|
||||
Message string `json:"message"`
|
||||
Ext []struct {
|
||||
ID string `json:"MessageId"`
|
||||
Message string `json:"Message"`
|
||||
} `json:"@Message.ExtendedInfo"`
|
||||
} `json:"error"`
|
||||
}
|
||||
if json.Unmarshal(data, &e) != nil {
|
||||
return nil, oneLine(string(data)) // some BMCs' web servers answer errors in HTML
|
||||
}
|
||||
var ids, msgs []string
|
||||
for _, x := range e.Error.Ext {
|
||||
if x.ID != "" {
|
||||
ids = append(ids, x.ID)
|
||||
}
|
||||
if x.Message != "" {
|
||||
msgs = append(msgs, x.Message)
|
||||
} else if x.ID != "" { // HPE often sends only the ID
|
||||
msgs = append(msgs, x.ID)
|
||||
}
|
||||
}
|
||||
if e.Error.Code != "" {
|
||||
ids = append(ids, e.Error.Code)
|
||||
}
|
||||
if len(msgs) == 0 && e.Error.Message != "" {
|
||||
msgs = append(msgs, e.Error.Message)
|
||||
}
|
||||
return ids, strings.Join(msgs, "; ")
|
||||
}
|
||||
|
||||
// patch sends a PATCH the way picky services want it (If-Match, 412, 428).
|
||||
func (r *rfClient) patch(ctx context.Context, path string, body obj) (*rfResp, error) {
|
||||
et := r.etags[path]
|
||||
resp, err := r.call(ctx, "PATCH", path, body, et)
|
||||
e := asRFError(err)
|
||||
switch {
|
||||
case e == nil:
|
||||
case e.code == http.StatusPreconditionRequired: // 428: wants If-Match (AMI)
|
||||
if _, gerr := r.call(ctx, "GET", path, nil, ""); gerr == nil && r.etags[path] != "" {
|
||||
resp, err = r.call(ctx, "PATCH", path, body, r.etags[path])
|
||||
}
|
||||
case e.code == http.StatusPreconditionFailed && et != "": // 412: weak ETag refused, or stale
|
||||
if strong := strings.TrimPrefix(et, "W/"); strong != et {
|
||||
resp, err = r.call(ctx, "PATCH", path, body, strong)
|
||||
}
|
||||
if e2 := asRFError(err); e2 != nil && e2.code == http.StatusPreconditionFailed {
|
||||
resp, err = r.call(ctx, "PATCH", path, body, "")
|
||||
}
|
||||
}
|
||||
if err == nil {
|
||||
delete(r.etags, path) // changed: never reuse it
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
// retry runs f again while the service says it is busy, for up to a minute.
|
||||
func retry(ctx context.Context, f func() error) error {
|
||||
deadline := time.Now().Add(time.Minute)
|
||||
for {
|
||||
err := f()
|
||||
e := asRFError(err)
|
||||
if e == nil {
|
||||
return err
|
||||
}
|
||||
d, ok := e.busy()
|
||||
if !ok || time.Now().Add(d).After(deadline) {
|
||||
return err
|
||||
}
|
||||
if serr := rfSleep(ctx, d); serr != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// follow waits for a 202's task (or Dell job) to finish.
|
||||
func (r *rfClient) follow(ctx context.Context, resp *rfResp) error {
|
||||
if resp == nil || resp.code != http.StatusAccepted {
|
||||
return nil
|
||||
}
|
||||
loc := resp.hdr.Get("Location")
|
||||
if loc == "" {
|
||||
var t struct {
|
||||
ID string `json:"@odata.id"`
|
||||
}
|
||||
_ = json.Unmarshal(resp.body, &t)
|
||||
loc = t.ID
|
||||
}
|
||||
if loc == "" {
|
||||
return nil
|
||||
}
|
||||
deadline := time.Now().Add(2 * time.Minute)
|
||||
for time.Now().Before(deadline) {
|
||||
res, err := r.call(ctx, "GET", loc, nil, "")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
var t obj
|
||||
_ = json.Unmarshal(res.body, &t)
|
||||
st := str(t["TaskState"]) + str(t["JobState"])
|
||||
switch st {
|
||||
case "Completed":
|
||||
return nil
|
||||
case "Exception", "Killed", "Cancelled", "Failed", "CompletedWithErrors":
|
||||
msg := ""
|
||||
if ms := toList(t["Messages"]); len(ms) > 0 {
|
||||
msg = str(asObj(ms[0])["Message"])
|
||||
}
|
||||
return fmt.Errorf("task %s: %s %s", loc, st, msg)
|
||||
}
|
||||
if res.code == http.StatusOK && st == "" { // the monitor answered with the result itself
|
||||
return nil
|
||||
}
|
||||
if err := rfSleep(ctx, 3*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return fmt.Errorf("task %s still running after 2 minutes", loc)
|
||||
}
|
||||
|
||||
// system finds the ComputerSystem: extra system=..., else the vendor's usual
|
||||
// one if it exists, else the first member of the collection.
|
||||
func (r *rfClient) system(ctx context.Context, c *bmcCreds, prof redfishProfile) (string, obj, error) {
|
||||
cands := append([]string(nil), prof.Systems...)
|
||||
if s := c.Extra["system"]; s != "" {
|
||||
cands = []string{s}
|
||||
}
|
||||
for _, p := range cands {
|
||||
var sys obj
|
||||
if _, err := r.do(ctx, "GET", p, nil, &sys); err == nil {
|
||||
return p, sys, nil
|
||||
} else if c.Extra["system"] != "" {
|
||||
return "", nil, err
|
||||
}
|
||||
}
|
||||
var coll struct {
|
||||
Members []struct {
|
||||
ID string `json:"@odata.id"`
|
||||
} `json:"Members"`
|
||||
}
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/Systems", nil, &coll); err != nil {
|
||||
return "", nil, err
|
||||
}
|
||||
if len(coll.Members) == 0 {
|
||||
return "", nil, errors.New("the BMC lists no systems")
|
||||
}
|
||||
p := strings.TrimRight(coll.Members[0].ID, "/") // iLO lists members with a trailing slash
|
||||
var sys obj
|
||||
_, err := r.do(ctx, "GET", p, nil, &sys)
|
||||
return p, sys, err
|
||||
}
|
||||
|
||||
// resetTypes are the ResetType values the system accepts.
|
||||
func (r *rfClient) resetTypes(ctx context.Context, sys obj) []string {
|
||||
act := asObj(dig(sys, "Actions", "#ComputerSystem.Reset"))
|
||||
var out []string
|
||||
for _, v := range toList(act["ResetType@Redfish.AllowableValues"]) {
|
||||
out = append(out, str(v))
|
||||
}
|
||||
if len(out) == 0 {
|
||||
if info := str(act["@Redfish.ActionInfo"]); info != "" {
|
||||
var ai struct {
|
||||
Parameters []struct {
|
||||
Name string `json:"Name"`
|
||||
AllowableValues []string `json:"AllowableValues"`
|
||||
} `json:"Parameters"`
|
||||
}
|
||||
if _, err := r.do(ctx, "GET", info, nil, &ai); err == nil {
|
||||
for _, p := range ai.Parameters {
|
||||
if p.Name == "ResetType" {
|
||||
out = p.AllowableValues
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func pickReset(allowed []string, prefs ...string) string {
|
||||
if len(allowed) == 0 {
|
||||
return prefs[0]
|
||||
}
|
||||
for _, p := range prefs {
|
||||
if hasTag(allowed, p) {
|
||||
return p
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func strList(v any) []string {
|
||||
var out []string
|
||||
for _, x := range toList(v) {
|
||||
out = append(out, str(x))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var redfishBootTargets = map[string]string{"pxe": "Pxe", "disk": "Hdd", "bios": "BiosSetup", "cdrom": "Cd", "none": "", "safe": ""}
|
||||
|
||||
// bootOverride sets the next boot (target "" clears the override).
|
||||
func (r *rfClient) bootOverride(ctx context.Context, c *bmcCreds, prof redfishProfile, path string, sys obj, target string) error {
|
||||
boot := asObj(sys["Boot"])
|
||||
if allowed := strList(boot["BootSourceOverrideTarget@Redfish.AllowableValues"]); target != "" && len(allowed) > 0 && !hasTag(allowed, target) {
|
||||
return fmt.Errorf("the BMC offers boot targets %s, not %s", strings.Join(allowed, ", "), target)
|
||||
}
|
||||
if target == "" {
|
||||
clears := prof.Clear
|
||||
if len(clears) == 0 {
|
||||
clears = []obj{{"BootSourceOverrideEnabled": "Disabled"}}
|
||||
}
|
||||
var err error
|
||||
for _, b := range clears {
|
||||
if err = retry(ctx, func() error { _, err := r.patch(ctx, path, obj{"Boot": b}); return err }); err == nil {
|
||||
return nil
|
||||
}
|
||||
if _, gerr := r.call(ctx, "GET", path, nil, ""); gerr != nil { // fresh ETag for the next try
|
||||
return err
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
body := obj{"BootSourceOverrideTarget": target, "BootSourceOverrideEnabled": "Once"}
|
||||
mode, asked := str(boot["BootSourceOverrideMode"]), c.Extra["boot_mode"] // UEFI | Legacy
|
||||
if asked != "" {
|
||||
mode = asked
|
||||
}
|
||||
withMode := mode != "" && (prof.BootMode != "omit" || asked != "")
|
||||
full := obj{}
|
||||
for k, v := range body {
|
||||
full[k] = v
|
||||
}
|
||||
if withMode {
|
||||
full["BootSourceOverrideMode"] = mode
|
||||
}
|
||||
send := func(p string, b obj) error {
|
||||
return retry(ctx, func() error {
|
||||
resp, err := r.patch(ctx, p, obj{"Boot": b})
|
||||
if err == nil {
|
||||
err = r.follow(ctx, resp)
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
err := send(path, full)
|
||||
e := asRFError(err)
|
||||
if e != nil && e.has("OperationSupportedInFutureStateURI") { // AMI: pending settings live in .../SD
|
||||
sd := path + "/SD"
|
||||
if _, gerr := r.call(ctx, "GET", sd, nil, ""); gerr == nil {
|
||||
err = send(sd, full)
|
||||
}
|
||||
e = asRFError(err)
|
||||
}
|
||||
if settings := str(dig(sys, "@Redfish.Settings", "SettingsObject", "@odata.id")); e != nil && settings != "" &&
|
||||
(e.code == http.StatusBadRequest || e.code == http.StatusMethodNotAllowed) { // e.g. iDRAC 10: .../Settings
|
||||
if _, gerr := r.call(ctx, "GET", settings, nil, ""); gerr == nil {
|
||||
err = send(settings, full)
|
||||
}
|
||||
e = asRFError(err)
|
||||
}
|
||||
if e != nil && withMode && prof.BootMode == "try" && asked == "" && e.code == http.StatusBadRequest {
|
||||
err = send(path, body) // some refuse the mode in the body (HPE DL160 Gen10)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func redfishOp(ctx context.Context, c *bmcCreds, op, arg string) (string, error) {
|
||||
prof := redfishProfileFor(c.Driver)
|
||||
r := newRF(c)
|
||||
if err := r.open(ctx, c, prof); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer r.logout()
|
||||
path, sys, err := r.system(ctx, c, prof)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
powerState := func(sys obj) string { return strings.ToLower(str(sys["PowerState"])) } // On | Off | PoweringOn | PoweringOff | Reset (iLO, briefly)
|
||||
on := powerState(sys) == "on" || powerState(sys) == "poweringon"
|
||||
allowed := r.resetTypes(ctx, sys)
|
||||
target := str(dig(sys, "Actions", "#ComputerSystem.Reset", "target"))
|
||||
if target == "" && dig(sys, "Actions", "#ComputerSystem.Reset") == nil {
|
||||
// no reset on the system (seen on KAYTUS): the chassis' one, if it has
|
||||
if ch := toList(dig(sys, "Links", "Chassis")); len(ch) > 0 {
|
||||
var chassis obj
|
||||
if _, err := r.do(ctx, "GET", str(asObj(ch[0])["@odata.id"]), nil, &chassis); err == nil {
|
||||
if t := str(dig(chassis, "Actions", "#Chassis.Reset", "target")); t != "" {
|
||||
target = t
|
||||
allowed = strList(dig(chassis, "Actions", "#Chassis.Reset", "ResetType@Redfish.AllowableValues"))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if target == "" {
|
||||
target = path + "/Actions/ComputerSystem.Reset"
|
||||
}
|
||||
post := func(t string) error {
|
||||
return retry(ctx, func() error {
|
||||
resp, err := r.call(ctx, "POST", target, obj{"ResetType": t}, "")
|
||||
if err == nil {
|
||||
err = r.follow(ctx, resp)
|
||||
}
|
||||
return err
|
||||
})
|
||||
}
|
||||
reset := func(prefs ...string) (string, error) {
|
||||
t := pickReset(allowed, prefs...)
|
||||
if t == "" {
|
||||
return "", fmt.Errorf("the BMC accepts none of %v (it offers %v)", prefs, allowed)
|
||||
}
|
||||
if err := post(t); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "Chassis Power Control: " + t + " (redfish)", nil
|
||||
}
|
||||
// waitOff: ForceOff is not instant; On before the system is off is refused
|
||||
waitOff := func() error {
|
||||
for i := 0; i < 30; i++ {
|
||||
var s obj
|
||||
if _, err := r.do(ctx, "GET", path, nil, &s); err == nil && powerState(s) == "off" {
|
||||
return nil
|
||||
}
|
||||
if err := rfSleep(ctx, 2*time.Second); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return errors.New("the system did not power off within a minute")
|
||||
}
|
||||
switch op {
|
||||
case "status":
|
||||
if on {
|
||||
return "Chassis Power is on", nil
|
||||
}
|
||||
return "Chassis Power is off", nil
|
||||
case "on":
|
||||
if on {
|
||||
return "Chassis Power is on already", nil
|
||||
}
|
||||
return reset("On", "ForceOn", "PushPowerButton")
|
||||
case "off":
|
||||
if !on {
|
||||
return "Chassis Power is off already", nil
|
||||
}
|
||||
return reset("ForceOff", "PushPowerButton")
|
||||
case "soft":
|
||||
if !on {
|
||||
return "Chassis Power is off already", nil
|
||||
}
|
||||
return reset("GracefulShutdown", "PushPowerButton")
|
||||
case "cycle", "reset":
|
||||
if !on {
|
||||
return reset("On", "ForceOn", "PushPowerButton")
|
||||
}
|
||||
prefs := []string{"PowerCycle", "ForcePowerCycle", "ForceRestart", "GracefulRestart"} // ForcePowerCycle: iBMC
|
||||
if op == "reset" {
|
||||
prefs = []string{"ForceRestart", "PowerCycle", "ForcePowerCycle", "GracefulRestart"}
|
||||
}
|
||||
if pickReset(allowed, prefs...) == "" && hasTag(allowed, "ForceOff") && hasTag(allowed, "On") {
|
||||
if err := post("ForceOff"); err != nil { // offered no restart at all (some AMI builds)
|
||||
return "", err
|
||||
}
|
||||
if err := waitOff(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if err := post("On"); err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "Chassis Power Control: ForceOff, On (redfish)", nil
|
||||
}
|
||||
return reset(prefs...)
|
||||
case "bootdev":
|
||||
t := redfishBootTargets[arg]
|
||||
if err := r.bootOverride(ctx, c, prof, path, sys, t); err != nil {
|
||||
return "", err
|
||||
}
|
||||
if t == "" {
|
||||
return "Boot override cleared (redfish)", nil
|
||||
}
|
||||
return "Set Boot Device to " + arg + " (redfish " + t + ", once)", nil
|
||||
}
|
||||
return "", fmt.Errorf("redfish: %s is not supported", op)
|
||||
}
|
||||
|
||||
// redfishProbe asks whether the BMC speaks Redfish with these credentials,
|
||||
// which vendor profile fits, and what the system offers. The answer is JSON
|
||||
// and is kept on the BMC record (shown in the UI).
|
||||
func redfishProbe(ctx context.Context, c *bmcCreds) (string, error) {
|
||||
r := newRF(c)
|
||||
var root obj
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/", nil, &root); err != nil { // the service root needs no login
|
||||
return "", fmt.Errorf("no Redfish service: %w", err)
|
||||
}
|
||||
var oemKeys []string
|
||||
for k := range asObj(root["Oem"]) {
|
||||
oemKeys = append(oemKeys, k)
|
||||
}
|
||||
vendor := str(root["Vendor"])
|
||||
if vendor == "" && len(oemKeys) > 0 {
|
||||
vendor = oemKeys[0]
|
||||
}
|
||||
res := obj{"redfish": str(root["RedfishVersion"]), "vendor": vendor}
|
||||
// the system is looked up with the profile the root suggests; the
|
||||
// manufacturer then settles it
|
||||
guess := matchRedfishProfile(vendor, "", oemKeys)
|
||||
if err := r.open(ctx, c, guess); err != nil {
|
||||
return "", fmt.Errorf("Redfish %s answers, but not to these credentials: %w", res["redfish"], err)
|
||||
}
|
||||
defer r.logout()
|
||||
path, sys, err := r.system(ctx, &bmcCreds{Extra: c.Extra}, guess)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("Redfish %s answers, but not to these credentials: %w", res["redfish"], err)
|
||||
}
|
||||
prof := matchRedfishProfile(vendor, str(sys["Manufacturer"]), oemKeys)
|
||||
boot := asObj(sys["Boot"])
|
||||
res["system"], res["manufacturer"], res["model"], res["power"] = path, str(sys["Manufacturer"]), str(sys["Model"]), str(sys["PowerState"])
|
||||
res["driver"], res["profile"] = prof.Driver, prof.Title
|
||||
if prof.Auth == "session" {
|
||||
res["auth"] = "session"
|
||||
}
|
||||
res["reset_types"] = r.resetTypes(ctx, sys)
|
||||
res["boot_targets"] = strList(boot["BootSourceOverrideTarget@Redfish.AllowableValues"])
|
||||
res["boot_mode"] = str(boot["BootSourceOverrideMode"])
|
||||
b, _ := json.Marshal(res)
|
||||
return string(b), nil
|
||||
}
|
||||
@@ -0,0 +1,666 @@
|
||||
package main
|
||||
|
||||
// Firmware over Redfish (UpdateService), run where the BMC is reachable (the
|
||||
// rack controller). Three operations, each short enough for one rack command:
|
||||
//
|
||||
// fw-inventory what is installed (FirmwareInventory)
|
||||
// fw-start hand a package to the BMC; returns the task to watch
|
||||
// fw-poll one look at that task
|
||||
//
|
||||
// The package reaches the BMC one of three ways, chosen from what the
|
||||
// UpdateService offers and the vendor profile's order:
|
||||
//
|
||||
// multipart POST MultipartHttpPushUri: UpdateParameters (JSON) + UpdateFile
|
||||
// simple UpdateService.SimpleUpdate with an ImageURI the BMC downloads
|
||||
// from the rack controller (or firmware_base)
|
||||
// push POST HttpPushUri with the file as the body
|
||||
//
|
||||
// The rack controller fetches the package from its own cache (filled from the
|
||||
// region) and sends it with a known length: many BMC web servers refuse
|
||||
// chunked uploads.
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"mime/multipart"
|
||||
"net/http"
|
||||
"net/textproto"
|
||||
"os"
|
||||
"path"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// fwComponent is one entry of a machine's firmware inventory.
|
||||
type fwComponent struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Version string `json:"version"`
|
||||
Component string `json:"component"` // bios | bmc | nic | raid | cpld | psu | drive | other
|
||||
Updateable bool `json:"updateable"`
|
||||
SoftwareID string `json:"software_id,omitempty"`
|
||||
}
|
||||
|
||||
// fwStartArg is what fw-start receives.
|
||||
type fwStartArg struct {
|
||||
URL string `json:"url"` // where the package is (the BMC pulls it, or the rack downloads it to push)
|
||||
Component string `json:"component"` // bios | bmc | ... (vendor plans pick targets by it)
|
||||
File string `json:"file"` // its file name
|
||||
Method string `json:"method,omitempty"` // multipart | simple | push; "" = profile order
|
||||
Apply string `json:"apply,omitempty"` // Immediate | OnReset; "" = the BMC's default
|
||||
Targets []string `json:"targets,omitempty"` // inventory resources to update; usually empty
|
||||
}
|
||||
|
||||
type fwStartResult struct {
|
||||
Method string `json:"method"`
|
||||
Task string `json:"task,omitempty"` // task monitor or task resource; "" when the BMC answered synchronously
|
||||
Note string `json:"note,omitempty"`
|
||||
}
|
||||
|
||||
// fwTimeouts bound each firmware operation; an upload to a slow BMC takes long.
|
||||
var fwTimeouts = map[string]time.Duration{"fw-inventory": 5 * time.Minute, "fw-start": 45 * time.Minute, "fw-poll": 2 * time.Minute, "fw-finish": time.Minute}
|
||||
|
||||
type fwPollResult struct {
|
||||
State string `json:"state"` // running | done | failed | unknown
|
||||
Pct int `json:"pct"`
|
||||
Msg string `json:"msg,omitempty"`
|
||||
}
|
||||
|
||||
// fwKinds sorts inventory entries by id and name, most specific first: a RAID
|
||||
// "controller" is not the management controller.
|
||||
var fwKinds = []struct {
|
||||
kind string
|
||||
re *regexp.Regexp
|
||||
}{
|
||||
{"bios", regexp.MustCompile(`(?i)\b(bios|uefi|system rom)\b`)},
|
||||
{"raid", regexp.MustCompile(`(?i)\b(raid|perc|smart ?array|megaraid|hba|storage controller|boss)\b`)},
|
||||
{"nic", regexp.MustCompile(`(?i)\b(nic|ethernet|network|connectx|mellanox|infiniband|e810|x710|bcm\d*|broadcom adv)`)},
|
||||
{"psu", regexp.MustCompile(`(?i)\b(psu|power supply)\b`)},
|
||||
{"cpld", regexp.MustCompile(`(?i)\b(cpld|fpga)\b`)},
|
||||
{"drive", regexp.MustCompile(`(?i)\b(drive|disk|ssd|nvme|hdd)\b`)},
|
||||
{"bmc", regexp.MustCompile(`(?i)\b(bmc|idrac|ilo ?\d?|xcc2?|imm2?|ibmc|irmc|cimc|remote access controller|management controller|baseboard management)\b`)},
|
||||
}
|
||||
|
||||
// fwKind sorts an inventory entry into a component class by its id and name.
|
||||
func fwKind(id, name string) string {
|
||||
s := id + " " + name
|
||||
for _, k := range fwKinds {
|
||||
if k.re.MatchString(s) {
|
||||
return k.kind
|
||||
}
|
||||
}
|
||||
return "other"
|
||||
}
|
||||
|
||||
func firmwareRun(ctx context.Context, c *bmcCreds, op, arg string) (string, error) {
|
||||
d, _ := driverByName(c.Driver)
|
||||
if d.Family != "redfish" {
|
||||
return "", fmt.Errorf("firmware updates need a Redfish driver (this BMC uses %s)", orDefault(c.Driver, "ipmi"))
|
||||
}
|
||||
prof := redfishProfileFor(c.Driver)
|
||||
r := newRF(c)
|
||||
r.hc.Timeout = 0 // uploads take minutes; the context bounds each call
|
||||
if err := r.open(ctx, c, prof); err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer r.logout()
|
||||
var out any
|
||||
var err error
|
||||
switch op {
|
||||
case "fw-inventory":
|
||||
out, err = r.fwInventory(ctx)
|
||||
case "fw-start":
|
||||
var a fwStartArg
|
||||
if err := json.Unmarshal([]byte(arg), &a); err != nil {
|
||||
return "", fmt.Errorf("fw-start: %w", err)
|
||||
}
|
||||
out, err = r.fwStart(ctx, c, prof, a)
|
||||
case "fw-poll":
|
||||
out, err = r.fwPoll(ctx, arg)
|
||||
case "fw-finish": // after the task ended: give back what fw-start claimed
|
||||
if prof.FWClaims {
|
||||
r.fwRelease(ctx)
|
||||
}
|
||||
out = obj{"ok": true}
|
||||
default:
|
||||
return "", fmt.Errorf("unknown firmware operation %q", op)
|
||||
}
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
b, _ := json.Marshal(out)
|
||||
return string(b), nil
|
||||
}
|
||||
|
||||
func (r *rfClient) fwInventory(ctx context.Context) ([]fwComponent, error) {
|
||||
var coll struct {
|
||||
Members []struct {
|
||||
ID string `json:"@odata.id"`
|
||||
} `json:"Members"`
|
||||
}
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService/FirmwareInventory", nil, &coll); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := []fwComponent{}
|
||||
for i, mem := range coll.Members {
|
||||
if i >= 300 {
|
||||
break
|
||||
}
|
||||
id := path.Base(strings.TrimRight(mem.ID, "/"))
|
||||
// Dell lists what is installed and what was before or is staged
|
||||
if strings.HasPrefix(id, "Previous-") || strings.HasPrefix(id, "Available-") {
|
||||
continue
|
||||
}
|
||||
var e obj
|
||||
if _, err := r.do(ctx, "GET", mem.ID, nil, &e); err != nil {
|
||||
continue
|
||||
}
|
||||
up, _ := e["Updateable"].(bool)
|
||||
fc := fwComponent{ID: str(e["Id"]), Name: str(e["Name"]), Version: str(e["Version"]), Updateable: up, SoftwareID: str(e["SoftwareId"])}
|
||||
if fc.ID == "" {
|
||||
fc.ID = id
|
||||
}
|
||||
fc.Component = fwKind(fc.ID, fc.Name)
|
||||
out = append(out, fc)
|
||||
}
|
||||
sort.Slice(out, func(i, j int) bool {
|
||||
if out[i].Component != out[j].Component {
|
||||
return out[i].Component < out[j].Component
|
||||
}
|
||||
return out[i].ID < out[j].ID
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// fwCaps is what an UpdateService offers.
|
||||
type fwCaps struct {
|
||||
simple, multipart, push string
|
||||
protocols []string
|
||||
}
|
||||
|
||||
func (r *rfClient) fwCaps(ctx context.Context) (fwCaps, error) {
|
||||
var us obj
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err != nil {
|
||||
return fwCaps{}, err
|
||||
}
|
||||
act := asObj(dig(us, "Actions", "#UpdateService.SimpleUpdate"))
|
||||
if r.pushURIs == nil {
|
||||
r.pushURIs = map[string]bool{}
|
||||
}
|
||||
for _, k := range []string{"MultipartHttpPushUri", "HttpPushUri"} {
|
||||
if u := str(us[k]); u != "" && !strings.Contains(u, "://") {
|
||||
r.pushURIs[u] = true
|
||||
}
|
||||
}
|
||||
return fwCaps{
|
||||
simple: str(act["target"]),
|
||||
protocols: strList(act["TransferProtocol@Redfish.AllowableValues"]),
|
||||
multipart: str(us["MultipartHttpPushUri"]),
|
||||
push: str(us["HttpPushUri"]),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// fwRequest is one way to hand a package over, as a vendor documents it.
|
||||
type fwRequest struct {
|
||||
Method string // multipart | simple | push: how this request carries the package
|
||||
URI string // where it goes
|
||||
AltURI string // tried when URI answers 404 (paths that moved between firmware generations)
|
||||
Params obj // multipart UpdateParameters, or the JSON body of a simple request
|
||||
NoParams bool // multipart without an UpdateParameters part
|
||||
Parts []fwPart // more parts before the file (AMI OemParameters)
|
||||
Field string // the file's form field (default UpdateFile)
|
||||
Form map[string]string // text fields after the file
|
||||
FailText string // a 2xx answer that contains this is a refusal (KAYTUS "Verification Failed")
|
||||
Claim bool // take HttpPushUriTargetsBusy first (Lenovo); fw-finish gives it back
|
||||
Then *fwRequest // a second call naming the uploaded file (xFusion, KAYTUS)
|
||||
Tasks []string // fixed tasks to watch when the answers name none (KAYTUS)
|
||||
}
|
||||
|
||||
type fwPart struct {
|
||||
Name string
|
||||
Body obj
|
||||
}
|
||||
|
||||
// fwPlanCtx is what a vendor plan decides from.
|
||||
type fwPlanCtx struct {
|
||||
Component, File, URL, Apply, Password string
|
||||
Vendor string // ServiceRoot Vendor
|
||||
Caps fwCaps // what the UpdateService offers
|
||||
Inventory []string // FirmwareInventory member ids
|
||||
}
|
||||
|
||||
func (x fwPlanCtx) has(id string) bool { return hasTag(x.Inventory, id) }
|
||||
|
||||
func (x fwPlanCtx) first(ids ...string) string {
|
||||
for _, id := range ids {
|
||||
if x.has(id) {
|
||||
return id
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func (x fwPlanCtx) applyOr(def string) string { return orDefault(x.Apply, def) }
|
||||
|
||||
func (r *rfClient) fwStart(ctx context.Context, c *bmcCreds, prof redfishProfile, a fwStartArg) (*fwStartResult, error) {
|
||||
caps, err := r.fwCaps(ctx)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("no UpdateService: %w", err)
|
||||
}
|
||||
forced := orDefault(a.Method, c.Extra["fw_method"])
|
||||
if prof.FWUnsupported != "" && forced == "" {
|
||||
return nil, errors.New(prof.FWUnsupported + " (to try a standard method anyway: -method multipart|simple|push)")
|
||||
}
|
||||
if prof.FWPlan != nil && forced == "" {
|
||||
x := fwPlanCtx{Component: a.Component, File: a.File, URL: a.URL, Apply: a.Apply, Password: r.pass, Caps: caps}
|
||||
var root obj
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/", nil, &root); err == nil {
|
||||
x.Vendor = str(root["Vendor"])
|
||||
}
|
||||
var coll struct {
|
||||
Members []struct {
|
||||
ID string `json:"@odata.id"`
|
||||
} `json:"Members"`
|
||||
}
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService/FirmwareInventory", nil, &coll); err == nil {
|
||||
for _, m := range coll.Members {
|
||||
x.Inventory = append(x.Inventory, path.Base(strings.TrimRight(m.ID, "/")))
|
||||
}
|
||||
}
|
||||
req, err := prof.FWPlan(x)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if raw := c.Extra["fw_params"]; raw != "" { // the operator's own UpdateParameters / body
|
||||
var o obj
|
||||
if err := json.Unmarshal([]byte(raw), &o); err != nil {
|
||||
return nil, fmt.Errorf("fw_params is not JSON: %w", err)
|
||||
}
|
||||
req.Params = o
|
||||
}
|
||||
return r.fwExec(ctx, req, a)
|
||||
}
|
||||
order := prof.FWMethods
|
||||
if len(order) == 0 {
|
||||
order = []string{"multipart", "simple", "push"}
|
||||
}
|
||||
if forced != "" {
|
||||
order = []string{forced}
|
||||
}
|
||||
var tried []string
|
||||
for _, m := range order {
|
||||
req := fwStandard(caps, m, a)
|
||||
if req == nil {
|
||||
continue
|
||||
}
|
||||
res, err := r.fwExec(ctx, req, a)
|
||||
if err == nil {
|
||||
return res, nil
|
||||
}
|
||||
// a BMC that does not like this way gets the next one; a refused
|
||||
// package (wrong model, signature, downgrade) stops here
|
||||
if e := asRFError(err); e == nil || (e.code != http.StatusMethodNotAllowed && e.code != http.StatusNotImplemented && e.code != http.StatusUnsupportedMediaType) {
|
||||
return nil, fmt.Errorf("%s: %w", m, err)
|
||||
}
|
||||
tried = append(tried, m+": "+err.Error())
|
||||
}
|
||||
if len(tried) > 0 {
|
||||
return nil, fmt.Errorf("the BMC took no upload method: %s", strings.Join(tried, "; "))
|
||||
}
|
||||
return nil, fmt.Errorf("the BMC's UpdateService offers none of %v", order)
|
||||
}
|
||||
|
||||
// fwStandard is the DMTF request for a method, nil when the service lacks it.
|
||||
func fwStandard(caps fwCaps, method string, a fwStartArg) *fwRequest {
|
||||
switch method {
|
||||
case "multipart":
|
||||
if caps.multipart == "" {
|
||||
return nil
|
||||
}
|
||||
params := obj{"Targets": orEmpty(a.Targets)}
|
||||
if a.Apply != "" {
|
||||
params["@Redfish.OperationApplyTime"] = a.Apply
|
||||
}
|
||||
return &fwRequest{Method: "multipart", URI: caps.multipart, Params: params}
|
||||
case "simple":
|
||||
if caps.simple == "" {
|
||||
return nil
|
||||
}
|
||||
body := obj{"ImageURI": a.URL}
|
||||
if len(caps.protocols) > 0 { // only when the service lists them: others want the scheme in the URI
|
||||
body["TransferProtocol"] = strings.ToUpper(strings.SplitN(a.URL, "://", 2)[0])
|
||||
}
|
||||
if len(a.Targets) > 0 {
|
||||
body["Targets"] = a.Targets
|
||||
}
|
||||
return &fwRequest{Method: "simple", URI: caps.simple, Params: body}
|
||||
case "push":
|
||||
if caps.push == "" {
|
||||
return nil
|
||||
}
|
||||
return &fwRequest{Method: "push", URI: caps.push}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// fwExec carries out a request (and its follow-up) and names the task to watch.
|
||||
func (r *rfClient) fwExec(ctx context.Context, req *fwRequest, a fwStartArg) (*fwStartResult, error) {
|
||||
if req.Claim {
|
||||
var us obj
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if us["HttpPushUriTargetsBusy"] == true {
|
||||
return nil, errors.New("the BMC's update service is busy with another upload (HttpPushUriTargetsBusy)")
|
||||
}
|
||||
if _, err := r.patch(ctx, "/redfish/v1/UpdateService", obj{"HttpPushUriTargetsBusy": true}); err != nil {
|
||||
return nil, fmt.Errorf("claim the update service: %w", err)
|
||||
}
|
||||
}
|
||||
resp, err := r.fwSend(ctx, req, a)
|
||||
if err != nil {
|
||||
if req.Claim {
|
||||
r.fwRelease(ctx)
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
res := &fwStartResult{Method: req.Method, Task: fwTaskOf(resp)}
|
||||
if req.Then != nil {
|
||||
tresp, err := r.fwSend(ctx, req.Then, a)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the BMC has the file, but did not start the update: %w", err)
|
||||
}
|
||||
res.Task = fwTaskOf(tresp)
|
||||
}
|
||||
if res.Task == "" && len(req.Tasks) > 0 {
|
||||
res.Task = strings.Join(req.Tasks, ",")
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (r *rfClient) fwRelease(ctx context.Context) {
|
||||
_, _ = r.call(ctx, "GET", "/redfish/v1/UpdateService", nil, "")
|
||||
_, _ = r.patch(ctx, "/redfish/v1/UpdateService", obj{"HttpPushUriTargetsBusy": false})
|
||||
}
|
||||
|
||||
func (r *rfClient) fwSend(ctx context.Context, req *fwRequest, a fwStartArg) (*rfResp, error) {
|
||||
var resp *rfResp
|
||||
var err error
|
||||
send := func(uri string) (*rfResp, error) {
|
||||
switch req.Method {
|
||||
case "simple":
|
||||
if proto := str(req.Params["TransferProtocol"]); proto != "" && len(a.URL) > 0 {
|
||||
// checked here so a service that pulls only over NFS/SCP says so clearly
|
||||
if caps, cerr := r.fwCaps(ctx); cerr == nil && len(caps.protocols) > 0 && !hasTag(caps.protocols, proto) {
|
||||
return nil, &rfError{method: "POST", path: uri, code: http.StatusNotImplemented, msg: fmt.Sprintf("transfer protocols %v, not %s", caps.protocols, proto)}
|
||||
}
|
||||
}
|
||||
return r.call(ctx, "POST", uri, req.Params, "")
|
||||
case "multipart", "push":
|
||||
f, size, ferr := fwFetch(ctx, a.URL)
|
||||
if ferr != nil {
|
||||
return nil, ferr
|
||||
}
|
||||
defer os.Remove(f.Name())
|
||||
defer f.Close()
|
||||
if req.Method == "push" {
|
||||
return r.upload(ctx, uri, "application/octet-stream", f, size, r.etags[uri])
|
||||
}
|
||||
return r.multipartUpload(ctx, uri, req, a.File, f, size)
|
||||
}
|
||||
return nil, fmt.Errorf("unknown method %q", req.Method)
|
||||
}
|
||||
resp, err = send(req.URI)
|
||||
if e := asRFError(err); e != nil && e.code == http.StatusNotFound && req.AltURI != "" {
|
||||
resp, err = send(req.AltURI)
|
||||
}
|
||||
if err == nil && req.FailText != "" && strings.Contains(string(resp.body), req.FailText) {
|
||||
return nil, fmt.Errorf("the BMC refused the package: %s", req.FailText)
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func orEmpty(s []string) []string {
|
||||
if s == nil {
|
||||
return []string{}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// fwTaskOf finds the task in an answer: Location, a Task body, or a Dell job id.
|
||||
func fwTaskOf(resp *rfResp) string {
|
||||
if resp == nil {
|
||||
return ""
|
||||
}
|
||||
if l := resp.hdr.Get("Location"); l != "" {
|
||||
return l
|
||||
}
|
||||
var t struct {
|
||||
ID string `json:"@odata.id"`
|
||||
Type string `json:"@odata.type"`
|
||||
}
|
||||
if json.Unmarshal(resp.body, &t) == nil && strings.Contains(t.Type, "Task") {
|
||||
return t.ID
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// fwFetch downloads the package to a temporary file (the rack's cache serves it).
|
||||
func fwFetch(ctx context.Context, url string) (*os.File, int64, error) {
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", url, nil)
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, 0, fmt.Errorf("package: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, 0, fmt.Errorf("package %s: %s", url, resp.Status)
|
||||
}
|
||||
f, err := os.CreateTemp("", "rollcall-fw-*")
|
||||
if err != nil {
|
||||
return nil, 0, err
|
||||
}
|
||||
n, err := io.Copy(f, resp.Body)
|
||||
if err == nil {
|
||||
_, err = f.Seek(0, io.SeekStart)
|
||||
}
|
||||
if err != nil {
|
||||
f.Close()
|
||||
os.Remove(f.Name())
|
||||
return nil, 0, err
|
||||
}
|
||||
return f, n, nil
|
||||
}
|
||||
|
||||
// upload POSTs a body of known length (no chunked encoding).
|
||||
func (r *rfClient) upload(ctx context.Context, target, ctype string, body io.Reader, size int64, ifMatch string) (*rfResp, error) {
|
||||
p, err := r.resolve(target)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", r.base+p, body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.ContentLength = size
|
||||
req.Header.Set("Content-Type", ctype)
|
||||
req.Header.Set("Accept", "application/json")
|
||||
if ifMatch != "" {
|
||||
req.Header.Set("If-Match", ifMatch)
|
||||
}
|
||||
if r.token != "" {
|
||||
req.Header.Set("X-Auth-Token", r.token)
|
||||
} else if r.user != "" {
|
||||
req.SetBasicAuth(r.user, r.pass)
|
||||
}
|
||||
resp, err := r.hc.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, _ := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
||||
out := &rfResp{code: resp.StatusCode, hdr: resp.Header, body: data}
|
||||
if resp.StatusCode >= 300 {
|
||||
ids, msg := redfishMessage(data)
|
||||
return out, &rfError{method: "POST", path: p, code: resp.StatusCode, ids: ids, msg: resp.Status + " " + msg}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// multipartUpload sends UpdateParameters, any OEM parts, the file and text
|
||||
// fields (DSP0266 "multipart HTTP push", plus what vendors add to it).
|
||||
func (r *rfClient) multipartUpload(ctx context.Context, target string, req *fwRequest, name string, f io.Reader, size int64) (*rfResp, error) {
|
||||
var head, after bytes.Buffer
|
||||
mw := multipart.NewWriter(&head)
|
||||
jsonPart := func(n string, v any) {
|
||||
h := textproto.MIMEHeader{}
|
||||
h.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q`, n))
|
||||
h.Set("Content-Type", "application/json")
|
||||
w, _ := mw.CreatePart(h)
|
||||
b, _ := json.Marshal(v)
|
||||
_, _ = w.Write(b)
|
||||
}
|
||||
if !req.NoParams {
|
||||
p := req.Params
|
||||
if p == nil {
|
||||
p = obj{"Targets": []string{}}
|
||||
}
|
||||
jsonPart("UpdateParameters", p)
|
||||
}
|
||||
for _, p := range req.Parts {
|
||||
jsonPart(p.Name, p.Body)
|
||||
}
|
||||
fh := textproto.MIMEHeader{}
|
||||
fh.Set("Content-Disposition", fmt.Sprintf(`form-data; name=%q; filename=%q`, orDefault(req.Field, "UpdateFile"), path.Base(name)))
|
||||
fh.Set("Content-Type", "application/octet-stream")
|
||||
_, _ = mw.CreatePart(fh)
|
||||
// text fields and the closing boundary come after the file
|
||||
keys := make([]string, 0, len(req.Form))
|
||||
for k := range req.Form {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
fmt.Fprintf(&after, "\r\n--%s\r\nContent-Disposition: form-data; name=%q\r\n\r\n%s", mw.Boundary(), k, req.Form[k])
|
||||
}
|
||||
fmt.Fprintf(&after, "\r\n--%s--\r\n", mw.Boundary())
|
||||
body := io.MultiReader(&head, f, &after)
|
||||
return r.upload(ctx, target, mw.FormDataContentType(), body, int64(head.Len())+size+int64(after.Len()), r.etags[target])
|
||||
}
|
||||
|
||||
// fwPoll looks at the task(s) of an update; several (comma-separated) are
|
||||
// watched in order and count as done when all are.
|
||||
func (r *rfClient) fwPoll(ctx context.Context, tasks string) (*fwPollResult, error) {
|
||||
if tasks == "" {
|
||||
return &fwPollResult{State: "unknown"}, nil
|
||||
}
|
||||
var last *fwPollResult
|
||||
seen := false
|
||||
for _, t := range strings.Split(tasks, ",") {
|
||||
res, err := r.fwPollOne(ctx, t)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if res.State == "unknown" { // not (or no longer) listed: the others decide
|
||||
continue
|
||||
}
|
||||
seen = true
|
||||
if res.State != "done" {
|
||||
return res, nil
|
||||
}
|
||||
last = res
|
||||
}
|
||||
if !seen {
|
||||
return &fwPollResult{State: "unknown", Msg: "the BMC no longer lists the task"}, nil
|
||||
}
|
||||
return last, nil
|
||||
}
|
||||
|
||||
var fwScheduledRe = regexp.MustCompile(`(?i)\bscheduled\b|completed successfully`)
|
||||
|
||||
func (r *rfClient) fwPollOne(ctx context.Context, task string) (*fwPollResult, error) {
|
||||
resp, err := r.call(ctx, "GET", task, nil, "")
|
||||
if e := asRFError(err); e != nil && e.code == http.StatusNotFound {
|
||||
// iDRAC: while it updates itself the task moves to the job service
|
||||
if i := strings.Index(task, "/TaskService/Tasks/"); i >= 0 {
|
||||
resp, err = r.call(ctx, "GET", "/redfish/v1/JobService/Jobs/"+path.Base(task), nil, "")
|
||||
}
|
||||
if e := asRFError(err); e != nil && e.code == http.StatusNotFound {
|
||||
return &fwPollResult{State: "unknown", Msg: "the BMC no longer lists the task"}, nil
|
||||
}
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var t obj
|
||||
_ = json.Unmarshal(resp.body, &t)
|
||||
st := str(t["TaskState"]) + str(t["JobState"])
|
||||
res := &fwPollResult{State: "running", Pct: -1}
|
||||
if p, ok := t["PercentComplete"].(float64); ok {
|
||||
res.Pct = int(p)
|
||||
}
|
||||
for _, v := range asObj(t["Oem"]) { // iBMC TaskPercentage "40%", iRMC TotalProgressPercent
|
||||
o := asObj(v)
|
||||
if p := strings.TrimSuffix(str(o["TaskPercentage"]), "%"); p != "" {
|
||||
res.Pct, _ = strconv.Atoi(p)
|
||||
}
|
||||
if p, ok := o["TotalProgressPercent"].(float64); ok {
|
||||
res.Pct = int(p)
|
||||
}
|
||||
}
|
||||
if ms := toList(t["Messages"]); len(ms) > 0 {
|
||||
res.Msg = str(asObj(ms[len(ms)-1])["Message"])
|
||||
}
|
||||
if res.Msg == "" {
|
||||
res.Msg = str(t["Message"]) // Dell jobs
|
||||
}
|
||||
if res.Pct < 0 { // no percentage on the task: some keep it on the update service
|
||||
res.Pct = 0
|
||||
var us obj
|
||||
if _, err := r.do(ctx, "GET", "/redfish/v1/UpdateService", nil, &us); err == nil {
|
||||
if h := asObj(dig(us, "Oem", "Hpe")); len(h) > 0 { // iLO: State, FlashProgressPercent, Result
|
||||
if p, ok := h["FlashProgressPercent"].(float64); ok {
|
||||
res.Pct = int(p)
|
||||
}
|
||||
if str(h["State"]) == "Error" {
|
||||
res.State, res.Msg = "failed", "iLO update service: "+oneLine(fmt.Sprint(h["Result"]))
|
||||
return res, nil
|
||||
}
|
||||
}
|
||||
if a := asObj(dig(us, "Oem", "AMIUpdateService")); len(a) > 0 { // "3% done."
|
||||
if m := regexp.MustCompile(`(\d+)%`).FindStringSubmatch(str(a["FlashPercentage"])); m != nil {
|
||||
res.Pct, _ = strconv.Atoi(m[1])
|
||||
}
|
||||
if res.Msg == "" {
|
||||
res.Msg = str(a["UpdateStatus"])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
switch st {
|
||||
case "Completed":
|
||||
res.State, res.Pct = "done", 100
|
||||
case "Exception", "Killed", "Cancelled", "Failed", "CompletedWithErrors", "Interrupted":
|
||||
res.State = "failed"
|
||||
case "Scheduled", "UserIntervention": // Dell: staged until the host reboots
|
||||
res.State = "done"
|
||||
case "":
|
||||
if resp.code == http.StatusOK && t["TaskState"] == nil && t["JobState"] == nil {
|
||||
res.State = "done" // a task monitor answers with the result once finished
|
||||
}
|
||||
}
|
||||
if res.State == "running" && res.Pct >= 100 && fwScheduledRe.MatchString(res.Msg) {
|
||||
res.State = "done" // staged for the next reboot
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
@@ -0,0 +1,273 @@
|
||||
package main
|
||||
|
||||
// Firmware update plans per vendor: the request each one documents for a
|
||||
// component. Sources are on each entry; "oss" ones come from the vendor's own
|
||||
// tools (Ansible collections, PowerShell modules, SDKs) where its API guide is
|
||||
// not public. An operator can replace a plan's UpdateParameters with
|
||||
// fw_params={...} on the BMC record, or skip the plan with fw_method=.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type fwVendor struct {
|
||||
plan func(fwPlanCtx) (*fwRequest, error)
|
||||
methods []string
|
||||
claims bool
|
||||
unsupported string
|
||||
formats []string
|
||||
notes string
|
||||
sources []string
|
||||
}
|
||||
|
||||
var fwVendors = map[string]fwVendor{
|
||||
"idrac": {
|
||||
// MultipartUpload; older iDRAC 9 without it pulls with SimpleUpdate.
|
||||
// OnReset by default: BIOS-type jobs wait for a reboot anyway, and an
|
||||
// iDRAC or CPLD update must not surprise anyone.
|
||||
plan: func(x fwPlanCtx) (*fwRequest, error) {
|
||||
apply := x.applyOr("OnReset")
|
||||
if x.Caps.multipart != "" {
|
||||
return &fwRequest{Method: "multipart", URI: x.Caps.multipart,
|
||||
Params: obj{"Targets": []string{}, "@Redfish.OperationApplyTime": apply, "Oem": obj{}}}, nil
|
||||
}
|
||||
if x.Caps.simple != "" {
|
||||
return &fwRequest{Method: "simple", URI: x.Caps.simple, Params: simpleBody(x, obj{"@Redfish.OperationApplyTime": apply})}, nil
|
||||
}
|
||||
return nil, errors.New("this iDRAC offers neither MultipartHttpPushUri nor SimpleUpdate")
|
||||
},
|
||||
formats: []string{".exe", ".d9", ".d7", ".pm"},
|
||||
notes: "Dell Update Packages (Windows .EXE). BIOS, RAID, NIC jobs apply at the next reboot (-reboot); iDRAC, Diags apply by themselves; a CPLD update does a virtual A/C cycle.",
|
||||
sources: []string{
|
||||
"https://github.com/dell/iDRAC-Redfish-Scripting/blob/master/Redfish%20Python/DeviceFirmwareMultipartUploadREDFISH.py",
|
||||
"https://github.com/dell/iDRAC-Redfish-Scripting/blob/master/Redfish%20Python/DeviceFirmwareSimpleUpdateTransferProtocolREDFISH.py",
|
||||
},
|
||||
},
|
||||
"ilo": {
|
||||
// SimpleUpdate over HTTP(S) flashes iLO .bin and UEFI .signed.flash;
|
||||
// a Smart Component (.fwpkg) answers 200 and flashes nothing, so
|
||||
// import extracts the flashable file from it.
|
||||
plan: func(x fwPlanCtx) (*fwRequest, error) {
|
||||
if x.Caps.simple == "" {
|
||||
return nil, errors.New("this iLO offers no SimpleUpdate")
|
||||
}
|
||||
return &fwRequest{Method: "simple", URI: x.Caps.simple, Params: obj{"ImageURI": x.URL}}, nil
|
||||
},
|
||||
formats: []string{".bin", ".signed.flash", ".flash", ".fwpkg"},
|
||||
notes: "iLO firmware flashes at once (iLO restarts); UEFI/BIOS is staged and flashed at the next reboot (-reboot). Components only a runtime agent can flash (some NICs) need SUM/iSUT in the OS. Downgrades follow the iLO downgrade policy.",
|
||||
sources: []string{
|
||||
"https://servermanagementportal.ext.hpe.com/docs/references_and_material/blogposts/firmware_updates/part3/firmware_update_part3",
|
||||
"https://servermanagementportal.ext.hpe.com/docs/redfishservices/ilos/supplementdocuments/updateservice/",
|
||||
},
|
||||
},
|
||||
"xcc": {
|
||||
// /mfwupdate, after taking the push service (HttpPushUriTargetsBusy).
|
||||
plan: func(x fwPlanCtx) (*fwRequest, error) {
|
||||
params := obj{"Targets": []string{}}
|
||||
if x.Apply != "" {
|
||||
params["@Redfish.OperationApplyTime"] = x.Apply
|
||||
}
|
||||
if x.Caps.multipart != "" {
|
||||
return &fwRequest{Method: "multipart", URI: x.Caps.multipart, Params: params, Claim: true}, nil
|
||||
}
|
||||
if x.Caps.push != "" {
|
||||
return &fwRequest{Method: "push", URI: x.Caps.push, Claim: true}, nil
|
||||
}
|
||||
return nil, errors.New("this XCC offers no push update URI")
|
||||
},
|
||||
claims: true,
|
||||
formats: []string{".uxz", ".zip", ".bin"},
|
||||
notes: "Update XCC before UEFI (Lenovo). One push at a time; the busy flag is released after the update.",
|
||||
sources: []string{
|
||||
"https://pubs.lenovo.com/xcc-restapi/simple_update_firmware_post_multi_httppush",
|
||||
"https://pubs.lenovo.com/xcc2-restapi/prop_firmware_update_service_get",
|
||||
"https://pubs.lenovo.com/xcc/updating_firmware_overview",
|
||||
},
|
||||
},
|
||||
"supermicro": {
|
||||
// /UpdateService/upload with the documented targets and preserve options.
|
||||
plan: func(x fwPlanCtx) (*fwRequest, error) {
|
||||
uri := orDefault(x.Caps.multipart, "/redfish/v1/UpdateService/upload")
|
||||
apply := x.applyOr("Immediate")
|
||||
var params obj
|
||||
switch x.Component {
|
||||
case "bios":
|
||||
params = obj{"Targets": []string{"/redfish/v1/Systems/1/Bios"}, "@Redfish.OperationApplyTime": apply,
|
||||
"Oem": obj{"Supermicro": obj{"BIOS": obj{"PreserveME": true, "PreserveNVRAM": true, "PreserveSMBIOS": true, "BackupBIOS": false}}}}
|
||||
case "bmc":
|
||||
params = obj{"Targets": []string{"/redfish/v1/Managers/1"}, "@Redfish.OperationApplyTime": apply,
|
||||
"Oem": obj{"Supermicro": obj{"BMC": obj{"PreserveCfg": true, "PreserveSdr": true, "PreserveSsl": true, "BackupBMC": true}}}}
|
||||
case "cpld":
|
||||
id := ""
|
||||
for _, i := range x.Inventory {
|
||||
if strings.HasPrefix(i, "Motherboard_CPLD_") && id == "" {
|
||||
id = i
|
||||
}
|
||||
}
|
||||
if id == "" {
|
||||
return nil, errors.New("the BMC lists no Motherboard_CPLD_* firmware")
|
||||
}
|
||||
params = obj{"Targets": []string{"/redfish/v1/UpdateService/FirmwareInventory/" + id}, "@Redfish.OperationApplyTime": apply}
|
||||
default: // NIC, storage AOC, PSU: the BMC picks the device (SFT-DCMS-SINGLE licence)
|
||||
params = obj{"Targets": []string{}, "@Redfish.OperationApplyTime": apply}
|
||||
}
|
||||
return &fwRequest{Method: "multipart", URI: uri, Params: params}, nil
|
||||
},
|
||||
notes: "A BIOS update powers the host off first. Gen 12/13 boards take other BIOS preserve options (PreserveOA, PreserveSETUPCONF, ...): set fw_params on the BMC if the BMC refuses these. Redfish updates need the SFT-OOB-LIC or SFT-DCMS-SINGLE licence; NIC/storage/PSU need DCMS. A CPLD update resets the BMC.",
|
||||
sources: []string{
|
||||
"https://www.supermicro.com/manuals/other/RedfishUserGuide.pdf",
|
||||
"https://github.com/bmc-toolbox/bmclib/blob/main/providers/supermicro/x12.go",
|
||||
},
|
||||
},
|
||||
"gigabyte": {plan: amiPlan, formats: []string{".hpm", ".rcu", ".ima_enc", ".ima", ".rbu", ".bin"}, notes: amiNotes, sources: []string{"https://download.gigabyte.com/FileList/Manual/server_manual_redfish_v1.11.0.pdf", "GIGABYTE Firmware Upgrade Guide v0.04 (2022)"}},
|
||||
"ami": {plan: amiPlan, notes: amiNotes, sources: []string{"https://download2.mitaccomputing.com/pub/doc/MiTAC_AST2600_Redfish_API_Doc_v.1_051024.pdf", "https://support.blackcoretech.com/support/solutions/articles/70000680445-update-bios-bmc-via-redfish-multipart-push"}},
|
||||
"asrockrack": {plan: amiPlan, notes: amiNotes + " ASRock Rack's own Redfish firmware notes could not be read: this is the AMI MegaRAC request, check it on one machine first.", sources: []string{"https://download.asrock.com/TSD/SMU/Manual/ASRock%20Rack%20Server%20Management_v2.8.0.pdf"}},
|
||||
"openbmc": {
|
||||
// update-multipart; the BMC image applies at once (it reboots), the rest OnReset.
|
||||
plan: func(x fwPlanCtx) (*fwRequest, error) {
|
||||
params := obj{"Targets": []string{}}
|
||||
if x.Component == "bmc" {
|
||||
params = obj{"Targets": []string{"/redfish/v1/Managers/bmc"}, "@Redfish.OperationApplyTime": x.applyOr("Immediate")}
|
||||
} else if x.Apply != "" {
|
||||
params["@Redfish.OperationApplyTime"] = x.Apply
|
||||
}
|
||||
if x.Caps.multipart != "" {
|
||||
return &fwRequest{Method: "multipart", URI: x.Caps.multipart, Params: params}, nil
|
||||
}
|
||||
if x.Caps.push != "" && x.Component == "bmc" {
|
||||
return &fwRequest{Method: "push", URI: x.Caps.push}, nil
|
||||
}
|
||||
return nil, errors.New("this OpenBMC offers no multipart update (SimpleUpdate is off in bmcweb by default)")
|
||||
},
|
||||
formats: []string{".tar"},
|
||||
notes: "Signed image tarballs whose MANIFEST MachineName matches the BMC.",
|
||||
sources: []string{"https://github.com/openbmc/bmcweb/blob/master/redfish-core/lib/update_service.hpp", "https://github.com/openbmc/docs/blob/master/REDFISH-cheatsheet.md"},
|
||||
},
|
||||
"xfusion": {
|
||||
// upload to the BMC's /tmp/web, then SimpleUpdate naming the file
|
||||
plan: func(x fwPlanCtx) (*fwRequest, error) {
|
||||
if x.Component != "bmc" && x.Component != "bios" && x.Component != "cpld" {
|
||||
return nil, fmt.Errorf("iBMC updates BMC, BIOS and CPLD out of band; %s firmware goes through Smart Provisioning in-band (not supported)", x.Component)
|
||||
}
|
||||
if x.Caps.simple == "" {
|
||||
return nil, errors.New("this iBMC offers no SimpleUpdate")
|
||||
}
|
||||
return &fwRequest{Method: "multipart", URI: "/redfish/v1/UpdateService/FirmwareInventory", NoParams: true, Field: "file",
|
||||
Then: &fwRequest{Method: "simple", URI: x.Caps.simple, Params: obj{"ImageURI": "/tmp/web/" + path.Base(x.File)}}}, nil
|
||||
},
|
||||
formats: []string{".hpm"},
|
||||
notes: "The BMC restarts after a BMC update.",
|
||||
sources: []string{
|
||||
"https://github.com/Open-xFusion/Server_Plugin_Ansible/blob/HEAD/ibmc_ansible/ibmc_redfish_api/api_manage_file.py",
|
||||
"https://github.com/Open-xFusion/Server_Plugin_Ansible/blob/HEAD/ibmc_ansible/ibmc_redfish_api/api_outband_fw_update.py",
|
||||
},
|
||||
},
|
||||
"kaytus": {
|
||||
// upload (with the BMC password), then an OEM SimpleUpdate; the tasks have fixed names
|
||||
plan: func(x fwPlanCtx) (*fwRequest, error) {
|
||||
oem := obj{"PreserveConf": true}
|
||||
var tasks []string
|
||||
switch x.Component {
|
||||
case "bmc":
|
||||
oem["FlashItem"], tasks = "OpenBMC_ENC", []string{"/redfish/v1/TaskService/Tasks/BMC"}
|
||||
case "bios":
|
||||
oem["FlashItem"], oem["BiosFlash"] = "BIOS", "Both"
|
||||
tasks = []string{"/redfish/v1/TaskService/Tasks/BIOS0", "/redfish/v1/TaskService/Tasks/BIOS1"}
|
||||
default:
|
||||
return nil, fmt.Errorf("KAYTUS: BMC and BIOS only (not %s)", x.Component)
|
||||
}
|
||||
return &fwRequest{Method: "multipart", URI: "/redfish/v1/UpdateService/FirmwareInventory", NoParams: true, Field: "fwimage",
|
||||
Form: map[string]string{"CurrentPassword": x.Password, "EncryptFlag": "False"}, FailText: "Verification Failed",
|
||||
Then: &fwRequest{Method: "simple", URI: orDefault(x.Caps.simple, "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate"), Params: obj{"Oem": obj{"Public": oem}}},
|
||||
Tasks: tasks}, nil
|
||||
},
|
||||
formats: []string{".hpm", ".bin"},
|
||||
notes: "No public Redfish guide: this is what KAYTUS's own SDK (ksManage) does on X3-generation BMCs; older generations use a non-Redfish API. A BIOS update with the host on waits for a power cycle (-reboot); the BMC restarts after a BMC update.",
|
||||
sources: []string{"https://github.com/KSManageOSS/ksManage/blob/HEAD/ksmanage_sdk/interface/CommonX3.py", "https://github.com/ieisystem/kaytus.ksmanage"},
|
||||
},
|
||||
"irmc": {
|
||||
// OEM actions per component, file in the "data" part
|
||||
plan: func(x fwPlanCtx) (*fwRequest, error) {
|
||||
p := "Fsas" // FTS on S5/S6 (Vendor ts_fujitsu or none), Fsas on newer firmware
|
||||
if x.Vendor == "" || strings.EqualFold(x.Vendor, "ts_fujitsu") {
|
||||
p = "FTS"
|
||||
}
|
||||
switch x.Component {
|
||||
case "bmc":
|
||||
return &fwRequest{Method: "multipart", URI: "/redfish/v1/Managers/iRMC/Actions/" + p + "Manager.FWUpdate",
|
||||
AltURI: "/redfish/v1/Managers/iRMC/Actions/Oem/" + p + "Manager.FWUpdate", NoParams: true, Field: "data"}, nil
|
||||
case "bios":
|
||||
return &fwRequest{Method: "multipart", URI: "/redfish/v1/Systems/0/Bios/Actions/Oem/" + p + "Bios.BiosUpdate", NoParams: true, Field: "data"}, nil
|
||||
}
|
||||
return nil, fmt.Errorf("iRMC: iRMC and BIOS only (not %s)", x.Component)
|
||||
},
|
||||
formats: []string{".bin", ".upc"},
|
||||
notes: "iRMC .bin, BIOS .UPC. With the host on, a BIOS update waits for a reboot (-reboot); the iRMC restarts after its own update.",
|
||||
sources: []string{"https://github.com/fujitsu/ansible-irmc-integration/blob/HEAD/plugins/modules/irmc_fwbios_update.py", "https://github.com/mmurayama/fujitsu-redfish-samples/blob/master/scripts/update_irmc_firmware.py"},
|
||||
},
|
||||
"cimc": {
|
||||
unsupported: "Cisco documents CIMC firmware updates over Redfish only as SimpleUpdate pulls over SCP (and HSU/HUU from NFS, CIFS or HTTPS shares); Rollcall serves packages over HTTP",
|
||||
sources: []string{"https://www.cisco.com/c/en/us/td/docs/unified_computing/ucs/c/sw/api/4_3/b-cisco-ucs-c-series-servers-rest-api-programmer-s-guide-release-4-3.pdf"},
|
||||
},
|
||||
}
|
||||
|
||||
const amiNotes = "MegaRAC: the BMC restarts after a BMC flash and accepts no logins while flashing; do not use its web UI meanwhile."
|
||||
|
||||
// amiPlan: /UpdateService/upload with UpdateParameters (the inventory target)
|
||||
// and OemParameters (ImageType); .hpm packages have their own targets.
|
||||
func amiPlan(x fwPlanCtx) (*fwRequest, error) {
|
||||
hpm := strings.HasSuffix(strings.ToLower(x.File), ".hpm")
|
||||
var target, typ string
|
||||
switch x.Component {
|
||||
case "bmc":
|
||||
if hpm {
|
||||
target, typ = "BMC", "HPM_BMC"
|
||||
} else {
|
||||
target, typ = orDefault(x.first("BMCImage1", "BMC"), "BMC"), "BMC"
|
||||
}
|
||||
case "bios":
|
||||
if hpm {
|
||||
target, typ = "BIOS2", "HPM_BIOS"
|
||||
} else {
|
||||
target, typ = "BIOS", "BIOS"
|
||||
}
|
||||
case "cpld":
|
||||
target, typ = "MB_CPLD1", "MB_CPLD"
|
||||
default:
|
||||
return nil, fmt.Errorf("MegaRAC: BMC, BIOS and CPLD (not %s)", x.Component)
|
||||
}
|
||||
if len(x.Inventory) > 0 && !x.has(target) {
|
||||
return nil, fmt.Errorf("the BMC lists no firmware %s (it has %s)", target, strings.Join(x.Inventory, ", "))
|
||||
}
|
||||
params := obj{"Targets": []string{"/redfish/v1/UpdateService/FirmwareInventory/" + target}}
|
||||
if x.Apply != "" {
|
||||
params["@Redfish.OperationApplyTime"] = x.Apply
|
||||
}
|
||||
return &fwRequest{Method: "multipart", URI: orDefault(x.Caps.multipart, "/redfish/v1/UpdateService/upload"), Params: params,
|
||||
Parts: []fwPart{{Name: "OemParameters", Body: obj{"ImageType": typ}}}}, nil
|
||||
}
|
||||
|
||||
// simpleBody is a SimpleUpdate body for the package URL, plus extra fields.
|
||||
func simpleBody(x fwPlanCtx, extra obj) obj {
|
||||
b := obj{"ImageURI": x.URL}
|
||||
if len(x.Caps.protocols) > 0 {
|
||||
b["TransferProtocol"] = strings.ToUpper(strings.SplitN(x.URL, "://", 2)[0])
|
||||
}
|
||||
for k, v := range extra {
|
||||
b[k] = v
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func init() {
|
||||
for i := range redfishProfiles {
|
||||
p := &redfishProfiles[i]
|
||||
if v, ok := fwVendors[p.Driver]; ok {
|
||||
p.FWPlan, p.FWMethods, p.FWClaims, p.FWUnsupported = v.plan, v.methods, v.claims, v.unsupported
|
||||
p.FWFormats, p.FWNotes, p.FWSources = v.formats, v.notes, v.sources
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,406 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// vendorBMC behaves the way one vendor's documentation (or the field notes in
|
||||
// Ironic, MAAS, bmclib) says it does; each flag is one such behaviour.
|
||||
type vendorBMC struct {
|
||||
mu sync.Mutex
|
||||
vendor, manufacturer string
|
||||
oem []string
|
||||
system string
|
||||
trailingSlash bool // collection members end in "/" (iLO)
|
||||
allowed, targets []string
|
||||
mode, power, etag string
|
||||
needIfMatch bool // 428 without If-Match (AMI)
|
||||
strongOnly bool // 412 for a weak If-Match
|
||||
futureSD bool // PATCH on the system: 400 OperationSupportedInFutureStateURI (ASRock Rack)
|
||||
refuseMode bool // 400 when the body carries BootSourceOverrideMode (HPE DL160)
|
||||
inPOST int // the first N PATCHes: 400 UnableToModifyDuringSystemPOST (HPE)
|
||||
flipsWithoutMode bool // a PATCH without the mode switches to Legacy (Supermicro)
|
||||
async bool // reset answers 202 and a task monitor
|
||||
sessionOnly bool // basic auth refused: log in through the SessionService (iBMC clients)
|
||||
actionInfo bool // reset values only through @Redfish.ActionInfo (iBMC)
|
||||
numericETag bool // ETag only as a bare number in @odata.etag (iRMC)
|
||||
chassisReset bool // no ComputerSystem.Reset; Chassis/1 has the reset (KAYTUS SDK)
|
||||
successAs400 bool // a working PATCH answers 400 "completed successfully" (KAYTUS SDK)
|
||||
refuseDisabled bool // Enabled "Disabled" is 400; clearing is Target None (iRMC)
|
||||
sessions int // open sessions
|
||||
|
||||
patchPaths, ifMatch, resets []string
|
||||
patches []obj
|
||||
taskPolls int
|
||||
}
|
||||
|
||||
func (f *vendorBMC) fail(w http.ResponseWriter, code int, id, msg string) {
|
||||
w.WriteHeader(code)
|
||||
_ = json.NewEncoder(w).Encode(obj{"error": obj{"code": "Base.1.0.GeneralError", "message": "see ExtendedInfo",
|
||||
"@Message.ExtendedInfo": []obj{{"MessageId": id, "Message": msg}}}})
|
||||
}
|
||||
|
||||
func (f *vendorBMC) handler() http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
if r.URL.Path == "/redfish/v1/SessionService/Sessions" && r.Method == "POST" {
|
||||
var c struct{ UserName, Password string }
|
||||
_ = json.NewDecoder(r.Body).Decode(&c)
|
||||
if c.UserName != "admin" || c.Password != "secret" {
|
||||
f.fail(w, 401, "Base.1.0.NoValidSession", "bad credentials")
|
||||
return
|
||||
}
|
||||
f.sessions++
|
||||
w.Header().Set("X-Auth-Token", "tok")
|
||||
w.Header().Set("Location", "/redfish/v1/SessionService/Sessions/1")
|
||||
w.WriteHeader(201)
|
||||
return
|
||||
}
|
||||
authed := r.Header.Get("X-Auth-Token") == "tok"
|
||||
if u, p, ok := r.BasicAuth(); !f.sessionOnly && ok && u == "admin" && p == "secret" {
|
||||
authed = true
|
||||
}
|
||||
if r.URL.Path != "/redfish/v1/" && !authed {
|
||||
f.fail(w, 401, "Base.1.0.InsufficientPrivilege", "bad credentials")
|
||||
return
|
||||
}
|
||||
oem := obj{}
|
||||
for _, k := range f.oem {
|
||||
oem[k] = obj{}
|
||||
}
|
||||
sys := strings.TrimSuffix(r.URL.Path, "/")
|
||||
switch {
|
||||
case r.URL.Path == "/redfish/v1/":
|
||||
_ = json.NewEncoder(w).Encode(obj{"RedfishVersion": "1.15.0", "Vendor": f.vendor, "Oem": oem})
|
||||
case r.URL.Path == "/redfish/v1/SessionService/Sessions/1" && r.Method == "DELETE":
|
||||
f.sessions--
|
||||
w.WriteHeader(204)
|
||||
case r.URL.Path == "/redfish/v1/Systems/1/ResetActionInfo":
|
||||
_ = json.NewEncoder(w).Encode(obj{"Parameters": []obj{{"Name": "ResetType", "AllowableValues": f.allowed}}})
|
||||
case r.URL.Path == "/redfish/v1/Chassis/1" && r.Method == "GET":
|
||||
_ = json.NewEncoder(w).Encode(obj{"Actions": obj{"#Chassis.Reset": obj{"target": "/redfish/v1/Chassis/1/Actions/Chassis.Reset", "ResetType@Redfish.AllowableValues": f.allowed}}})
|
||||
case r.URL.Path == "/redfish/v1/Systems":
|
||||
m := f.system
|
||||
if f.trailingSlash {
|
||||
m += "/"
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(obj{"Members": []obj{{"@odata.id": m}}})
|
||||
case (sys == f.system || sys == f.system+"/SD") && r.Method == "GET":
|
||||
if f.etag != "" && !f.numericETag {
|
||||
w.Header().Set("ETag", f.etag)
|
||||
}
|
||||
boot := obj{"BootSourceOverrideTarget@Redfish.AllowableValues": f.targets, "BootSourceOverrideEnabled": "Disabled"}
|
||||
if f.mode != "" {
|
||||
boot["BootSourceOverrideMode"] = f.mode
|
||||
}
|
||||
reset := obj{"target": f.system + "/Actions/ComputerSystem.Reset", "ResetType@Redfish.AllowableValues": f.allowed}
|
||||
if f.actionInfo {
|
||||
reset = obj{"target": f.system + "/Actions/ComputerSystem.Reset", "@Redfish.ActionInfo": "/redfish/v1/Systems/1/ResetActionInfo"}
|
||||
}
|
||||
res := obj{"PowerState": f.power, "Manufacturer": f.manufacturer, "Model": "Test", "Boot": boot,
|
||||
"Actions": obj{"#ComputerSystem.Reset": reset}, "Links": obj{"Chassis": []obj{{"@odata.id": "/redfish/v1/Chassis/1"}}}}
|
||||
if f.chassisReset {
|
||||
res["Actions"] = obj{}
|
||||
}
|
||||
if f.numericETag {
|
||||
res["@odata.etag"] = json.Number(f.etag)
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(res)
|
||||
case (sys == f.system || sys == f.system+"/SD") && r.Method == "PATCH":
|
||||
im := r.Header.Get("If-Match")
|
||||
f.patchPaths, f.ifMatch = append(f.patchPaths, sys), append(f.ifMatch, im)
|
||||
var body obj
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
boot := asObj(body["Boot"])
|
||||
switch {
|
||||
case f.needIfMatch && im == "":
|
||||
f.fail(w, 428, "Ami.1.0.PreconditionHeaderMissing", "If-Match header is required")
|
||||
case f.strongOnly && strings.HasPrefix(im, "W/"):
|
||||
f.fail(w, 412, "HttpStatus.1.0.PreconditionFailed", "ETag does not match")
|
||||
case f.futureSD && sys == f.system:
|
||||
f.fail(w, 400, "Ami.1.0.OperationSupportedInFutureStateURI", "moved to FutureState URI(/redfish/v1/Systems/Self/SD)")
|
||||
case f.inPOST > 0:
|
||||
f.inPOST--
|
||||
f.fail(w, 400, "iLO.2.14.UnableToModifyDuringSystemPOST", "")
|
||||
case f.refuseDisabled && boot["BootSourceOverrideEnabled"] == "Disabled":
|
||||
f.fail(w, 400, "Base.1.0.PropertyValueNotInList", "Disabled")
|
||||
case f.refuseMode && boot["BootSourceOverrideMode"] != nil:
|
||||
f.fail(w, 400, "Base.1.0.PropertyNotWritable", "BootSourceOverrideMode")
|
||||
default:
|
||||
if boot["BootSourceOverrideTarget"] != nil && !hasTag(f.targets, str(boot["BootSourceOverrideTarget"])) {
|
||||
f.fail(w, 400, "Base.1.0.PropertyValueNotInList", "target")
|
||||
return
|
||||
}
|
||||
if m := str(boot["BootSourceOverrideMode"]); m != "" {
|
||||
f.mode = m
|
||||
} else if f.flipsWithoutMode {
|
||||
f.mode = "Legacy"
|
||||
}
|
||||
f.patches = append(f.patches, boot)
|
||||
if f.successAs400 {
|
||||
f.fail(w, 400, "Base.1.0.Success", "The request completed successfully.")
|
||||
return
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
}
|
||||
case r.URL.Path == f.system+"/Actions/ComputerSystem.Reset" || (f.chassisReset && r.URL.Path == "/redfish/v1/Chassis/1/Actions/Chassis.Reset"):
|
||||
var body struct{ ResetType string }
|
||||
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||
if !hasTag(f.allowed, body.ResetType) {
|
||||
f.fail(w, 400, "Base.1.0.ActionParameterNotSupported", "ResetType")
|
||||
return
|
||||
}
|
||||
f.resets = append(f.resets, body.ResetType)
|
||||
switch body.ResetType {
|
||||
case "On", "ForceOn":
|
||||
f.power = "On"
|
||||
case "ForceOff", "GracefulShutdown":
|
||||
f.power = "Off"
|
||||
}
|
||||
if f.async {
|
||||
w.Header().Set("Location", "/redfish/v1/TaskService/Tasks/7")
|
||||
w.WriteHeader(202)
|
||||
return
|
||||
}
|
||||
w.WriteHeader(204)
|
||||
case r.URL.Path == "/redfish/v1/TaskService/Tasks/7":
|
||||
f.taskPolls++
|
||||
st := "Running"
|
||||
if f.taskPolls >= 2 {
|
||||
st = "Completed"
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(obj{"TaskState": st})
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func runBMC(t *testing.T, f *vendorBMC, driver string) (*bmcCreds, func()) {
|
||||
t.Helper()
|
||||
old := rfSleep
|
||||
rfSleep = func(context.Context, time.Duration) error { return nil }
|
||||
srv := httptest.NewTLSServer(f.handler())
|
||||
return creds(srv.URL, driver), func() { srv.Close(); rfSleep = old }
|
||||
}
|
||||
|
||||
func mustRun(t *testing.T, c *bmcCreds, op, arg string) string {
|
||||
t.Helper()
|
||||
out, err := powerRun(context.Background(), "", c, op, arg)
|
||||
if err != nil {
|
||||
t.Fatalf("%s %s: %v", op, arg, err)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
var stdTargets = []string{"None", "Pxe", "Hdd", "Cd", "BiosSetup", "UefiHttp"}
|
||||
|
||||
func TestMatchRedfishProfile(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
vendor, manufacturer string
|
||||
oem []string
|
||||
want string
|
||||
}{
|
||||
{"Dell", "Dell Inc.", []string{"Dell"}, "idrac"},
|
||||
{"HPE", "HPE", []string{"Hpe"}, "ilo"},
|
||||
{"Lenovo", "Lenovo", nil, "xcc"},
|
||||
{"Supermicro", "Supermicro", []string{"Supermicro"}, "supermicro"},
|
||||
{"", "GIGA-BYTE TECHNOLOGY CO., LTD", []string{"Ami"}, "gigabyte"},
|
||||
{"AsrockRack", "", []string{"Ami"}, "asrockrack"},
|
||||
{"", "To Be Filled By O.E.M.", []string{"Ami"}, "ami"},
|
||||
{"OpenBMC", "", nil, "openbmc"},
|
||||
{"Contoso", "Contoso", nil, "redfish"},
|
||||
} {
|
||||
if got := matchRedfishProfile(c.vendor, c.manufacturer, c.oem).Driver; got != c.want {
|
||||
t.Errorf("%q/%q/%v: %s, want %s", c.vendor, c.manufacturer, c.oem, got, c.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Supermicro: a boot override without the mode can switch the board to Legacy.
|
||||
func TestRedfishSupermicroKeepsBootMode(t *testing.T) {
|
||||
f := &vendorBMC{vendor: "Supermicro", manufacturer: "Supermicro", system: "/redfish/v1/Systems/1", mode: "UEFI", power: "On",
|
||||
targets: stdTargets, flipsWithoutMode: true, etag: `"1"`,
|
||||
allowed: []string{"On", "ForceOff", "GracefulShutdown", "GracefulRestart", "ForceRestart", "Nmi", "ForceOn", "PowerCycle"}}
|
||||
c, done := runBMC(t, f, "supermicro")
|
||||
defer done()
|
||||
mustRun(t, c, "bootdev", "pxe")
|
||||
if f.mode != "UEFI" || f.patches[0]["BootSourceOverrideMode"] != "UEFI" || f.patches[0]["BootSourceOverrideEnabled"] != "Once" || f.ifMatch[0] != `"1"` {
|
||||
t.Errorf("mode now %s, patch %v, if-match %v", f.mode, f.patches, f.ifMatch)
|
||||
}
|
||||
mustRun(t, c, "cycle", "")
|
||||
if f.resets[0] != "PowerCycle" {
|
||||
t.Errorf("resets %v", f.resets)
|
||||
}
|
||||
}
|
||||
|
||||
// Dell: changing the mode starts a BIOS job; it is not sent.
|
||||
func TestRedfishIDRACOmitsBootMode(t *testing.T) {
|
||||
f := &vendorBMC{vendor: "Dell", manufacturer: "Dell Inc.", system: "/redfish/v1/Systems/System.Embedded.1", mode: "UEFI", power: "Off",
|
||||
targets: stdTargets, allowed: []string{"On", "ForceOff", "ForceRestart", "GracefulShutdown", "PushPowerButton", "Nmi"}}
|
||||
c, done := runBMC(t, f, "idrac")
|
||||
defer done()
|
||||
mustRun(t, c, "bootdev", "pxe")
|
||||
if _, sent := f.patches[0]["BootSourceOverrideMode"]; sent {
|
||||
t.Errorf("idrac got the mode: %v", f.patches[0])
|
||||
}
|
||||
c.Extra["boot_mode"] = "Legacy" // the operator asks: then it goes
|
||||
mustRun(t, c, "bootdev", "pxe")
|
||||
if f.patches[1]["BootSourceOverrideMode"] != "Legacy" {
|
||||
t.Errorf("asked mode not sent: %v", f.patches[1])
|
||||
}
|
||||
mustRun(t, c, "on", "")
|
||||
mustRun(t, c, "cycle", "") // no PowerCycle on iDRAC9 3.x: ForceRestart
|
||||
if strings.Join(f.resets, ",") != "On,ForceRestart" {
|
||||
t.Errorf("resets %v", f.resets)
|
||||
}
|
||||
}
|
||||
|
||||
// HPE: no boot changes during POST; some refuse the mode; members end in "/".
|
||||
func TestRedfishILOPostStateAndMode(t *testing.T) {
|
||||
f := &vendorBMC{vendor: "HPE", manufacturer: "HPE", oem: []string{"Hpe"}, system: "/redfish/v1/Systems/1", trailingSlash: true,
|
||||
mode: "UEFI", power: "On", targets: stdTargets, inPOST: 2, refuseMode: true, etag: `W/"9"`,
|
||||
allowed: []string{"On", "ForceOff", "GracefulShutdown", "ForceRestart", "Nmi", "PushPowerButton", "GracefulRestart"}}
|
||||
c, done := runBMC(t, f, "redfish") // generic driver, found through the collection
|
||||
defer done()
|
||||
out := mustRun(t, c, "bootdev", "pxe")
|
||||
if len(f.patches) != 1 || f.patches[0]["BootSourceOverrideTarget"] != "Pxe" || f.patches[0]["BootSourceOverrideMode"] != nil {
|
||||
t.Errorf("%s: patches %v", out, f.patches)
|
||||
}
|
||||
var p obj
|
||||
_ = json.Unmarshal([]byte(mustRun(t, c, "probe", "")), &p)
|
||||
if p["driver"] != "ilo" || p["system"] != "/redfish/v1/Systems/1" || p["boot_mode"] != "UEFI" {
|
||||
t.Errorf("probe %v", p)
|
||||
}
|
||||
}
|
||||
|
||||
// AMI (Gigabyte): Systems/Self, If-Match required, a short ResetType list.
|
||||
func TestRedfishAMIIfMatchAndCycle(t *testing.T) {
|
||||
f := &vendorBMC{manufacturer: "GIGA-BYTE TECHNOLOGY CO., LTD", oem: []string{"Ami"}, system: "/redfish/v1/Systems/Self",
|
||||
mode: "UEFI", power: "On", targets: stdTargets, needIfMatch: true, strongOnly: true, etag: `W/"abc"`,
|
||||
allowed: []string{"On", "ForceOff", "GracefulShutdown", "ForceRestart"}}
|
||||
c, done := runBMC(t, f, "gigabyte")
|
||||
defer done()
|
||||
mustRun(t, c, "bootdev", "pxe")
|
||||
if f.ifMatch[len(f.ifMatch)-1] != `"abc"` || f.patches[0]["BootSourceOverrideMode"] != "UEFI" {
|
||||
t.Errorf("if-match tries %v, patch %v", f.ifMatch, f.patches)
|
||||
}
|
||||
mustRun(t, c, "cycle", "")
|
||||
if strings.Join(f.resets, ",") != "ForceRestart" {
|
||||
t.Errorf("resets %v", f.resets)
|
||||
}
|
||||
// a build that offers neither PowerCycle nor a restart: off, then on
|
||||
f.allowed = []string{"On", "ForceOff"}
|
||||
f.resets = nil
|
||||
mustRun(t, c, "cycle", "")
|
||||
if strings.Join(f.resets, ",") != "ForceOff,On" {
|
||||
t.Errorf("emulated cycle %v", f.resets)
|
||||
}
|
||||
}
|
||||
|
||||
// ASRock Rack: boot settings live in the future-state resource .../SD.
|
||||
func TestRedfishASRockFutureState(t *testing.T) {
|
||||
f := &vendorBMC{vendor: "AsrockRack", oem: []string{"Ami"}, system: "/redfish/v1/Systems/Self", mode: "UEFI", power: "Off",
|
||||
targets: stdTargets, futureSD: true, etag: `"5"`, allowed: []string{"On", "ForceOff", "GracefulShutdown", "ForceRestart"}}
|
||||
c, done := runBMC(t, f, "asrockrack")
|
||||
defer done()
|
||||
mustRun(t, c, "bootdev", "pxe")
|
||||
if strings.Join(f.patchPaths, ",") != "/redfish/v1/Systems/Self,/redfish/v1/Systems/Self/SD" || len(f.patches) != 1 {
|
||||
t.Errorf("paths %v patches %v", f.patchPaths, f.patches)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedfishTaskAndTargets(t *testing.T) {
|
||||
f := &vendorBMC{vendor: "Contoso", system: "/redfish/v1/Systems/1", power: "Off", targets: []string{"None", "Pxe", "Hdd"},
|
||||
async: true, allowed: []string{"On", "ForceOff"}}
|
||||
c, done := runBMC(t, f, "redfish")
|
||||
defer done()
|
||||
mustRun(t, c, "on", "")
|
||||
if f.taskPolls < 2 {
|
||||
t.Errorf("task polled %d times", f.taskPolls)
|
||||
}
|
||||
if _, err := powerRun(context.Background(), "", c, "bootdev", "bios"); err == nil || !strings.Contains(err.Error(), "offers boot targets") {
|
||||
t.Errorf("an unoffered target: %v", err)
|
||||
}
|
||||
mustRun(t, c, "bootdev", "none")
|
||||
if f.patches[0]["BootSourceOverrideEnabled"] != "Disabled" {
|
||||
t.Errorf("clear %v", f.patches)
|
||||
}
|
||||
}
|
||||
|
||||
// iBMC: sessions, ActionInfo, ForcePowerCycle, If-Match on PATCH.
|
||||
func TestRedfishXFusionSession(t *testing.T) {
|
||||
f := &vendorBMC{oem: []string{"xFusion"}, manufacturer: "xFusion", system: "/redfish/v1/Systems/1", power: "On", mode: "UEFI",
|
||||
targets: stdTargets, sessionOnly: true, actionInfo: true, needIfMatch: true, etag: `W/"77"`,
|
||||
allowed: []string{"On", "ForceOff", "GracefulShutdown", "ForceRestart", "Nmi", "ForcePowerCycle"}}
|
||||
c, done := runBMC(t, f, "xfusion")
|
||||
defer done()
|
||||
mustRun(t, c, "cycle", "")
|
||||
mustRun(t, c, "bootdev", "pxe")
|
||||
if strings.Join(f.resets, ",") != "ForcePowerCycle" || len(f.patches) != 1 || f.ifMatch[0] != `W/"77"` || f.sessions != 0 {
|
||||
t.Errorf("resets %v patches %v if-match %v open sessions %d", f.resets, f.patches, f.ifMatch, f.sessions)
|
||||
}
|
||||
var p obj
|
||||
_ = json.Unmarshal([]byte(mustRun(t, c, "probe", "")), &p)
|
||||
if p["driver"] != "xfusion" || p["auth"] != "session" || f.sessions != 0 {
|
||||
t.Errorf("probe %v, sessions %d", p, f.sessions)
|
||||
}
|
||||
}
|
||||
|
||||
// iRMC: the ETag only in the body as a number; never Enabled Disabled.
|
||||
func TestRedfishIRMC(t *testing.T) {
|
||||
f := &vendorBMC{vendor: "ts_fujitsu", manufacturer: "FUJITSU", system: "/redfish/v1/Systems/0", power: "On", mode: "UEFI",
|
||||
targets: stdTargets, numericETag: true, etag: "1714", needIfMatch: true, refuseDisabled: true,
|
||||
allowed: []string{"ForceOff", "ForceRestart", "Nmi", "PushPowerButton"}}
|
||||
c, done := runBMC(t, f, "irmc")
|
||||
defer done()
|
||||
mustRun(t, c, "bootdev", "pxe")
|
||||
if f.ifMatch[0] != "1714" || f.patches[0]["BootSourceOverrideMode"] != nil {
|
||||
t.Errorf("if-match %v patch %v", f.ifMatch, f.patches)
|
||||
}
|
||||
mustRun(t, c, "bootdev", "none")
|
||||
if f.patches[1]["BootSourceOverrideTarget"] != "None" || f.patches[1]["BootSourceOverrideEnabled"] != nil {
|
||||
t.Errorf("clear %v", f.patches[1])
|
||||
}
|
||||
mustRun(t, c, "soft", "") // no GracefulShutdown: the power button
|
||||
if strings.Join(f.resets, ",") != "PushPowerButton" {
|
||||
t.Errorf("resets %v", f.resets)
|
||||
}
|
||||
}
|
||||
|
||||
// KAYTUS (third-party SDK): reset on the chassis; a PATCH that worked answers 400.
|
||||
func TestRedfishKaytus(t *testing.T) {
|
||||
f := &vendorBMC{oem: []string{"Public"}, manufacturer: "KAYTUS", system: "/redfish/v1/Systems/1", power: "Off", mode: "UEFI",
|
||||
targets: stdTargets, chassisReset: true, successAs400: true,
|
||||
allowed: []string{"On", "ForceOff", "GracefulShutdown", "ForceRestart", "PowerCycle"}}
|
||||
c, done := runBMC(t, f, "redfish")
|
||||
defer done()
|
||||
mustRun(t, c, "on", "")
|
||||
mustRun(t, c, "bootdev", "pxe")
|
||||
if strings.Join(f.resets, ",") != "On" || len(f.patches) != 1 {
|
||||
t.Errorf("resets %v patches %v", f.resets, f.patches)
|
||||
}
|
||||
if p := matchRedfishProfile("", "KAYTUS", []string{"Public"}); p.Driver != "kaytus" {
|
||||
t.Errorf("profile %s", p.Driver)
|
||||
}
|
||||
}
|
||||
|
||||
// CIMC: /Systems/<serial>, found through the collection.
|
||||
func TestRedfishCIMCSerialPath(t *testing.T) {
|
||||
f := &vendorBMC{manufacturer: "Cisco Systems Inc", oem: []string{"Cisco"}, system: "/redfish/v1/Systems/WZP21330G5B", power: "On",
|
||||
targets: stdTargets, allowed: []string{"On", "ForceOff", "GracefulShutdown", "GracefulRestart", "ForceRestart", "Nmi", "PowerCycle"}}
|
||||
c, done := runBMC(t, f, "cimc")
|
||||
defer done()
|
||||
mustRun(t, c, "cycle", "")
|
||||
if strings.Join(f.resets, ",") != "PowerCycle" {
|
||||
t.Errorf("resets %v", f.resets)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/ecdsa"
|
||||
"crypto/elliptic"
|
||||
"crypto/rand"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Regressions from the security review.
|
||||
|
||||
func TestRackBinaryArchIsNoPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
_ = os.WriteFile(filepath.Join(dir, "secret"), []byte("key"), 0o600)
|
||||
a := &App{cfg: &Config{RackBinaries: filepath.Join(dir, "bins")}}
|
||||
for _, arch := range []string{"x/../../secret", "../secret", "amd64/../../secret", ""} {
|
||||
if p, _, ok := a.rackBinary(arch); ok {
|
||||
t.Errorf("arch %q served %s", arch, p)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormMAC(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"52:54:00:AB:cd:01": "52:54:00:ab:cd:01",
|
||||
"52-54-00-ab-cd-01": "52:54:00:ab:cd:01",
|
||||
"../../etc/x": "",
|
||||
"52:54:00:ab:cd:01\nx": "",
|
||||
"": "",
|
||||
} {
|
||||
if got := normMAC(in); got != want {
|
||||
t.Errorf("normMAC(%q) = %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
if n := pxeFileName("../../../etc"); pxeNameRe.MatchString(n) {
|
||||
t.Errorf("%q passes as a pxelinux file name", n)
|
||||
}
|
||||
if !pxeNameRe.MatchString(pxeFileName("52:54:00:ab:cd:01")) {
|
||||
t.Error("a real MAC does not make a pxelinux file name")
|
||||
}
|
||||
}
|
||||
|
||||
func TestValidDomain(t *testing.T) {
|
||||
for _, ok := range []string{"node01", "node01.int.example.com", "a-b.c"} {
|
||||
if !validDomain(ok) {
|
||||
t.Errorf("%q refused", ok)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"", "*.int.example", "a b", "x\ndhcp-script=/x", "-a", "a..b", "Node01"} {
|
||||
if validDomain(bad) {
|
||||
t.Errorf("%q accepted", bad)
|
||||
}
|
||||
}
|
||||
// a reservation name that is not a plain name never reaches DNS
|
||||
if n := dnsNameIn(IPAddr{Hostname: "victim.int.example. 300 IN A 6.6.6.6 ;.int.example"}, nil, "int.example"); n != "" {
|
||||
t.Errorf("dns name %q", n)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRedfishResolveStaysOnBMC(t *testing.T) {
|
||||
r := &rfClient{base: "https://10.0.0.11:8443"}
|
||||
for _, ok := range []string{"/redfish/v1/Systems/1", "/redfish/v1/Systems/1/", "https://10.0.0.11:8443/redfish/v1/TaskService/Tasks/5"} {
|
||||
if _, err := r.resolve(ok); err != nil {
|
||||
t.Errorf("%s: %v", ok, err)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"@evil.example/x", "https://evil.example/redfish/v1", "/api/x", "/redfish/../etc", "//evil.example/redfish/v1"} {
|
||||
if p, err := r.resolve(bad); err == nil {
|
||||
t.Errorf("%s resolved to %s", bad, p)
|
||||
}
|
||||
}
|
||||
if b := bmcBase(&bmcCreds{Address: "https://u:p@bmc:8443/x/y"}, "https", 443); b != "https://invalid.invalid" {
|
||||
t.Errorf("userinfo address gave %s", b)
|
||||
}
|
||||
if b := bmcBase(&bmcCreds{Address: "https://bmc:8443/x"}, "https", 443); b != "https://bmc:8443" {
|
||||
t.Errorf("base %s", b)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIdentitiesOnlyFromTheRoot(t *testing.T) {
|
||||
root, _ := newCA("test root")
|
||||
ikey, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
dns, nets, _ := parsePermits("int.example.com")
|
||||
_, icert, err := root.intermediate(&ikey.PublicKey, "acme", dns, nets, time.Hour)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
k, _ := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
|
||||
_, direct, _ := root.sign(&k.PublicKey, "admin:alice", nil, x509.ExtKeyUsageClientAuth)
|
||||
_, viaInter, _ := (&CA{Cert: icert, Key: ikey}).sign(&k.PublicKey, "admin:alice", nil, x509.ExtKeyUsageClientAuth)
|
||||
r := httptest.NewRequest("GET", "/", nil)
|
||||
r.TLS = &tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{{direct, root.Cert}}}
|
||||
if rootIssued(r) == nil {
|
||||
t.Error("a root-issued certificate was not accepted")
|
||||
}
|
||||
r.TLS = &tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{{viaInter, icert, root.Cert}}}
|
||||
if rootIssued(r) != nil {
|
||||
t.Error("a certificate from an intermediate passed as an identity")
|
||||
}
|
||||
if err := regionLeaf(tls.ConnectionState{VerifiedChains: [][]*x509.Certificate{{direct, root.Cert}}}); err == nil {
|
||||
t.Error("an admin certificate passed as the region's")
|
||||
}
|
||||
}
|
||||
|
||||
// The hint API names the BMC but never hands out its password.
|
||||
func TestBMCHintHasNoPassword(t *testing.T) {
|
||||
a := &App{cfg: &Config{BMCHints: []BMCHint{{MAC: "52:54:00:AA:BB:01", Address: "10.0.0.11", User: "admin", Password: "s3cret"}}}}
|
||||
rec := httptest.NewRecorder()
|
||||
a.handleHint(rec, httptest.NewRequest("GET", "/api/v1/bmc-hint?macs=02:00:00:00:00:09,52-54-00-aa-bb-01", nil))
|
||||
var h map[string]any
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &h)
|
||||
if h["address"] != "10.0.0.11" || h["user"] != "admin" || h["password"] != nil || strings.Contains(rec.Body.String(), "s3cret") {
|
||||
t.Errorf("hint %s", rec.Body.String())
|
||||
}
|
||||
if a.hintFor([]string{"../x", ""}) != nil {
|
||||
t.Error("garbage matched a hint")
|
||||
}
|
||||
}
|
||||
|
||||
// X-Forwarded-For counts only from a rack controller, and only its own entry.
|
||||
func TestClientIP(t *testing.T) {
|
||||
a := &App{cfg: &Config{RackName: "region", RackSecret: "rs"}}
|
||||
r := httptest.NewRequest("GET", "/api/v1/jobs/self", nil)
|
||||
r.RemoteAddr = "10.213.77.60:41000"
|
||||
r.Header.Set("X-Forwarded-For", "10.213.77.50")
|
||||
if ip := a.clientIP(r); ip != "10.213.77.60" {
|
||||
t.Errorf("a machine's own X-Forwarded-For was believed: %s", ip)
|
||||
}
|
||||
r.RemoteAddr = "10.0.0.151:5000"
|
||||
r.Header.Set("X-Rollcall-Rack", "hall-b")
|
||||
r.Header.Set("X-Rollcall-Rack-Secret", "rs")
|
||||
r.Header.Set("X-Forwarded-For", "10.213.78.99, 10.213.78.50")
|
||||
if ip := a.clientIP(r); ip != "10.213.78.50" {
|
||||
t.Errorf("through a rack: %s", ip)
|
||||
}
|
||||
}
|
||||
|
||||
// An HTTPS BMC is trusted by the certificate it showed first; another one
|
||||
// gets no credentials.
|
||||
func TestBMCCertificatePin(t *testing.T) {
|
||||
seen := 0
|
||||
srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
if _, _, ok := r.BasicAuth(); ok {
|
||||
seen++
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"PowerState":"On","Members":[{"@odata.id":"/redfish/v1/Systems/1"}]}`))
|
||||
}))
|
||||
defer srv.Close()
|
||||
c := creds(srv.URL, "redfish")
|
||||
fp, err := powerRun(context.Background(), "", c, "tls-fingerprint", "")
|
||||
if err != nil || fp != certSHA256(srv.Certificate()) {
|
||||
t.Fatalf("fingerprint %q %v", fp, err)
|
||||
}
|
||||
if seen != 0 {
|
||||
t.Error("credentials were sent while reading the certificate")
|
||||
}
|
||||
c.Extra["tls_fingerprint"] = fp
|
||||
if _, err := powerRun(context.Background(), "", c, "status", ""); err != nil {
|
||||
t.Fatalf("pinned and matching: %v", err)
|
||||
}
|
||||
c.Extra["tls_fingerprint"] = strings.Repeat("ab", 32)
|
||||
seen = 0
|
||||
if _, err := powerRun(context.Background(), "", c, "status", ""); err == nil || !strings.Contains(err.Error(), "certificate changed") || seen != 0 {
|
||||
t.Errorf("another certificate: %v (credentials sent %d times)", err, seen)
|
||||
}
|
||||
}
|
||||
|
||||
// A rack's HTTPS URL is its own host; the region's addresses get no rack certificate.
|
||||
func TestAgentTLSURLs(t *testing.T) {
|
||||
ok := RackController{URL: "http://10.213.78.1", TLSURL: "https://10.213.78.1"}
|
||||
if validTLSURL(ok) != "https://10.213.78.1" {
|
||||
t.Error("own host refused")
|
||||
}
|
||||
for _, bad := range []string{"https://10.0.0.234", "http://10.213.78.1", "https://10.213.78.1/x", "https://evil.example"} {
|
||||
if validTLSURL(RackController{URL: "http://10.213.78.1", TLSURL: bad}) != "" {
|
||||
t.Errorf("%s accepted", bad)
|
||||
}
|
||||
}
|
||||
a := &App{cfg: &Config{ServerURL: "http://10.213.77.1", TLSNames: []string{"region.example"}}}
|
||||
for _, h := range []string{"10.213.77.1", "region.example", "127.0.0.1", "localhost"} {
|
||||
if !a.isRegionName(h) {
|
||||
t.Errorf("%s is the region's", h)
|
||||
}
|
||||
}
|
||||
if a.isRegionName("10.213.78.1") {
|
||||
t.Error("a rack's address taken for the region's")
|
||||
}
|
||||
}
|
||||
+164
-33
@@ -39,6 +39,12 @@ type App struct {
|
||||
ipamSt *ipamState
|
||||
|
||||
adminCC adminCertCache
|
||||
|
||||
dnsp dnsProvider // nil = no direct DNS writes
|
||||
dnsSt *dnsState
|
||||
// fixed leases on the site's DHCP server (dhcpres.go); nil = none
|
||||
dhcpRes dhcpResProvider
|
||||
dhcpResSt *dhcpResState
|
||||
}
|
||||
|
||||
type apiError struct {
|
||||
@@ -148,9 +154,14 @@ func logging(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
start := time.Now()
|
||||
rw := &statusWriter{ResponseWriter: w, code: 200}
|
||||
r.Body = http.MaxBytesReader(rw, r.Body, 16<<20) // handlers set tighter limits; nothing takes more
|
||||
next.ServeHTTP(rw, r)
|
||||
if (!strings.HasPrefix(r.URL.Path, "/rollcall/") && !strings.HasPrefix(r.URL.Path, "/images/") && !strings.HasPrefix(r.URL.Path, "/os/")) || rw.code >= 400 {
|
||||
log.Printf("%s %s %s %d %s", r.RemoteAddr, r.Method, r.URL.RequestURI(), rw.code, time.Since(start).Round(time.Millisecond))
|
||||
uri := r.URL.RequestURI()
|
||||
if r.URL.Query().Has("token") { // /?token=... signs the UI in; it does not go to the journal
|
||||
uri = r.URL.Path + "?token=REDACTED"
|
||||
}
|
||||
log.Printf("%s %s %s %d %s", r.RemoteAddr, r.Method, uri, rw.code, time.Since(start).Round(time.Millisecond))
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -193,6 +204,14 @@ func (a *App) routes() http.Handler {
|
||||
mux.HandleFunc("GET /api/v1/nodes/self/overlay", a.handleNodeOverlay) // node token auth inside
|
||||
// disk-based deployment
|
||||
mux.HandleFunc("GET /api/v1/os", a.adminAuth(a.handleOS))
|
||||
mux.HandleFunc("GET /api/v1/dhcp-reservations", a.adminAuth(a.handleDHCPRes))
|
||||
mux.HandleFunc("POST /api/v1/dhcp-reservations/sync", a.adminAuth(a.handleDHCPResSync))
|
||||
mux.HandleFunc("GET /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo))
|
||||
mux.HandleFunc("POST /api/v1/firmware", a.adminAuth(a.handleFirmwareRepo))
|
||||
mux.HandleFunc("DELETE /api/v1/firmware/{pkg}", a.adminAuth(a.handleFirmwareRemove))
|
||||
mux.HandleFunc("GET /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/firmware", a.adminAuth(a.handleMachineFirmware))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/firmware/update", a.adminAuth(a.handleFirmwareUpdate))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/deploy", a.adminAuth(a.handleDeploy))
|
||||
mux.HandleFunc("POST /api/v1/machines/{id}/release", a.adminAuth(a.handleRelease))
|
||||
mux.HandleFunc("GET /api/v1/machines/{id}/jobs", a.adminAuth(a.handleJobs))
|
||||
@@ -233,6 +252,8 @@ func (a *App) routes() http.Handler {
|
||||
mux.HandleFunc("POST /api/v1/ips", a.adminAuth(a.handleIPs))
|
||||
mux.HandleFunc("DELETE /api/v1/ips/{ip}", a.adminAuth(a.handleIPDelete))
|
||||
mux.HandleFunc("GET /api/v1/ipam", a.adminAuth(a.handleIPAM))
|
||||
mux.HandleFunc("GET /api/v1/dns", a.adminAuth(a.handleDNS))
|
||||
mux.HandleFunc("POST /api/v1/dns/sync", a.adminAuth(a.handleDNSSync))
|
||||
mux.HandleFunc("GET /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))
|
||||
mux.HandleFunc("POST /api/v1/admin-certs", a.adminAuth(a.handleAdminCerts))
|
||||
mux.HandleFunc("DELETE /api/v1/admin-certs/{ref}", a.adminAuth(a.handleAdminCertRevoke))
|
||||
@@ -251,6 +272,7 @@ func (a *App) routes() http.Handler {
|
||||
mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode))
|
||||
mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only
|
||||
mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew))
|
||||
mux.HandleFunc("POST /api/v1/rack/server-cert", a.rackAuth(a.handleRackServerCert))
|
||||
mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister))
|
||||
mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState))
|
||||
mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork))
|
||||
@@ -272,10 +294,28 @@ func (a *App) routes() http.Handler {
|
||||
mux.Handle("GET /rollcall/", http.StripPrefix("/rollcall/", http.FileServer(http.Dir(imgDir))))
|
||||
mux.Handle("GET /images/", http.StripPrefix("/images/", http.FileServer(http.Dir(a.cfg.ImagesDir))))
|
||||
mux.Handle("GET /os/", http.StripPrefix("/os/", http.FileServer(http.Dir(a.cfg.OSDir))))
|
||||
mux.HandleFunc("GET /firmware/{pkg}/{file}", a.handleFirmwareFile) // BMCs download packages (no listing)
|
||||
// embedded web UI
|
||||
sub, _ := fs.Sub(webFS, "web")
|
||||
mux.Handle("GET /", a.uiOverHTTPS(http.FileServer(http.FS(sub))))
|
||||
return logging(mux)
|
||||
mux.Handle("GET /", a.uiOverHTTPS(uiHeaders(http.FileServer(http.FS(sub)))))
|
||||
// Browsers send admin client certificates on their own, so a page elsewhere
|
||||
// could post to the API with the admin's identity: cross-origin requests
|
||||
// that change state are refused (Sec-Fetch-Site / Origin). The CLI, agents
|
||||
// and racks send neither header and are not affected.
|
||||
return logging(http.NewCrossOriginProtection().Handler(mux))
|
||||
}
|
||||
|
||||
// uiHeaders: the UI loads only its own scripts and cannot be framed, so
|
||||
// markup that slips through (machine-reported strings) does not run.
|
||||
func uiHeaders(next http.Handler) http.Handler {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
h := w.Header()
|
||||
h.Set("Content-Security-Policy", "default-src 'self'; script-src 'self'; style-src 'self' 'unsafe-inline'; img-src 'self' data:; connect-src 'self'; object-src 'none'; frame-ancestors 'none'; base-uri 'none'; form-action 'self'")
|
||||
h.Set("X-Content-Type-Options", "nosniff")
|
||||
h.Set("X-Frame-Options", "DENY")
|
||||
h.Set("Referrer-Policy", "no-referrer")
|
||||
next.ServeHTTP(w, r)
|
||||
})
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- helpers over inventory JSON
|
||||
@@ -322,14 +362,23 @@ func listLen(v any) int {
|
||||
return 0
|
||||
}
|
||||
|
||||
func normMAC(s string) string { return strings.ToLower(strings.ReplaceAll(s, "-", ":")) }
|
||||
// normMAC is the canonical form of a hardware address, "" when s is not one.
|
||||
// MACs name files (pxelinux.cfg/01-...) and dnsmasq lines, so nothing else passes.
|
||||
func normMAC(s string) string {
|
||||
hw, err := net.ParseMAC(strings.TrimSpace(s))
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
return hw.String()
|
||||
}
|
||||
|
||||
var nonAlnum = regexp.MustCompile(`[^a-z0-9]`)
|
||||
|
||||
func machineID(uuid, serial string, macs []string) string {
|
||||
u := strings.ToLower(uuid)
|
||||
if u != "" && u != "00000000-0000-0000-0000-000000000000" && !strings.Contains(u, "not ") {
|
||||
u = strings.ReplaceAll(u, "-", "")
|
||||
if u != "" && u != "00000000-0000-0000-0000-000000000000" && !strings.Contains(u, "not ") &&
|
||||
nonAlnum.ReplaceAllString(u, "") != "" {
|
||||
u = nonAlnum.ReplaceAllString(u, "") // the ID names files and URLs: letters and digits only
|
||||
if len(u) > 12 {
|
||||
u = u[:12]
|
||||
}
|
||||
@@ -459,7 +508,7 @@ func (a *App) writePXE(m *Machine) error {
|
||||
// ---------------------------------------------------------------- handlers
|
||||
|
||||
func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 32<<20))
|
||||
body, err := io.ReadAll(http.MaxBytesReader(w, r.Body, 4<<20))
|
||||
if err != nil {
|
||||
fail(w, 400, err)
|
||||
return
|
||||
@@ -486,20 +535,53 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
}
|
||||
}
|
||||
// identity strings come from the machine: one line each, they go into PXE menus and logs
|
||||
line := func(v any) *string {
|
||||
s := strings.Map(func(c rune) rune {
|
||||
if c < ' ' || c == 0x7f {
|
||||
return ' '
|
||||
}
|
||||
return c
|
||||
}, str(v))
|
||||
if s = strings.TrimSpace(s); len(s) > 128 {
|
||||
s = s[:128]
|
||||
}
|
||||
return strPtr(s)
|
||||
}
|
||||
m := &Machine{
|
||||
ID: machineID(str(ident["uuid"]), str(ident["serial"]), macs),
|
||||
UUID: strPtr(ident["uuid"]),
|
||||
Serial: strPtr(ident["serial"]),
|
||||
Hostname: strPtr(inv["hostname"]),
|
||||
Manufacturer: strPtr(dig(inv, "system", "system-manufacturer")),
|
||||
Product: strPtr(dig(inv, "system", "system-product-name")),
|
||||
UUID: line(ident["uuid"]),
|
||||
Serial: line(ident["serial"]),
|
||||
Hostname: line(inv["hostname"]),
|
||||
Manufacturer: line(dig(inv, "system", "system-manufacturer")),
|
||||
Product: line(dig(inv, "system", "system-product-name")),
|
||||
MACs: macs,
|
||||
}
|
||||
// Pull the BMC password out of the payload before anything is persisted.
|
||||
var secret []byte
|
||||
detail := "bmc=none"
|
||||
if bmc, _ := inv["bmc"].(obj); bmc == nil || str(bmc["source"]) == "hint" || str(bmc["address"]) == "" {
|
||||
// a configured out-of-band BMC: the region's credentials, whatever the agent sent
|
||||
if h := a.hintFor(macs); h != nil {
|
||||
port := h.Port
|
||||
if port == 0 {
|
||||
port = 623
|
||||
}
|
||||
nb := obj{"source": "hint", "address": h.Address, "port": port, "user": h.User, "configured": true, "password": "<from bmc_hints>"}
|
||||
if bmc != nil {
|
||||
nb["agent"] = obj{"source": bmc["source"], "errors": bmc["errors"]}
|
||||
}
|
||||
inv["bmc"] = nb
|
||||
if h.Password != "" {
|
||||
if secret, err = a.sealer.Seal(h.Password, m.ID); err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if bmc, ok := inv["bmc"].(obj); ok {
|
||||
if pw := str(bmc["password"]); pw != "" {
|
||||
if pw := str(bmc["password"]); pw != "" && str(bmc["source"]) != "hint" {
|
||||
secret, err = a.sealer.Seal(pw, m.ID)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
@@ -512,9 +594,20 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
m.Inventory, _ = json.Marshal(inv)
|
||||
|
||||
// Anyone on the provisioning network can post here, so a known machine is
|
||||
// only re-commissioned when that is expected: an operator asked
|
||||
// (recommission), or it has nothing to lose (ready, a failed job). A report
|
||||
// for a deployed or running machine is refused, not merged.
|
||||
old, oldErr := a.store.Get(r.Context(), m.ID)
|
||||
if oldErr == nil && !recommissionable(old.State) {
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "commission refused", fmt.Sprintf("state %s; from %s", old.State, r.RemoteAddr))
|
||||
log.Printf("commission refused for %s (state %s) from %s", m.ID, old.State, r.RemoteAddr)
|
||||
fail(w, 409, fmt.Errorf("machine %s is %s; recommission it first", m.ID, old.State))
|
||||
return
|
||||
}
|
||||
// a BMC an operator configured (driver_set) survives commissioning; what the agent
|
||||
// found is kept next to it for reference
|
||||
if old, err := a.store.Get(r.Context(), m.ID); err == nil && old.BMC != nil {
|
||||
if oldErr == nil && old.BMC != nil {
|
||||
var ob obj
|
||||
if json.Unmarshal(old.BMC, &ob) == nil && ob["driver_set"] == true {
|
||||
var found obj
|
||||
@@ -533,10 +626,18 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
return
|
||||
}
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail)
|
||||
_ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r))
|
||||
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
|
||||
go a.probeBMC(full)
|
||||
if isNew || oldErr != nil || old.State == "recommission" || old.RackController == "" {
|
||||
_ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r)) // a ready machine is not moved by whoever reports it
|
||||
}
|
||||
go func(id string) { // check a hinted BMC from where it is reachable, then look for Redfish
|
||||
ctx := context.Background()
|
||||
if full, err := a.store.Get(ctx, id); err == nil {
|
||||
a.verifyHintedBMC(full)
|
||||
}
|
||||
if full, err := a.store.Get(ctx, id); err == nil {
|
||||
a.probeBMC(full)
|
||||
}
|
||||
}(m.ID)
|
||||
if full, err := a.store.Get(r.Context(), m.ID); err == nil {
|
||||
m = full // carries profile / node token when this is a re-commission of a stateless node
|
||||
}
|
||||
@@ -550,22 +651,41 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) {
|
||||
writeJSON(w, 200, map[string]any{"id": m.ID, "state": "ready", "new": isNew})
|
||||
}
|
||||
|
||||
func (a *App) handleHint(w http.ResponseWriter, r *http.Request) {
|
||||
func recommissionable(state string) bool {
|
||||
return state == "new" || state == "ready" || state == "recommission" || strings.HasSuffix(state, "-failed")
|
||||
}
|
||||
|
||||
// hintFor is the configured out-of-band BMC for one of these MACs.
|
||||
func (a *App) hintFor(macs []string) *BMCHint {
|
||||
want := map[string]bool{}
|
||||
for _, m := range strings.Split(r.URL.Query().Get("macs"), ",") {
|
||||
want[normMAC(m)] = true
|
||||
}
|
||||
for _, h := range a.cfg.BMCHints {
|
||||
if want[normMAC(h.MAC)] {
|
||||
port := h.Port
|
||||
if port == 0 {
|
||||
port = 623
|
||||
}
|
||||
writeJSON(w, 200, obj{"address": h.Address, "port": port, "user": h.User, "password": h.Password})
|
||||
return
|
||||
for _, m := range macs {
|
||||
if n := normMAC(m); n != "" {
|
||||
want[n] = true
|
||||
}
|
||||
}
|
||||
writeJSON(w, 200, obj{})
|
||||
for i, h := range a.cfg.BMCHints {
|
||||
if want[normMAC(h.MAC)] {
|
||||
return &a.cfg.BMCHints[i]
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// handleHint tells a commissioning agent that a BMC is configured for it: the
|
||||
// address, never the password (the agent token is public, MACs are visible on
|
||||
// the wire). The region attaches the credentials itself when the machine
|
||||
// reports in (handleCommission) and checks them from the rack controller.
|
||||
func (a *App) handleHint(w http.ResponseWriter, r *http.Request) {
|
||||
h := a.hintFor(strings.Split(r.URL.Query().Get("macs"), ","))
|
||||
if h == nil {
|
||||
writeJSON(w, 200, obj{})
|
||||
return
|
||||
}
|
||||
port := h.Port
|
||||
if port == 0 {
|
||||
port = 623
|
||||
}
|
||||
writeJSON(w, 200, obj{"address": h.Address, "port": port, "user": h.User, "credentials": "on the region"})
|
||||
}
|
||||
|
||||
func (a *App) handleList(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -672,7 +792,11 @@ func (a *App) handlePower(w http.ResponseWriter, r *http.Request) {
|
||||
if action != "status" {
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "power "+action, out)
|
||||
}
|
||||
writeJSON(w, 200, obj{"id": m.ID, "action": action, "ok": err == nil, "output": out})
|
||||
res := obj{"id": m.ID, "action": action, "ok": err == nil, "output": out}
|
||||
if err != nil {
|
||||
res["error"] = err.Error()
|
||||
}
|
||||
writeJSON(w, 200, res)
|
||||
}
|
||||
|
||||
func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) {
|
||||
@@ -725,7 +849,14 @@ func serve(cfgPath string) error {
|
||||
if err := app.startIPAM(); err != nil {
|
||||
return fmt.Errorf("ipam: %w", err)
|
||||
}
|
||||
if err := app.startDHCPRes(); err != nil {
|
||||
return err
|
||||
}
|
||||
if err := app.startDNS(); err != nil {
|
||||
return fmt.Errorf("dns: %w", err)
|
||||
}
|
||||
go app.jobWatchdog()
|
||||
go app.resumeFirmware()
|
||||
handler := app.routes()
|
||||
if cfg.TLSListen != "" {
|
||||
names := append([]string{}, cfg.TLSNames...)
|
||||
@@ -738,7 +869,7 @@ func serve(cfgPath string) error {
|
||||
}
|
||||
pool := x509.NewCertPool()
|
||||
pool.AddCert(app.ca.Cert)
|
||||
tsrv := &http.Server{Addr: cfg.TLSListen, Handler: handler, ReadHeaderTimeout: 10 * time.Second,
|
||||
tsrv := &http.Server{Addr: cfg.TLSListen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute,
|
||||
TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{pair}, ClientCAs: pool, ClientAuth: tls.VerifyClientCertIfGiven}}
|
||||
go func() {
|
||||
log.Printf("rollcall HTTPS on %s (CA %s)", cfg.TLSListen, app.ca.Fingerprint())
|
||||
@@ -747,7 +878,7 @@ func serve(cfgPath string) error {
|
||||
}
|
||||
}()
|
||||
}
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: handler, ReadHeaderTimeout: 10 * time.Second}
|
||||
srv := &http.Server{Addr: cfg.Listen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, ReadTimeout: 2 * time.Minute, IdleTimeout: 2 * time.Minute}
|
||||
log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "")
|
||||
return srv.ListenAndServe()
|
||||
}
|
||||
@@ -124,6 +124,9 @@ type nodeAssignment struct {
|
||||
}
|
||||
|
||||
func (s *Store) Assign(ctx context.Context, id string, n nodeAssignment) error {
|
||||
if !validDomain(n.Name) {
|
||||
return fmt.Errorf("node name %q is not a host name (letters, digits, hyphens)", n.Name)
|
||||
}
|
||||
tag, err := s.pool.Exec(ctx, `UPDATE machines SET profile=$2, node_name=$3, mgmt_ip=$4, ib_ip=$5,
|
||||
node_token_hash=$6, node_token=$7, state='stateless', booted_at=NULL WHERE id=$1`,
|
||||
id, n.Profile, n.Name, nilIfEmpty(n.MgmtIP), nilIfEmpty(n.IBIP), n.TokenHash, n.Token)
|
||||
@@ -379,6 +382,7 @@ var tmplFuncs = template.FuncMap{
|
||||
"sub": func(a, b int) int { return a - b },
|
||||
"pct": func(v, p int) int { return v * p / 100 },
|
||||
"seq": func(n int) []int {
|
||||
n = max(0, min(n, 4096)) // match rules run on every commissioning: no unbounded loops
|
||||
s := make([]int, n)
|
||||
for i := range s {
|
||||
s[i] = i
|
||||
@@ -650,6 +654,7 @@ func (a *App) statelessAppend(m *Machine, p *Profile, token string) string {
|
||||
"hostname=" + deref(m.NodeName),
|
||||
"live-config.nocomponents=user-setup,sudo",
|
||||
"rollcall.server=" + base, "rollcall.node=" + token}
|
||||
parts = append(parts, a.agentTLSArgs(base)...)
|
||||
if p.Append != "" {
|
||||
parts = append(parts, p.Append)
|
||||
}
|
||||
@@ -743,6 +748,11 @@ func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 400, fmt.Errorf("profile is required"))
|
||||
return
|
||||
}
|
||||
req.Name = strings.ToLower(req.Name)
|
||||
if req.Name != "" && !validDomain(req.Name) {
|
||||
fail(w, 400, fmt.Errorf("name %q is not a host name (letters, digits, hyphens)", req.Name))
|
||||
return
|
||||
}
|
||||
p, err := a.store.Profile(r.Context(), req.Profile)
|
||||
if err != nil {
|
||||
fail(w, 404, fmt.Errorf("profile %s: %w", req.Profile, err))
|
||||
@@ -867,6 +877,10 @@ func (a *App) handleNodeOverlay(w http.ResponseWriter, r *http.Request) {
|
||||
fail(w, 401, errors.New("unknown node token"))
|
||||
return
|
||||
}
|
||||
if err := a.tokenSource(r.Context(), r, m, nil, "node"); err != nil {
|
||||
fail(w, 403, err)
|
||||
return
|
||||
}
|
||||
if m.State != "booted" {
|
||||
_ = a.store.AddEvent(r.Context(), m.ID, "booted", r.RemoteAddr)
|
||||
log.Printf("node %s (%s) booted from %s", deref(m.NodeName), m.ID, r.RemoteAddr)
|
||||
@@ -893,7 +907,12 @@ func (a *App) handleRender(w http.ResponseWriter, r *http.Request) {
|
||||
if stage == "" {
|
||||
stage = "system"
|
||||
}
|
||||
data, err := a.renderFile(r.Context(), r.PathValue("profile"), stage, path.Clean(r.PathValue("path")))
|
||||
rel := path.Clean("/" + r.PathValue("path"))[1:] // rooted first, so ".." cannot climb out of the overlay
|
||||
if (stage != "system" && stage != "runtime") || rel == "" {
|
||||
fail(w, 400, errors.New("stage is system or runtime, path a file in the overlay"))
|
||||
return
|
||||
}
|
||||
data, err := a.renderFile(r.Context(), r.PathValue("profile"), stage, rel)
|
||||
if err != nil {
|
||||
fail(w, 500, err)
|
||||
return
|
||||
|
||||
@@ -73,6 +73,7 @@ CREATE TABLE IF NOT EXISTS jobs (
|
||||
finished timestamptz,
|
||||
error text
|
||||
);
|
||||
ALTER TABLE jobs ADD COLUMN IF NOT EXISTS bound_ip text;
|
||||
CREATE INDEX IF NOT EXISTS jobs_machine ON jobs(machine_id, created);
|
||||
CREATE INDEX IF NOT EXISTS jobs_token ON jobs(token_hash);
|
||||
CREATE TABLE IF NOT EXISTS tags (
|
||||
@@ -164,8 +165,12 @@ CREATE TABLE IF NOT EXISTS rack_controllers (
|
||||
registered timestamptz NOT NULL DEFAULT now()
|
||||
);
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text;
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS tls_url text;
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_serial text;
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS prev_cert_until timestamptz;
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz;
|
||||
ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS arch text;
|
||||
ALTER TABLE vlans ADD COLUMN IF NOT EXISTS dhcp_mode text NOT NULL DEFAULT '';
|
||||
CREATE TABLE IF NOT EXISTS join_tokens (
|
||||
hash text PRIMARY KEY,
|
||||
name text,
|
||||
@@ -235,6 +240,12 @@ func openStore(ctx context.Context, url string) (*Store, error) {
|
||||
if _, err := pool.Exec(ctx, adminCertSchema); err != nil {
|
||||
return nil, fmt.Errorf("schema: %w", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, dnsRecordsSchema); err != nil {
|
||||
return nil, fmt.Errorf("schema: %w", err)
|
||||
}
|
||||
if _, err := pool.Exec(ctx, firmwareSchema); err != nil {
|
||||
return nil, fmt.Errorf("schema: %w", err)
|
||||
}
|
||||
return &Store{pool: pool}, nil
|
||||
}
|
||||
|
||||
|
||||
+71
-16
@@ -125,7 +125,7 @@ function render() {
|
||||
case 'network': main.innerHTML = pageNetwork(); wireNetwork(); break;
|
||||
case 'tags': main.innerHTML = pageTags(); wireTags(); break;
|
||||
case 'profiles': main.innerHTML = pageProfiles(); wireProfiles(); break;
|
||||
case 'images': main.innerHTML = pageImages(); break;
|
||||
case 'images': main.innerHTML = pageImages(); wireImages(); break;
|
||||
default: main.innerHTML = pageOverview(); wireOverview();
|
||||
}
|
||||
}
|
||||
@@ -281,7 +281,7 @@ function pageMachines() {
|
||||
<td class="check"><input type="checkbox" data-sel="${esc(m.id)}"${S.sel.has(m.id) ? ' checked' : ''}></td>
|
||||
<td><b>${esc(nameOf(m))}</b><br><span class="muted mono">${esc(m.mgmt_ip || m.id)}</span></td>
|
||||
<td>${stateCell(m)}</td>
|
||||
<td>${m.location && (m.location.rack || m.location.room || m.location.building) ? `${esc([m.location.building, m.location.room].filter(Boolean).join(' / '))}<br><b>${esc(m.location.rack || '')}</b>${m.location.unit ? ` <span class="muted">U${m.location.unit}</span>` : ''}` : '<span class="muted">—</span>'}</td>
|
||||
<td>${m.location && (m.location.rack || m.location.room || m.location.building) ? `${esc([m.location.building, m.location.room].filter(Boolean).join(' / '))}<br><b>${esc(m.location.rack || '')}</b>${m.location.unit ? ` <span class="muted">U${esc(m.location.unit)}</span>` : ''}` : '<span class="muted">—</span>'}</td>
|
||||
<td>${m.profile ? esc(m.profile) : m.os ? `${esc(m.os)}<br><span class="muted mono">${esc(m.os_version || '')}</span>` : '<span class="muted">—</span>'}</td>
|
||||
<td>${tagPills(m)}</td>
|
||||
<td>${esc(((m.manufacturer || '') + ' ' + (m.product || '')).trim().slice(0, 26))}<br><span class="muted mono">${esc((m.serial || '').slice(0, 20))}</span></td>
|
||||
@@ -355,7 +355,7 @@ async function each(ids, label, fn) {
|
||||
await refresh(true);
|
||||
}
|
||||
const act = {
|
||||
power: (ids, action) => each(ids, `power ${action}`, async id => { const r = await api('POST', `/api/v1/machines/${id}/power`, { action }); return r.output || (r.ok ? 'ok' : 'failed'); }),
|
||||
power: (ids, action) => each(ids, `power ${action}`, async id => { const r = await api('POST', `/api/v1/machines/${id}/power`, { action }); if (!r.ok) throw new Error(r.error || r.output || 'the BMC command failed'); return r.output || 'ok'; }),
|
||||
bootdev: (ids, device) => each(ids, `bootdev ${device}`, async id => { const r = await api('POST', `/api/v1/machines/${id}/bootdev`, { device }); return r.output || (r.ok ? 'ok' : 'failed'); }),
|
||||
async deploy(ids) {
|
||||
const one = ids.length === 1 ? S.machines.find(m => m.id === ids[0]) : null;
|
||||
@@ -446,14 +446,14 @@ function table(cols, rows) { return rows.length ? `<table><thead><tr>${cols.map(
|
||||
function jobsTable(jobs, withMachine) {
|
||||
return table([...(withMachine ? ['Machine'] : []), 'Job', 'Kind', 'State', 'Target', 'Progress', 'Started', 'Finished', 'Error'], jobs.map(j => {
|
||||
const sp = j.spec || {}, p = j.progress || {}; const m = S.machines.find(x => x.id === j.machine_id);
|
||||
const prog = (j.state === 'running' || j.state === 'pending') ? `<div class="bar"><span style="width:${Math.round(p.pct || 0)}%"></span></div><span class="muted">${esc(p.phase || '')} ${Math.round(p.pct || 0)}% ${esc((p.msg || '').slice(0, 60))}</span>` : (p.details && p.details.seconds ? `<span class="muted">${p.details.seconds}s</span>` : '');
|
||||
const prog = (j.state === 'running' || j.state === 'pending') ? `<div class="bar"><span style="width:${Math.round(p.pct || 0)}%"></span></div><span class="muted">${esc(p.phase || '')} ${Math.round(p.pct || 0)}% ${esc((p.msg || '').slice(0, 60))}</span>` : (p.details && p.details.seconds ? `<span class="muted">${esc(p.details.seconds)}s</span>` : '');
|
||||
return [...(withMachine ? [`<a href="#/machine/${esc(j.machine_id)}">${esc(m ? nameOf(m) : j.machine_id)}</a>`] : []), `<span class="mono">${esc(j.id)}</span>`, esc(j.kind), pill(j.state), `${esc(sp.os || '')}${sp.version ? ' <span class="muted mono">' + esc(sp.version) + '</span>' : ''}${sp.disk ? ' → <span class="mono">' + esc(sp.disk) + '</span>' : ''}`, prog, `<span class="muted" title="${esc(when(j.started || j.created))}">${ago(j.started || j.created)}</span>`, `<span class="muted">${esc(when(j.finished))}</span>`, j.error ? `<span class="bad">${esc(j.error)}</span>` : ''];
|
||||
}));
|
||||
}
|
||||
function pageMachine(d, tab) {
|
||||
const s = d.summary, inv = d.inventory || {}, sys = inv.system || {}, cpu = inv.cpu || {}, mem = inv.memory || {}, st = inv.storage || {}, b = d.bmc || {};
|
||||
const m = S.machines.find(x => x.id === s.id) || s;
|
||||
const tabs = ['overview', 'hardware', 'network', 'bmc', 'jobs', 'events', 'raw'];
|
||||
const tabs = ['overview', 'hardware', 'network', 'bmc', 'firmware', 'jobs', 'events', 'raw'];
|
||||
let body = '';
|
||||
if (tab === 'overview') {
|
||||
const job = s.job || (d.jobs || [])[0]; const p = (job && job.progress) || {}; const active = job && (job.state === 'pending' || job.state === 'running');
|
||||
@@ -471,15 +471,15 @@ function pageMachine(d, tab) {
|
||||
['BMC', b.address ? `<span class="dot ${b.configured ? 'ok' : 'bad'}"></span><span class="mono">${esc(b.address)}</span> ${esc(b.user || '')} · ${d.has_bmc_secret ? '<span class="ok">credentials stored</span>' : '<span class="bad">no credentials</span>'}` : '<span class="bad">none</span>'],
|
||||
['Current job', job ? `<span class="mono">${esc(job.id)}</span> ${esc(job.kind)} ${pill(job.state)}${active ? `<div class="bar" style="margin-top:4px"><span style="width:${Math.round(p.pct || 0)}%"></span></div><span class="muted">${esc(p.phase || '')} ${Math.round(p.pct || 0)}% ${esc(p.msg || '')}</span>` : ''}${job.error ? `<br><span class="bad">${esc(job.error)}</span>` : ''}` : ''],
|
||||
]))}
|
||||
${card(`Storage · ${st.disk_count ?? 0} disks · ${human(st.total_bytes)}`, table(['Dev', 'Size', 'Bus', 'Kind', 'Model', 'Serial', 'Health'], (st.disks || []).map(x => { const sm = (st.smart || {})[x.name] || {}; return [esc(x.name), human(x.size), esc(x.tran), x.rota === false ? 'ssd' : 'hdd', esc(x.model), `<span class="mono">${esc(x.serial)}</span>`, sm.healthy === true ? '<span class="ok">ok</span>' : sm.healthy === false ? '<span class="bad">FAIL</span>' : '']; })))}
|
||||
${card(`Storage · ${esc(st.disk_count ?? 0)} disks · ${human(st.total_bytes)}`, table(['Dev', 'Size', 'Bus', 'Kind', 'Model', 'Serial', 'Health'], (st.disks || []).map(x => { const sm = (st.smart || {})[x.name] || {}; return [esc(x.name), human(x.size), esc(x.tran), x.rota === false ? 'ssd' : 'hdd', esc(x.model), `<span class="mono">${esc(x.serial)}</span>`, sm.healthy === true ? '<span class="ok">ok</span>' : sm.healthy === false ? '<span class="bad">FAIL</span>' : '']; })))}
|
||||
${card(`Network · ${(inv.network || []).length} NICs`, table(['If', 'MAC', 'Speed', 'Link', 'Switch port'], (inv.network || []).map(n => { const l = ((inv.lldp || {})[n.name]); const e = l ? (Array.isArray(l) ? l[0] : l) : null; const sw = e ? `${Object.keys(e.chassis || {})[0] || ''} ${((e.port || {}).id || {}).value || ''}` : ''; return [esc(n.name), `<span class="mono">${esc(n.mac)}</span>`, esc(n.speed), n.link === true ? '<span class="ok">up</span>' : n.link === false ? '<span class="muted">down</span>' : '', esc(sw)]; })))}
|
||||
${card('Recent events', table(['When', 'Event', 'Detail'], (d.events || []).slice(-8).reverse().map(e => [`<span class="muted" title="${esc(when(e.ts))}">${ago(e.ts)}</span>`, esc(e.event), `<span class="mono muted">${esc(e.detail.slice(0, 90))}</span>`])), true)}
|
||||
</div>`;
|
||||
} else if (tab === 'hardware') {
|
||||
body = `<div class="grid">
|
||||
${card('CPU', kv([['Model', esc(cpu['Model name'])], ['Sockets × cores × threads', `${esc(cpu['Socket(s)'])} × ${esc(cpu['Core(s) per socket'])} × ${esc(cpu['Thread(s) per core'])} = ${esc(cpu['CPU(s)'])}`], ['NUMA nodes', esc(cpu['NUMA node(s)'])], ['Max MHz', esc(cpu['CPU max MHz'])], ['L3', esc(cpu['L3 cache'])], ['Virtualization', esc(cpu['Virtualization'])], ['Features', esc((cpu.features || []).join(' '))]]))}
|
||||
${card(`Memory ${human(mem.total_bytes)} · ${mem.dimms_populated ?? '?'}/${mem.dimm_slots ?? '?'} DIMMs`, table(['Slot', 'Size', 'Type', 'Speed', 'Vendor', 'Part', 'Serial'], (mem.dimms || []).filter(x => x.populated).map(x => [esc(x.locator), esc(x.size), esc(x.type), esc(x.configured_speed || x.speed), esc(x.manufacturer), `<span class="mono">${esc(x.part_number)}</span>`, `<span class="mono">${esc(x.serial)}</span>`])))}
|
||||
${card(`Storage · ${st.disk_count ?? 0} disks · ${human(st.total_bytes)}`, table(['Dev', 'Size', 'Bus', 'Kind', 'Model', 'Serial', 'FW', 'Hours', 'Health'], (st.disks || []).map(x => { const sm = (st.smart || {})[x.name] || {}; return [esc(x.name), human(x.size), esc(x.tran), x.rota === false ? 'ssd' : 'hdd', esc(x.model), `<span class="mono">${esc(x.serial)}</span>`, esc(sm.firmware), esc(sm.power_on_hours), sm.healthy === true ? '<span class="ok">ok</span>' : sm.healthy === false ? '<span class="bad">FAIL</span>' : '']; })) + ((st.controllers || []).length ? '<p class="muted">Controllers</p>' + table(['PCI', 'Vendor', 'Device', 'Driver'], st.controllers.map(c => [`<span class="mono">${esc(c.slot)}</span>`, esc(c.vendor), esc(c.device), esc(c.driver)])) : ''), true)}
|
||||
${card(`Memory ${human(mem.total_bytes)} · ${esc(mem.dimms_populated ?? '?')}/${esc(mem.dimm_slots ?? '?')} DIMMs`, table(['Slot', 'Size', 'Type', 'Speed', 'Vendor', 'Part', 'Serial'], (mem.dimms || []).filter(x => x.populated).map(x => [esc(x.locator), esc(x.size), esc(x.type), esc(x.configured_speed || x.speed), esc(x.manufacturer), `<span class="mono">${esc(x.part_number)}</span>`, `<span class="mono">${esc(x.serial)}</span>`])))}
|
||||
${card(`Storage · ${esc(st.disk_count ?? 0)} disks · ${human(st.total_bytes)}`, table(['Dev', 'Size', 'Bus', 'Kind', 'Model', 'Serial', 'FW', 'Hours', 'Health'], (st.disks || []).map(x => { const sm = (st.smart || {})[x.name] || {}; return [esc(x.name), human(x.size), esc(x.tran), x.rota === false ? 'ssd' : 'hdd', esc(x.model), `<span class="mono">${esc(x.serial)}</span>`, esc(sm.firmware), esc(sm.power_on_hours), sm.healthy === true ? '<span class="ok">ok</span>' : sm.healthy === false ? '<span class="bad">FAIL</span>' : '']; })) + ((st.controllers || []).length ? '<p class="muted">Controllers</p>' + table(['PCI', 'Vendor', 'Device', 'Driver'], st.controllers.map(c => [`<span class="mono">${esc(c.slot)}</span>`, esc(c.vendor), esc(c.device), esc(c.driver)])) : ''), true)}
|
||||
${(inv.gpus || []).length ? card('GPUs / accelerators', table(['PCI', 'Vendor', 'Device', 'Driver'], inv.gpus.map(g => [`<span class="mono">${esc(g.slot)}</span>`, esc(g.vendor), esc(g.device), esc(g.driver)])), true) : ''}
|
||||
${card('System', kv([['Board', esc([sys['baseboard-manufacturer'], sys['baseboard-product-name'], sys['baseboard-serial-number']].filter(Boolean).join(' / '))], ['SKU', esc(sys['system-sku-number'])], ['Chassis', esc([sys['chassis-type'], sys['chassis-serial-number'], sys['chassis-asset-tag']].filter(Boolean).join(' / '))], ['TPM', esc((inv.tpm || []).join(', ') || 'none')], ['Kernel (commissioning)', esc((inv.kernel || '').split(' (')[0])], ['Agent run', esc(inv.duration_s ? inv.duration_s + ' s' : '')]]))}
|
||||
${(inv.pci || []).length ? card(`PCI devices · ${inv.pci.length}`, table(['Slot', 'Class', 'Vendor', 'Device', 'Driver'], inv.pci.map(p => [`<span class="mono">${esc(p.slot)}</span>`, esc(p.class), esc(p.vendor), esc(p.device), esc(p.driver)])), true) : ''}
|
||||
@@ -496,11 +496,13 @@ function pageMachine(d, tab) {
|
||||
} else if (tab === 'bmc') {
|
||||
body = `<div class="actions" style="margin-bottom:12px"><button class="btn primary" id="bmc-edit">Edit BMC…</button><button class="btn" id="bmc-probe" title="ask the BMC whether it speaks Redfish with the stored account">Detect Redfish</button><span class="muted">Commands run on rack controller ${esc(s.rack_controller || '')}</span></div>
|
||||
<div class="grid">${card('BMC', kv([['Driver', `<b>${esc(b.driver || 'ipmi')}</b>${b.driver_set ? ' <span class="muted">(set by operator)</span>' : ''}${b.redfish ? ` · Redfish ${esc(b.redfish.redfish)} ${esc(b.redfish.manufacturer || '')} ${esc(b.redfish.model || '')}` : ''}`],
|
||||
['Redfish offers', b.redfish && (b.redfish.reset_types || b.redfish.boot_targets) ? `<span class="mono">${esc((b.redfish.reset_types || []).join(' '))}</span><br><span class="muted">boot</span> <span class="mono">${esc((b.redfish.boot_targets || []).join(' '))}</span>${b.redfish.boot_mode ? ` · mode ${esc(b.redfish.boot_mode)}` : ''}${b.redfish.profile ? `<br><span class="muted">profile ${esc(b.redfish.profile)}</span>` : ''}` : ''],
|
||||
['Settings', b.extra && Object.keys(b.extra).length ? `<span class="mono">${esc(Object.entries(b.extra).map(([k, v]) => k + '=' + v).join(' '))}</span>` : ''],
|
||||
['Source', esc(b.source)], ['Address', `<span class="mono">${esc(b.address)}${b.port && b.port !== 623 ? ':' + esc(b.port) : ''}</span>`], ['MAC', `<span class="mono">${esc(b.mac)}</span>`], ['User', esc(b.user)], ['Password', d.has_bmc_secret ? '<span class="ok">stored encrypted on the server</span>' : '<span class="bad">none</span>'], ['Configured', b.configured ? '<span class="ok">yes</span>' : '<span class="bad">no</span>'], ['Verified out-of-band', b.verified_oob === true ? '<span class="ok">yes</span>' : b.verified_oob === false ? '<span class="bad">no</span>' : ''], ['Channel', esc(b.channel)], ['LAN', b.lan ? `<span class="mono">${esc(JSON.stringify(b.lan))}</span>` : ''], ['Firmware', esc((b.mc_info || {})['Firmware Revision'])], ['Manufacturer', esc((b.mc_info || {})['Manufacturer Name'])], ['Errors', (b.errors || []).length ? `<span class="bad">${esc(b.errors.join('; '))}</span>` : '']]))}
|
||||
${(b.users_before || []).length ? card('User table before commissioning', table(['ID', 'Name', 'Priv', 'IPMI'], b.users_before.map(u => [u.id, esc(u.name), esc(u.priv), u.ipmi_msg ? 'yes' : 'no']))) : ''}
|
||||
${(b.users_before || []).length ? card('User table before commissioning', table(['ID', 'Name', 'Priv', 'IPMI'], b.users_before.map(u => [esc(u.id), esc(u.name), esc(u.priv), u.ipmi_msg ? 'yes' : 'no']))) : ''}
|
||||
${b.fru ? card('FRU', `<pre class="json">${esc(JSON.stringify(b.fru, null, 1))}</pre>`) : ''}</div>`;
|
||||
} else if (tab === 'jobs') body = jobsTable(d.jobs || [], false);
|
||||
} else if (tab === 'firmware') body = `<div class="actions" style="margin-bottom:12px"><button class="btn" id="fw-read" title="read the firmware inventory from the BMC (Redfish)">Read from BMC</button><span class="muted">Packages come from the repository (OS images page); the update runs on rack controller ${esc(s.rack_controller || '')}</span></div><div id="fwbox" class="muted">loading…</div>`;
|
||||
else if (tab === 'jobs') body = jobsTable(d.jobs || [], false);
|
||||
else if (tab === 'events') body = table(['Time', 'Event', 'Detail'], (d.events || []).slice().reverse().map(e => [esc(when(e.ts)), esc(e.event), `<span class="mono">${esc(e.detail)}</span>`]));
|
||||
else body = `<pre class="json">${esc(JSON.stringify(d, null, 1))}</pre>`;
|
||||
|
||||
@@ -521,6 +523,7 @@ function pageMachine(d, tab) {
|
||||
function wireMachine(d) {
|
||||
const id = d.summary.id, ids = [id];
|
||||
if (route().tab === 'network') wireNetPlan(id);
|
||||
if (route().tab === 'firmware') loadFirmware(id, false);
|
||||
wireMenus();
|
||||
$$('#dpower [data-item]').forEach(b => b.onclick = () => act.power(ids, b.dataset.item));
|
||||
$$('#dboot [data-item]').forEach(b => b.onclick = () => act.bootdev(ids, b.dataset.item));
|
||||
@@ -534,6 +537,54 @@ function wireMachine(d) {
|
||||
$$('[data-untag]').forEach(a => a.onclick = ev => { ev.preventDefault(); each(ids, 'untag', async () => (await api('POST', `/api/v1/machines/${id}/tags`, { remove: [a.dataset.untag] })).effective.join(' ')); });
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- firmware
|
||||
async function loadFirmware(id, read) {
|
||||
const box = $('#fwbox'); if (!box) return;
|
||||
const btn = $('#fw-read'); if (btn) btn.onclick = () => loadFirmware(id, true);
|
||||
if (read) box.innerHTML = '<span class="muted">reading the inventory from the BMC…</span>';
|
||||
let d;
|
||||
try { d = await api(read ? 'POST' : 'GET', `/api/v1/machines/${id}/firmware`); } catch (e) { box.innerHTML = `<span class="bad">${esc(e.message)}</span>`; return; }
|
||||
const inv = (d.inventory || {}).components || [];
|
||||
const offers = d.offers || [];
|
||||
box.className = '';
|
||||
box.innerHTML = `<div class="grid">
|
||||
${card(`Installed · ${inv.length} components${d.inventory && d.inventory.at && inv.length ? ` · read ${esc(ago(d.inventory.at))} ago` : ''}`, inv.length ? table(['Component', 'Name', 'Version', 'Updateable'], inv.map(c => [esc(c.component), esc(c.name), `<span class="mono">${esc(c.version)}</span>`, c.updateable ? 'yes' : '<span class="muted">no</span>'])) : '<span class="muted">not read yet: Read from BMC</span>', true)}
|
||||
${card(`Repository for ${esc(d.driver || '?')} ${esc(d.model || '')}`, offers.length ? table(['Component', 'Version', 'Installed now', '', ''], offers.map(o => [esc(o.package.component), `<span class="mono">${esc(o.package.version)}</span>`, `<span class="mono">${esc(o.installed || '—')}</span>`, o.same ? '<span class="ok">installed</span>' : '<span class="warn">differs</span>', `<button class="btn sm" data-fwpkg="${esc(o.package.id)}">${o.same ? 'Reinstall…' : 'Update…'}</button>`])) : '<span class="muted">no package in the repository for this vendor and model</span>', true)}
|
||||
</div>`;
|
||||
$$('[data-fwpkg]').forEach(b => b.onclick = () => fwUpdate(id, offers.find(o => o.package.id === b.dataset.fwpkg).package));
|
||||
}
|
||||
async function fwUpdate(id, p) {
|
||||
const v = await dialog(`Update ${p.component} to ${p.version}`, [
|
||||
{ name: 'apply', label: 'Apply', type: 'select', value: '', options: [['', "the BMC's default"], ['OnReset', 'at the next reset'], ['Immediate', 'immediately']] },
|
||||
{ name: 'reboot', label: 'Power-cycle the host afterwards and verify the version', type: 'checkbox', value: p.component === 'bios' },
|
||||
{ name: 'force', label: 'Send even if vendor or model does not match', type: 'checkbox', value: false },
|
||||
], `${p.file} (${human(p.size)}). The BMC checks the package itself; an update cannot be cancelled once the BMC has it. A BMC update restarts the BMC for a few minutes.`, 'Start update');
|
||||
if (!v) return;
|
||||
try { const r = await api('POST', `/api/v1/machines/${id}/firmware/update`, { package: p.id, apply: v.apply, reboot: v.reboot, force: v.force }); toast(`firmware job ${r.job} started`); await loadDetail(id); } catch (e) { toast(e.message, true); }
|
||||
}
|
||||
function wireImages() {
|
||||
const el = $('#fwrepo'); if (!el) return;
|
||||
const show = () => api('GET', '/api/v1/firmware').then(ps => {
|
||||
el.className = '';
|
||||
el.innerHTML = ps.length ? table(['Package', 'Vendor', 'Component', 'Version', 'Models', 'Size', 'Imported', ''], ps.map(p => [`<span class="mono">${esc(p.id)}</span><br><span class="muted">${esc(p.file)}</span>`, esc(p.vendor), esc(p.component), `<span class="mono">${esc(p.version)}</span>`, esc((p.models || []).join(', ') || 'any'), human(p.size), esc(when(p.imported)), `<button class="btn sm" data-fwrm="${esc(p.id)}">remove</button>`])) : '<span class="muted">no packages yet</span>';
|
||||
$$('[data-fwrm]').forEach(b => b.onclick = async () => { if (confirm(`Remove ${b.dataset.fwrm}?`)) { try { await api('DELETE', '/api/v1/firmware/' + b.dataset.fwrm); show(); } catch (e) { toast(e.message, true); } } });
|
||||
}).catch(e => { el.innerHTML = `<span class="bad">${esc(e.message)}</span>`; });
|
||||
show();
|
||||
$('#fwimport').onclick = async () => {
|
||||
const drivers = await api('GET', '/api/v1/power-drivers').then(ds => ds.filter(d => d.family === 'redfish' && d.name !== 'redfish').map(d => [d.name, d.description]), () => []);
|
||||
const v = await dialog('Import a firmware package', [
|
||||
{ name: 'url', label: 'Download URL', placeholder: 'https://dl.dell.com/FOLDER.../BIOS_XXXX_WN64_2.19.1.EXE' },
|
||||
{ name: 'vendor', label: 'Vendor (BMC driver)', type: 'select', options: drivers },
|
||||
{ name: 'component', label: 'Component', type: 'select', options: ['bios', 'bmc', 'nic', 'raid', 'cpld', 'psu', 'drive', 'other'].map(c => [c, c]) },
|
||||
{ name: 'version', label: 'Version', placeholder: 'as the component reports it once installed' },
|
||||
{ name: 'models', label: 'Models', placeholder: 'R650, R750 (empty: any model of this vendor)' },
|
||||
{ name: 'sha256', label: 'SHA-256', placeholder: "from the vendor's download page (recommended)" },
|
||||
], 'The region downloads the package; rack controllers serve it to the BMCs. A local file: rollcall firmware import FILE on the region.', 'Import');
|
||||
if (!v) return;
|
||||
try { const p = await api('POST', '/api/v1/firmware', { url: v.url.trim(), vendor: v.vendor, component: v.component, version: v.version.trim(), models: v.models.split(',').map(x => x.trim()).filter(Boolean), sha256: v.sha256.trim() }); toast(`imported ${p.id}`); show(); } catch (e) { toast(e.message, true); }
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- jobs
|
||||
function pageJobs() {
|
||||
const active = S.jobs.filter(j => j.state === 'pending' || j.state === 'running'), rest = S.jobs.filter(j => !(j.state === 'pending' || j.state === 'running'));
|
||||
@@ -637,6 +688,9 @@ function wireProfiles() {
|
||||
api('GET', '/api/v1/images').then(imgs => { const el = $('#rootimgs'); if (el) el.innerHTML = table(['Image', 'Complete', 'Squashfs', 'Built'], imgs.map(i => [esc(i.name), i.complete ? '<span class="ok">yes</span>' : '<span class="bad">no</span>', human(i['filesystem.squashfs']), esc(when(i.built))])); }).catch(() => { });
|
||||
}
|
||||
function pageImages() {
|
||||
return pageOSImages() + `<div class="card wide" style="margin-top:16px"><h3>Firmware packages <button class="btn sm" id="fwimport" style="margin-left:8px">Import…</button></h3><div id="fwrepo" class="muted">loading…</div><p class="muted" style="margin:8px 0 0">Updates run from a machine's Firmware tab, or <code>rollcall firmware update MACHINE|tag:T PACKAGE</code>.</p></div>`;
|
||||
}
|
||||
function pageOSImages() {
|
||||
return `<div class="page-head"><h1>OS images</h1><span class="sub">disk images for deployment · refreshed nightly by <code>rollcall os sync</code> from the vendors' checksum files</span></div>
|
||||
${table(['OS', 'Current version', 'Family', 'Compressed', 'Disk needed', 'Imported', 'Track', 'Keep', 'Versions', 'Status'], S.os.map(i => { const c = i.current || {}; const s = i.source || {}; return [`<b>${esc(i.name)}</b>${s.description ? `<br><span class="muted">${esc(s.description)}</span>` : ''}`, `<span class="mono">${esc(c.version || '—')}</span>`, esc(s.family || c.family), c.size ? human(c.size) : '', c.raw_size ? human(c.raw_size) : '', c.imported ? `${esc(when(c.imported))} <span class="muted">(${ago(c.imported)})</span>` : '', esc(s.track || ''), s.keep || '', (i.versions || []).map(v => `<span class="mono${v.current ? '' : ' muted'}">${esc(v.version)}</span>`).join('<br>'), i.warning ? `<span class="warn">${esc(i.warning)}</span>` : '<span class="ok">ok</span>']; }))}
|
||||
<p class="muted" style="margin-top:10px">Pinned sources keep <code>current</code> until <code>rollcall os promote NAME VERSION</code>; a deployment always uses the exact version it started with. Import a local file with <code>rollcall os import NAME FILE</code>.</p>`;
|
||||
@@ -696,7 +750,7 @@ function pageNetwork() {
|
||||
const sizeTxt = s.size ? s.size : '∞';
|
||||
const ranges = (s.ranges || []).map(r => `<span class="pill tag" title="${esc(r.comment || '')}">${esc(r.kind)} ${esc(r.start)}–${esc(r.end.split('.').pop())} <a href="#" data-rmrange="${r.id}" title="remove">×</a></span>`).join(' ');
|
||||
return [`<b class="mono">${esc(s.cidr)}</b>${s.name ? `<br><span class="muted">${esc(s.name)}</span>` : ''}`, esc(s.purpose), `${esc(s.fabric || '—')} / ${s.vid || 'untagged'}${s.mtu && s.mtu !== 1500 ? ` · mtu ${s.mtu}` : ''}`,
|
||||
s.dhcp ? `<span class="ok">on</span> · ${esc(s.primary_rack)}${s.secondary_rack ? ' + ' + esc(s.secondary_rack) : ''}` : '<span class="muted">off</span>', `<span class="mono">${esc(s.gateway || '—')}</span>`,
|
||||
s.dhcp ? `<span class="ok">${s.dhcp_mode === 'proxy' ? 'proxy' : 'on'}</span> · ${esc(s.primary_rack)}${s.secondary_rack ? ' + ' + esc(s.secondary_rack) : ''}` : '<span class="muted">off</span>', `<span class="mono">${esc(s.gateway || '—')}</span>`,
|
||||
`<span class="mono">${esc((s.dns || []).join(', '))}</span>${s.domain ? `<br><span class="muted">${esc(s.domain)}</span>` : ''}`, `<div class="bar" title="${s.used}/${s.size}"><span style="width:${pct}%"></span></div><span class="muted">${s.used} / ${sizeTxt}</span>${s.family === 6 ? ' <span class="pill tag">IPv6</span>' : ''}`,
|
||||
ranges + ` <button class="btn sm" data-addrange="${esc(s.cidr)}">+ range</button>`, `<button class="btn sm" data-ips="${esc(s.cidr)}">addresses</button> <button class="btn sm" data-editsubnet="${esc(s.cidr)}">edit</button> <button class="btn sm danger" data-rmsubnet="${esc(s.cidr)}">delete</button>`];
|
||||
}));
|
||||
@@ -723,7 +777,7 @@ async function editSubnet(s) {
|
||||
{ name: 'purpose', label: 'Purpose', type: 'select', value: s ? s.purpose : 'provisioning', options: ['provisioning', 'mgmt', 'bmc', 'storage', 'ib', 'other'].map(x => [x, x]) },
|
||||
{ name: 'gateway', label: 'Gateway', value: s ? s.gateway : '' }, { name: 'dns', label: 'DNS servers', value: s ? (s.dns || []).join(',') : '', placeholder: 'comma separated' },
|
||||
{ name: 'domain', label: 'Domain', value: s ? s.domain : '' }, { name: 'mtu', label: 'VLAN MTU', value: s ? String(s.mtu || 1500) : '1500' },
|
||||
{ name: 'dhcp', label: 'Serve DHCP/PXE', type: 'checkbox', value: s ? s.dhcp : false },
|
||||
{ name: 'dhcp', label: 'DHCP / PXE', type: 'select', value: s ? (s.dhcp ? (s.dhcp_mode || 'serve') : 'off') : 'off', options: [['off', 'off: another network, or later'], ['serve', 'serve: this rack hands out addresses and PXE'], ['proxy', 'proxy: the site\'s DHCP server hands out addresses; this rack only answers PXE']] },
|
||||
{ name: 'rack', label: 'Rack controller', type: 'select', value: s ? s.primary_rack : '', options: S.controllers.map(c => [c.name, `${c.name}${c.online ? '' : ' (offline)'}`]) },
|
||||
], 'Ranges are added afterwards: dynamic (PXE pool), reserved (never handed out), static (where machines get addresses).', 'Save');
|
||||
if (!v) return;
|
||||
@@ -732,7 +786,7 @@ async function editSubnet(s) {
|
||||
if (fabric && !S.fabrics.some(f => f.name === fabric)) await api('POST', '/api/v1/fabrics', { name: fabric });
|
||||
const vid = Number(v.vid) || 0;
|
||||
await api('POST', '/api/v1/subnets', { cidr: v.cidr.trim(), name: v.name, fabric, vid, purpose: v.purpose, gateway: v.gateway.trim(), dns: v.dns.split(',').map(x => x.trim()).filter(Boolean), domain: v.domain.trim() });
|
||||
if (fabric) await api('POST', '/api/v1/vlans', { fabric, vid, mtu: Number(v.mtu) || 1500, dhcp: v.dhcp, primary_rack: v.dhcp ? v.rack : '' });
|
||||
if (fabric) await api('POST', '/api/v1/vlans', { fabric, vid, mtu: Number(v.mtu) || 1500, dhcp: v.dhcp !== 'off', dhcp_mode: v.dhcp === 'proxy' ? 'proxy' : '', primary_rack: v.dhcp !== 'off' ? v.rack : '' });
|
||||
toast(`subnet ${v.cidr} saved`); refresh(true);
|
||||
} catch (e) { toast(e.message, true); }
|
||||
}
|
||||
@@ -777,15 +831,16 @@ function wireNetwork() {
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- BMC driver settings
|
||||
const powerDriverNames = [['ipmi', 'IPMI 2.0 (ipmitool)'], ['redfish', 'Redfish, any vendor'], ['idrac', 'Dell iDRAC 8/9'], ['ilo', 'HPE iLO 4/5/6'], ['supermicro', 'Supermicro (Redfish)'], ['xcc', 'Lenovo XClarity'], ['openbmc', 'OpenBMC'], ['proxmox', 'Proxmox VE VM'], ['manual', 'manual (no power control)']];
|
||||
async function editBMC(id, b) {
|
||||
// the server's list: one entry per vendor profile it knows
|
||||
const drivers = await api('GET', '/api/v1/power-drivers').then(ds => ds.map(d => [d.name, d.description]), () => [['ipmi', 'IPMI 2.0'], ['redfish', 'Redfish']]);
|
||||
const v = await dialog('BMC settings', [
|
||||
{ name: 'driver', label: 'Driver', type: 'select', value: b.driver || 'ipmi', options: powerDriverNames },
|
||||
{ name: 'driver', label: 'Driver', type: 'select', value: b.driver || 'ipmi', options: drivers },
|
||||
{ name: 'address', label: 'Address', value: b.address || '', placeholder: '10.0.0.5, bmc.example or https://host:8443' },
|
||||
{ name: 'port', label: 'Port', value: b.port ? String(b.port) : '', placeholder: "driver's default" },
|
||||
{ name: 'user', label: 'User', value: b.user || '', help: 'proxmox: token id, e.g. root@pam!rollcall' },
|
||||
{ name: 'password', label: 'Password', placeholder: 'leave empty to keep the stored one', help: 'proxmox: the token secret' },
|
||||
{ name: 'extra', label: 'Settings', type: 'textarea', rows: 3, value: Object.entries(b.extra || {}).map(([k, x]) => `${k}=${x}`).join('\n'), placeholder: 'system=/redfish/v1/Systems/1\nverify_tls=true\ncipher=17\nnode=pve1\nvmid=211', help: 'one KEY=VALUE per line; a line KEY= removes it' },
|
||||
{ name: 'extra', label: 'Settings', type: 'textarea', rows: 3, value: Object.entries(b.extra || {}).map(([k, x]) => `${k}=${x}`).join('\n'), placeholder: 'system=/redfish/v1/Systems/1\nboot_mode=UEFI\nverify_tls=true\ncipher=17\nnode=pve1\nvmid=211', help: 'one KEY=VALUE per line; a line KEY= removes it' },
|
||||
{ name: 'verify', label: 'Check with a status call before saving', type: 'checkbox', value: true },
|
||||
], 'The password is sealed on the region and only travels to the rack controller over mutual TLS.', 'Save');
|
||||
if (!v) return;
|
||||
|
||||
@@ -130,7 +130,7 @@ function topoSVG(opt = {}) {
|
||||
o.push(`<line x1="${M + 250}" y1="${y}" x2="${W - M}" y2="${y}"/>`);
|
||||
o.push(`<text class="t1" x="${M}" y="${y - 4}">${esc((g.fabric || 'no fabric') + ' · ' + (g.vid ? 'vlan ' + g.vid : 'untagged'))}</text>`);
|
||||
if (!compact) o.push(`<text class="t2" x="${M}" y="${y + 11}">${esc([...v4, ...v6].map(s => s.cidr).join(' '))}</text>`);
|
||||
o.push(`<text class="t3" x="${W - M}" y="${y - 5}" text-anchor="end">${esc(g.purpose)}${g.dhcp ? ' · DHCP/PXE' : ''}${g.mtu && g.mtu !== 1500 ? ' · mtu ' + g.mtu : ''} · ${used} addr</text>`);
|
||||
o.push(`<text class="t3" x="${W - M}" y="${y - 5}" text-anchor="end">${esc(g.purpose)}${g.dhcp ? (g.subnets.some(s => s.dhcp_mode === 'proxy') ? ' · proxy DHCP (PXE only)' : ' · DHCP/PXE') : ''}${g.mtu && g.mtu !== 1500 ? ' · mtu ' + g.mtu : ''} · ${used} addr</text>`);
|
||||
o.push('</g>');
|
||||
});
|
||||
// region + external IPAM
|
||||
|
||||
+37
-17
@@ -25,7 +25,9 @@
|
||||
# --nat UPLINK this host routes for them: forward + masquerade out of UPLINK
|
||||
# (default with --iface and no --gateway: the default route's interface)
|
||||
# --domain D DNS domain handed to machines (default rollcall.internal)
|
||||
# --no-dhcp model the network but leave DHCP to the existing server
|
||||
# --no-dhcp model the network but leave DHCP to the existing server (no PXE from here)
|
||||
# --proxy-dhcp the site's DHCP server keeps handing out addresses; this host only answers
|
||||
# PXE clients with where to boot from (no ranges, no NAT)
|
||||
# --ssh-key K SSH public key (or a file of them) for root on deployed machines; repeatable.
|
||||
# Default: the keys of the user running sudo, then root's.
|
||||
# --os NAME import one OS image now (e.g. debian-13); a nightly timer syncs the catalog
|
||||
@@ -46,6 +48,7 @@ while [ $# -gt 0 ]; do
|
||||
--os) OS=$2; shift ;;
|
||||
--ssh-key) SSHKEYS+=("$2"); shift ;;
|
||||
--no-dhcp) DHCP=0 ;;
|
||||
--proxy-dhcp) DHCP=2 ;;
|
||||
--yes|-y) YES=1 ;;
|
||||
-h|--help) sed -n '2,31p' "$0"; exit 0 ;;
|
||||
*) echo "unknown option $1 (--help)" >&2; exit 2 ;;
|
||||
@@ -70,9 +73,10 @@ if [ -n "$IFACE" ]; then
|
||||
[ -n "$CIDR_ADDR" ] || die "$IFACE has no IPv4 address; give it its address on the provisioning network first"
|
||||
read -r ADDR NET PREFIX FIRST <<<"$(python3 -c 'import ipaddress,sys;i=ipaddress.ip_interface(sys.argv[1]);print(i.ip,i.network,i.network.prefixlen,i.network.network_address)' "$CIDR_ADDR")"
|
||||
[ "$PREFIX" -le 24 ] || die "$NET is smaller than a /24; model it by hand (rollcall subnet add / range add)"
|
||||
if [ -z "$GATEWAY" ]; then GATEWAY=$ADDR; NAT=${NAT:-$UPLINK}; fi
|
||||
case $DHCP in 1) DHCPTXT="served here" ;; 2) DHCPTXT="proxy (the site's server hands out addresses)" ;; *) DHCPTXT=off ;; esac
|
||||
if [ -z "$GATEWAY" ] && [ $DHCP != 2 ]; then GATEWAY=$ADDR; NAT=${NAT:-$UPLINK}; fi
|
||||
[ -n "$NAT" ] && [ "$NAT" = "$IFACE" ] && die "--nat must be the uplink, not $IFACE"
|
||||
echo " provisioning $IFACE $CIDR_ADDR (subnet $NET), machines' gateway $GATEWAY${NAT:+ via NAT out of $NAT}, DHCP $([ $DHCP = 1 ] && echo on || echo off)"
|
||||
echo " provisioning $IFACE $CIDR_ADDR (subnet $NET), machines' gateway ${GATEWAY:-from the site DHCP server}${NAT:+ via NAT out of $NAT}, DHCP $DHCPTXT"
|
||||
else
|
||||
[ -n "$UPLINK" ] || die "no default route: give this host network access first"
|
||||
ADDR=$(addr_of "$UPLINK" | cut -d/ -f1)
|
||||
@@ -91,7 +95,7 @@ fi
|
||||
# ------------------------------------------------------------------ 1. packages
|
||||
say "1/9 packages"
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
PKGS="postgresql dnsmasq pxelinux syslinux-common syslinux-efi qemu-utils ipmitool jq zstd curl ca-certificates python3 nftables lldpd"
|
||||
PKGS="postgresql dnsmasq pxelinux syslinux-common syslinux-efi shim-signed grub-efi-amd64-signed qemu-utils ipmitool jq zstd curl ca-certificates python3 nftables lldpd"
|
||||
[ $BUILD = 1 ] && PKGS="$PKGS live-build debootstrap squashfs-tools xorriso"
|
||||
apt-get update -q >/dev/null
|
||||
apt-get install -yq $PKGS >/dev/null
|
||||
@@ -111,9 +115,9 @@ if [ -f "$CONF" ]; then
|
||||
else
|
||||
PGPASS=$(head -c 32 /dev/urandom | base64 | tr -dc A-Za-z0-9 | head -c 28)
|
||||
if [ "$(runuser -u postgres -- psql -tAc "select 1 from pg_roles where rolname='rollcall'")" = 1 ]; then
|
||||
runuser -u postgres -- psql -q -c "ALTER ROLE rollcall PASSWORD '$PGPASS'"
|
||||
printf "ALTER ROLE rollcall PASSWORD '%s';\n" "$PGPASS" | runuser -u postgres -- psql -q # stdin: not in ps
|
||||
else
|
||||
runuser -u postgres -- psql -q -c "CREATE ROLE rollcall LOGIN PASSWORD '$PGPASS'"
|
||||
printf "CREATE ROLE rollcall LOGIN PASSWORD '%s';\n" "$PGPASS" | runuser -u postgres -- psql -q
|
||||
fi
|
||||
[ "$(runuser -u postgres -- psql -tAc "select 1 from pg_database where datname='rollcall'")" = 1 ] || runuser -u postgres -- psql -q -c "CREATE DATABASE rollcall OWNER rollcall"
|
||||
ok "role and database rollcall"
|
||||
@@ -237,15 +241,21 @@ if [ -s $IMG/vmlinuz ] && [ -s $IMG/initrd.img ] && [ -s $IMG/filesystem.squashf
|
||||
ok "present, kept"
|
||||
elif [ -n "$IMAGE_URL" ]; then
|
||||
tmp=$(mktemp -d -p /srv/rollcall)
|
||||
auth=(); [ -n "${IMAGE_TOKEN:-}" ] && auth=(-H "Authorization: token $IMAGE_TOKEN")
|
||||
pre=""
|
||||
if ! curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS"; then
|
||||
pre=commissioning- # a release: the image files carry a prefix
|
||||
curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/${pre}SHA256SUMS" || die "no SHA256SUMS or ${pre}SHA256SUMS at $IMAGE_URL"
|
||||
auth=()
|
||||
if [ -n "${IMAGE_TOKEN:-}" ]; then # the token goes through a file, not the command line
|
||||
printf 'Authorization: token %s\n' "$IMAGE_TOKEN" > "$tmp/.auth" && chmod 600 "$tmp/.auth"
|
||||
auth=(-H "@$tmp/.auth")
|
||||
fi
|
||||
# a release carries the image as commissioning-* (its plain SHA256SUMS is the binaries');
|
||||
# a region's /rollcall directory has the plain names
|
||||
pre=commissioning-
|
||||
if ! curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/${pre}SHA256SUMS"; then
|
||||
pre=""
|
||||
curl -fsSL "${auth[@]}" -o "$tmp/SHA256SUMS" "$IMAGE_URL/SHA256SUMS" || die "no commissioning-SHA256SUMS or SHA256SUMS at $IMAGE_URL${IMAGE_TOKEN:+}$( [ -z "${IMAGE_TOKEN:-}" ] && echo ' (private repository? set IMAGE_TOKEN)')"
|
||||
fi
|
||||
for f in vmlinuz initrd.img filesystem.squashfs; do curl -fSL --progress-bar "${auth[@]}" -o "$tmp/$f" "$IMAGE_URL/$pre$f"; done
|
||||
(cd "$tmp" && sha256sum -c --quiet SHA256SUMS) || die "checksum mismatch"
|
||||
mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/ && rm -rf "$tmp"
|
||||
mv "$tmp"/vmlinuz "$tmp"/initrd.img "$tmp"/filesystem.squashfs "$tmp"/SHA256SUMS $IMG/; rm -rf "$tmp"
|
||||
chmod 0644 $IMG/*
|
||||
ok "downloaded and verified from $IMAGE_URL"
|
||||
elif [ $BUILD = 1 ]; then
|
||||
@@ -269,17 +279,27 @@ else
|
||||
FAB=provisioning
|
||||
rollcall fabric list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$FAB" || rollcall fabric add "$FAB" >/dev/null
|
||||
if ! rollcall subnet list 2>/dev/null | awk 'NR>1{print $1}' | grep -qx "$NET"; then
|
||||
rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null
|
||||
rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null
|
||||
rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null
|
||||
rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null
|
||||
ok "subnet $NET: reserved $(off 1)-$(off 20), static $(off 50)-$(off 99), dynamic $(off 100)-$(off 200)"
|
||||
if [ $DHCP = 2 ]; then
|
||||
# the site's DHCP server owns the addresses: no ranges (machines keep DHCP);
|
||||
# set a static range aside later (rollcall range add) to give them fixed ones
|
||||
rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning ${GATEWAY:+-gateway "$GATEWAY"} -domain "$DOMAIN" -name provisioning >/dev/null
|
||||
ok "subnet $NET (addresses from the site's DHCP server)"
|
||||
else
|
||||
rollcall subnet add "$NET" -fabric "$FAB" -vid 0 -purpose provisioning -gateway "$GATEWAY" -dns "$ADDR" -domain "$DOMAIN" -name provisioning >/dev/null
|
||||
rollcall range add "$NET" reserved "$(off 1)" "$(off 20)" >/dev/null
|
||||
rollcall range add "$NET" static "$(off 50)" "$(off 99)" >/dev/null
|
||||
rollcall range add "$NET" dynamic "$(off 100)" "$(off 200)" >/dev/null
|
||||
ok "subnet $NET: reserved $(off 1)-$(off 20), static $(off 50)-$(off 99), dynamic $(off 100)-$(off 200)"
|
||||
fi
|
||||
else
|
||||
ok "subnet $NET exists, kept"
|
||||
fi
|
||||
if [ $DHCP = 1 ]; then
|
||||
rollcall vlan add "$FAB" 0 -dhcp -rack "$RACK" >/dev/null
|
||||
ok "DHCP/PXE on $IFACE"
|
||||
elif [ $DHCP = 2 ]; then
|
||||
rollcall vlan add "$FAB" 0 -proxy-dhcp -rack "$RACK" >/dev/null
|
||||
ok "proxy DHCP on $IFACE: PXE answers only; addresses from the site's DHCP server"
|
||||
fi
|
||||
if [ -n "$NAT" ]; then
|
||||
echo "net.ipv4.ip_forward=1" > /etc/sysctl.d/60-rollcall-forward.conf
|
||||
|
||||
+13
-2
@@ -15,7 +15,7 @@ set -eu
|
||||
cd "$(dirname "$0")"
|
||||
[ -r /etc/rollcall/rack.json ] || { echo "enroll first: ./rollcall rack join CODE -url http://ADDR (code from: rollcall controller join-code on the region)" >&2; exit 1; }
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
apt-get install -y -q dnsmasq ipmitool pxelinux syslinux-common syslinux-efi jq >/dev/null
|
||||
apt-get install -y -q dnsmasq ipmitool pxelinux syslinux-common syslinux-efi shim-signed grub-efi-amd64-signed jq openssl >/dev/null
|
||||
|
||||
getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall
|
||||
[ -x ./rollcall ] && install -m 0755 rollcall /usr/local/bin/rollcall
|
||||
@@ -38,6 +38,14 @@ for f in ldlinux.c32 menu.c32 libutil.c32 libcom32.c32 poweroff.c32; do cp /usr/
|
||||
cp /usr/lib/SYSLINUX.EFI/efi64/syslinux.efi "$TFTP/efi64/"
|
||||
for f in ldlinux.e64 menu.c32 libutil.c32 libcom32.c32; do cp /usr/lib/syslinux/modules/efi64/$f "$TFTP/efi64/"; done
|
||||
ln -sfn ../pxelinux.cfg "$TFTP/efi64/pxelinux.cfg"
|
||||
# UEFI, Secure Boot included: signed shim and GRUB; rollcall writes the GRUB menus next to pxelinux's
|
||||
if [ -f /usr/lib/shim/shimx64.efi.signed ] && [ -f /usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed ]; then
|
||||
install -m 0644 /usr/lib/shim/shimx64.efi.signed "$TFTP"/bootx64.efi
|
||||
install -m 0644 /usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed "$TFTP"/grubx64.efi
|
||||
install -d -o root -g rollcall -m 2775 "$TFTP"/grub
|
||||
else
|
||||
echo "WARNING: shim-signed or grub-efi-amd64-signed missing: UEFI machines cannot PXE-boot (apt install shim-signed grub-efi-amd64-signed)" >&2
|
||||
fi
|
||||
ln -sfn "$CACHE/rollcall" "$TFTP/rollcall"
|
||||
ln -sfn "$CACHE/images" "$TFTP/images"
|
||||
ln -sfn ../rollcall "$TFTP/efi64/rollcall"
|
||||
@@ -48,15 +56,18 @@ install -d -o rollcall -g rollcall "$DNS" "$DNS/conf.d"
|
||||
for f in hosts hosts.dhcp; do [ -e "$DNS/$f" ] || install -m 0644 -o rollcall -g rollcall /dev/null "$DNS/$f"; done
|
||||
# bind-dynamic binds address by address, so systemd-resolved's 127.0.0.53 stub does not get in the way;
|
||||
# until the region assigns a VLAN only loopback is served
|
||||
printf '# rollcall rack controller: the configuration is generated from the region'"'"'s network model\nbind-dynamic\ninterface=lo\nconf-dir=%s/conf.d/,*.conf\n' "$DNS" > /etc/dnsmasq.d/rollcall.conf
|
||||
printf '# rollcall rack controller: the configuration is generated from the region'"'"'s network model\nbind-dynamic\ninterface=lo\nconf-dir=/var/lib/rollcall-dnsmasq/,*.conf\n' > /etc/dnsmasq.d/rollcall.conf
|
||||
install -d -m 0755 /var/lib/rollcall-dnsmasq
|
||||
install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh
|
||||
install -m 0755 rollcall-rack-update.sh /usr/local/bin/rollcall-rack-update.sh
|
||||
install -d -m 0755 -o rollcall -g rollcall /var/lib/rollcall/update
|
||||
install -d -m 0700 -o rollcall -g rollcall /var/lib/rollcall/rack-cert # renewed certificates (the joined ones are root's)
|
||||
install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service rollcall-rack.service rollcall-rack-update.path rollcall-rack-update.service /etc/systemd/system/
|
||||
|
||||
systemctl daemon-reload
|
||||
systemctl enable dnsmasq.service rollcall-dnsmasq.path rollcall-rack-update.path >/dev/null 2>&1
|
||||
systemctl start rollcall-rack-update.path
|
||||
/usr/local/bin/rollcall-dnsmasq-reload.sh >/dev/null 2>&1 || true
|
||||
systemctl restart dnsmasq.service || { journalctl -u dnsmasq -n 5 --no-pager >&2; exit 1; }
|
||||
systemctl start rollcall-dnsmasq.path
|
||||
systemctl enable --now rollcall-rack.service
|
||||
|
||||
+14
-3
@@ -16,6 +16,14 @@ chown rollcall:rollcall /etc/rollcall
|
||||
# rollcall writes per-MAC pxelinux configs
|
||||
chgrp rollcall /srv/tftp/pxelinux.cfg && chmod 2775 /srv/tftp/pxelinux.cfg
|
||||
install -m 0644 /usr/lib/syslinux/modules/bios/poweroff.c32 /srv/tftp/poweroff.c32
|
||||
# UEFI, Secure Boot included: signed shim and GRUB; rollcall writes the GRUB menus next to pxelinux's
|
||||
if [ -f /usr/lib/shim/shimx64.efi.signed ] && [ -f /usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed ]; then
|
||||
install -m 0644 /usr/lib/shim/shimx64.efi.signed /srv/tftp/bootx64.efi
|
||||
install -m 0644 /usr/lib/grub/x86_64-efi-signed/grubnetx64.efi.signed /srv/tftp/grubx64.efi
|
||||
install -d -o root -g rollcall -m 2775 /srv/tftp/grub
|
||||
else
|
||||
echo "WARNING: shim-signed or grub-efi-amd64-signed missing: UEFI machines cannot PXE-boot (apt install shim-signed grub-efi-amd64-signed)" >&2
|
||||
fi
|
||||
sed -e "s/@AGENT_TOKEN@/$(jq -r .agent_token /etc/rollcall/config.json)/g" -e "s#@SERVER@#$(jq -r .server_url /etc/rollcall/config.json)#g" pxelinux-default > /srv/tftp/pxelinux.cfg/default && chmod 0644 /srv/tftp/pxelinux.cfg/default
|
||||
[ -e /srv/tftp/rollcall ] || ln -s /srv/rollcall/image /srv/tftp/rollcall
|
||||
[ -d /srv/tftp/efi64 ] && ln -sfn ../rollcall /srv/tftp/efi64/rollcall
|
||||
@@ -39,9 +47,12 @@ for f in hosts hosts.dhcp; do [ -e /var/lib/rollcall/dnsmasq/$f ] || install -m
|
||||
install -m 0755 rollcall-dnsmasq-reload.sh /usr/local/bin/rollcall-dnsmasq-reload.sh
|
||||
install -m 0644 rollcall-dnsmasq.path rollcall-dnsmasq.service /etc/systemd/system/
|
||||
# the embedded rack controller writes conf.d/rollcall.conf once a VLAN with dhcp is assigned to it;
|
||||
# a hand-written dnsmasq configuration for the same interface has to go then (see README, Network)
|
||||
if [ -d /etc/dnsmasq.d ] && [ ! -e /etc/dnsmasq.d/rollcall.conf ]; then
|
||||
printf '# rollcall: generated from the network model (rollcall subnet / vlan)\nconf-dir=/var/lib/rollcall/dnsmasq/conf.d/,*.conf\n' > /etc/dnsmasq.d/rollcall.conf
|
||||
# a hand-written dnsmasq configuration for the same interface has to go then (see README, Network).
|
||||
# dnsmasq (root) reads the checked copy rollcall-dnsmasq-reload.sh keeps in a directory of root's.
|
||||
install -d -m 0755 /var/lib/rollcall-dnsmasq
|
||||
if [ -d /etc/dnsmasq.d ]; then
|
||||
printf '# rollcall: generated from the network model (rollcall subnet / vlan), checked by rollcall-dnsmasq-reload.sh\nconf-dir=/var/lib/rollcall-dnsmasq/,*.conf\n' > /etc/dnsmasq.d/rollcall.conf
|
||||
/usr/local/bin/rollcall-dnsmasq-reload.sh >/dev/null 2>&1 || true
|
||||
fi
|
||||
legacy=$(grep -lE '^(dhcp-range|bind-interfaces|enable-tftp)' /etc/dnsmasq.conf /etc/dnsmasq.d/*.conf 2>/dev/null | grep -v '/rollcall.conf$' || true)
|
||||
if [ -n "$legacy" ]; then
|
||||
|
||||
@@ -8,15 +8,17 @@ TAG=${1:?usage: publish-image.sh TAG [API]}
|
||||
API=${2:-https://gitea.mynodes.xyz/api/v1/repos/ilker/rollcall}
|
||||
IMG=${IMG:-/srv/rollcall/image}
|
||||
: "${GITEA_TOKEN:?set GITEA_TOKEN (write:repository)}"
|
||||
HDR=$(mktemp); trap 'rm -f "$HDR"' EXIT
|
||||
printf 'Authorization: token %s\n' "$GITEA_TOKEN" > "$HDR" # not on curl's command line (ps)
|
||||
cd "$IMG"
|
||||
sha256sum vmlinuz initrd.img filesystem.squashfs > SHA256SUMS
|
||||
id=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" "$API/releases/tags/$TAG" | python3 -c 'import json,sys;print(json.load(sys.stdin)["id"])')
|
||||
existing=$(curl -fsS -H "Authorization: token $GITEA_TOKEN" "$API/releases/$id/assets" | python3 -c 'import json,sys;[print(a["id"],a["name"]) for a in json.load(sys.stdin)]')
|
||||
id=$(curl -fsS -H "@$HDR" "$API/releases/tags/$TAG" | python3 -c 'import json,sys;print(json.load(sys.stdin)["id"])')
|
||||
existing=$(curl -fsS -H "@$HDR" "$API/releases/$id/assets" | python3 -c 'import json,sys;[print(a["id"],a["name"]) for a in json.load(sys.stdin)]')
|
||||
for f in vmlinuz initrd.img filesystem.squashfs SHA256SUMS; do
|
||||
name=commissioning-$f
|
||||
old=$(echo "$existing" | awk -v n="$name" '$2==n{print $1}')
|
||||
[ -n "$old" ] && curl -fsS -X DELETE -H "Authorization: token $GITEA_TOKEN" "$API/releases/$id/assets/$old" >/dev/null
|
||||
[ -n "$old" ] && curl -fsS -X DELETE -H "@$HDR" "$API/releases/$id/assets/$old" >/dev/null
|
||||
echo "uploading $name ($(du -h "$f" | cut -f1))"
|
||||
curl -fsS -H "Authorization: token $GITEA_TOKEN" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" >/dev/null
|
||||
curl -fsS -H "@$HDR" -F "attachment=@$f" "$API/releases/$id/assets?name=$name" >/dev/null
|
||||
done
|
||||
echo "done: bootstrap.sh --image-url ${API%/api/v1/repos/*}/$(echo "$API" | sed 's#.*/repos/##')/releases/download/$TAG"
|
||||
@@ -7,29 +7,61 @@
|
||||
# some of those), the last configuration that worked is put back and dnsmasq
|
||||
# is started with it, so DHCP stays up. The rejected content is kept next to it
|
||||
# and not tried again until it changes.
|
||||
#
|
||||
# rollcall (an unprivileged user) writes $SRC; dnsmasq starts as root and reads
|
||||
# only the copy in $LIVE, a directory of root's. The copy is made without
|
||||
# following links and holds only the options rollcall generates: nothing that
|
||||
# runs a program, includes another file or changes dnsmasq's user.
|
||||
set -u
|
||||
D=/var/lib/rollcall/dnsmasq
|
||||
CONF=$D/conf.d/rollcall.conf
|
||||
GOOD=$D/rollcall.conf.accepted
|
||||
BAD=$D/rollcall.conf.rejected
|
||||
SRC=/var/lib/rollcall/dnsmasq/conf.d/rollcall.conf
|
||||
LIVE=/var/lib/rollcall-dnsmasq
|
||||
CONF=$LIVE/rollcall.conf
|
||||
GOOD=$LIVE/rollcall.conf.accepted
|
||||
BAD=$LIVE/rollcall.conf.rejected
|
||||
STAMP=/run/rollcall-dnsmasq.sum
|
||||
ALLOWED='addn-hosts bind-dynamic bogus-priv dhcp-authoritative dhcp-boot dhcp-hostsfile dhcp-match dhcp-option dhcp-range domain domain-needed enable-ra enable-tftp expand-hosts interface local log-dhcp no-resolv pxe-service ra-param server tftp-root'
|
||||
|
||||
install -d -m 0755 -o root -g root "$LIVE"
|
||||
umask 022
|
||||
sum() { cat "$1" 2>/dev/null | sha256sum | cut -d' ' -f1; }
|
||||
put() { cp "$1" "$CONF.tmp" && mv -f "$CONF.tmp" "$CONF"; } # within $LIVE: root's
|
||||
restore() { # put the last working configuration back (or none)
|
||||
if [ -s "$GOOD" ]; then cp "$GOOD" "$CONF.tmp" && mv -f "$CONF.tmp" "$CONF"; else rm -f "$CONF"; fi
|
||||
if [ -s "$GOOD" ]; then put "$GOOD"; else rm -f "$CONF"; fi
|
||||
sum "$CONF" > "$STAMP"
|
||||
}
|
||||
|
||||
new=$(sum "$CONF")
|
||||
if [ "$new" = "$(cat "$STAMP" 2>/dev/null)" ]; then
|
||||
# take rollcall's file: a regular file, not a link, at most 1 MB
|
||||
cand=$LIVE/candidate.tmp
|
||||
rm -f "$cand"
|
||||
if [ -e "$SRC" ]; then
|
||||
if ! dd if="$SRC" of="$cand" iflag=nofollow bs=1M count=1 status=none 2>/dev/null; then
|
||||
echo "$SRC is not a regular file; ignored" >&2
|
||||
exit 1
|
||||
fi
|
||||
printf '%s\n' $ALLOWED > "$LIVE/allowed.tmp"
|
||||
bad=$(grep -vE '^[[:space:]]*(#|$)' "$cand" | sed -E 's/^[[:space:]]*([^=[:space:]]*).*/\1/' | sort -u |
|
||||
grep -vxFf "$LIVE/allowed.tmp" | tr '\n' ' ')
|
||||
rm -f "$LIVE/allowed.tmp"
|
||||
if [ -n "$bad" ]; then
|
||||
rm -f "$cand"
|
||||
echo "generated dnsmasq configuration has options rollcall does not write ($bad); not used" >&2
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
|
||||
new=$(sum "$cand")
|
||||
if [ "$new" = "$(cat "$STAMP" 2>/dev/null)" ] && { [ -e "$CONF" ] || [ ! -e "$cand" ]; }; then
|
||||
rm -f "$cand"
|
||||
systemctl kill -s HUP dnsmasq.service
|
||||
exit 0
|
||||
fi
|
||||
if [ -e "$CONF" ] && [ "$new" = "$(sum "$BAD")" ]; then
|
||||
if [ -e "$cand" ] && [ "$new" = "$(sum "$BAD")" ]; then
|
||||
rm -f "$cand"
|
||||
restore
|
||||
echo "generated dnsmasq configuration was rejected before and has not changed; keeping the last working one" >&2
|
||||
exit 1
|
||||
fi
|
||||
if [ -e "$cand" ]; then mv -f "$cand" "$CONF"; else rm -f "$CONF"; fi
|
||||
if out=$(dnsmasq --test 2>&1) && systemctl restart dnsmasq.service; then
|
||||
echo "$new" > "$STAMP"
|
||||
if [ -e "$CONF" ]; then cp "$CONF" "$GOOD"; else rm -f "$GOOD"; fi
|
||||
|
||||
@@ -1,27 +1,41 @@
|
||||
#!/bin/sh
|
||||
# Installs a rollcall binary the rack controller staged (rollcall-rack-update.path).
|
||||
# The rack checked the region's checksum and the binary's version; this checks
|
||||
# them again, keeps the running binary as .prev, restarts the rack and puts the
|
||||
# previous binary back if the rack does not register with the region in 60 s.
|
||||
#
|
||||
# The staging directory belongs to the rack's unprivileged user, so nothing in
|
||||
# it is trusted: the binary is copied into a directory of root's and installed
|
||||
# only if the region's signature over it verifies against the CA certificate
|
||||
# written at join (root's, from rack.json ca_file). Then the running binary is
|
||||
# kept as .prev, the rack restarted, and the previous binary put back if the
|
||||
# rack does not register with the region in 60 s.
|
||||
set -u
|
||||
D=/var/lib/rollcall/update
|
||||
BIN=/usr/local/bin/rollcall
|
||||
[ -f "$D/ready" ] || exit 0
|
||||
want=$(cat "$D/ready"); rm -f "$D/ready"
|
||||
cd "$D" || exit 1
|
||||
result() { echo "$(date -Is) $*" > "$D/last-result"; echo "$*"; }
|
||||
sha256sum -c rollcall.new.sha256 >/dev/null 2>&1 || { result "$want: checksum mismatch, not installed"; exit 1; }
|
||||
got=$(./rollcall.new version 2>/dev/null | awk '{print $2}')
|
||||
want=$(head -c 64 "$D/ready" | tr -cd 'A-Za-z0-9._+-'); rm -f "$D/ready"
|
||||
# root's work directory, on a file system that allows executing (/run is noexec)
|
||||
install -d -m 0700 -o root -g root /var/lib/rollcall-update
|
||||
W=$(mktemp -d /var/lib/rollcall-update/w.XXXXXX) || exit 1
|
||||
trap 'rm -rf "$W"' EXIT
|
||||
# last-result is read by the rack (backoff); renamed in, so a link planted there is replaced, not followed
|
||||
result() { echo "$(date -Is) $*" > "$W/result" && mv -fT "$W/result" "$D/last-result"; echo "$*"; }
|
||||
CA=$(jq -r '.ca_file // "/etc/rollcall/rack/ca.pem"' /etc/rollcall/rack.json 2>/dev/null)
|
||||
[ -n "$CA" ] && [ "$(stat -c %u "$CA" 2>/dev/null)" = 0 ] || { result "$want: no root-owned CA certificate ($CA), not installed"; exit 1; }
|
||||
cp -- "$D/rollcall.new" "$W/bin" && cp -- "$D/rollcall.new.sig" "$W/sig" || { result "$want: staged files missing, not installed"; exit 1; }
|
||||
openssl x509 -in "$CA" -pubkey -noout > "$W/pub" 2>/dev/null &&
|
||||
openssl dgst -sha256 -verify "$W/pub" -signature "$W/sig" "$W/bin" >/dev/null 2>&1 ||
|
||||
{ result "$want: signature does not verify against $CA, not installed"; exit 1; }
|
||||
chmod 0755 "$W/bin"
|
||||
got=$("$W/bin" version 2>/dev/null | awk '{print $2}')
|
||||
[ "$got" = "$want" ] || { result "$want: binary reports '$got', not installed"; exit 1; }
|
||||
cur=$("$BIN" version 2>/dev/null | awk '{print $2}')
|
||||
cp -p "$BIN" "$BIN.prev"
|
||||
install -m 0755 rollcall.new "$BIN.tmp" && mv -f "$BIN.tmp" "$BIN"
|
||||
install -m 0755 "$W/bin" "$BIN.tmp" && mv -f "$BIN.tmp" "$BIN"
|
||||
rm -f "$D/rollcall.new" "$D/rollcall.new.sha256" "$D/rollcall.new.sig"
|
||||
since=$(date +%s)
|
||||
systemctl restart rollcall-rack.service
|
||||
for _ in $(seq 1 30); do
|
||||
sleep 2
|
||||
if journalctl -u rollcall-rack.service --since "@$since" -o cat 2>/dev/null | grep -q "^.*registered with"; then
|
||||
rm -f rollcall.new rollcall.new.sha256
|
||||
result "updated $cur -> $want"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
@@ -15,7 +15,7 @@ NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/srv/tftp/pxelinux.cfg /var/lib/rollcall /var/cache/rollcall
|
||||
ReadWritePaths=/srv/tftp/pxelinux.cfg -/srv/tftp/grub /var/lib/rollcall /var/cache/rollcall
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -16,7 +16,7 @@ NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
ProtectHome=yes
|
||||
PrivateTmp=yes
|
||||
ReadWritePaths=/etc/rollcall /srv/tftp/pxelinux.cfg /var/lib/rollcall
|
||||
ReadWritePaths=/etc/rollcall /srv/tftp/pxelinux.cfg -/srv/tftp/grub /var/lib/rollcall
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
@@ -0,0 +1,16 @@
|
||||
--- a/src/rfc2131.c
|
||||
+++ b/src/rfc2131.c
|
||||
@@ -977,7 +977,12 @@
|
||||
int redirect4011 = 0;
|
||||
struct dhcp_opt *option;
|
||||
|
||||
- /* OK only dhcp-option-pxe options. */
|
||||
+ /* OK only dhcp-option-pxe options.
|
||||
+ option_filter() appends netid to the end of the context's tag list.
|
||||
+ tmp->netid is static and still links to the previous request's
|
||||
+ (stack) tags here; start it afresh, as the non-proxy path does,
|
||||
+ or the list can become circular and the tag walks never end. */
|
||||
+ tmp->netid.next = NULL;
|
||||
tagif_netid = option_filter(netid, tmp->netid.net ? &tmp->netid : NULL, daemon->dhcp_opts, 2);
|
||||
boot = find_boot(tagif_netid);
|
||||
|
||||
@@ -0,0 +1,24 @@
|
||||
# dnsmasq: busy loop in proxy DHCP with a tagged range
|
||||
|
||||
Found 2026-10-29 while testing Rollcall's proxy DHCP mode (gdb on a stuck lab
|
||||
rack controller). Affects dnsmasq 2.91 (Debian 13) and 2.93 (latest release).
|
||||
Rollcall works around it by leaving the tag off proxy ranges
|
||||
(`cmd/rollcall/network.go`, `dnsmasqConfig`).
|
||||
|
||||
| File | |
|
||||
|---|---|
|
||||
| `report.txt` | the report to send upstream (plain text, patch inline) |
|
||||
| `repro.sh` | reproducer: `sudo ./repro.sh /path/to/dnsmasq` (two netns + veth, two PXE DISCOVERs) |
|
||||
| `0001-proxy-dhcp-reset-context-tags.patch` | one-line fix against 2.93, applies with `patch -p1` or `git apply` |
|
||||
|
||||
Results on a Debian 13 VM:
|
||||
|
||||
| dnsmasq | Configuration | CPU after two PXE requests |
|
||||
|---|---|---|
|
||||
| 2.91 (Debian) | tagged proxy range, `log-dhcp` | 100% |
|
||||
| 2.93 | same | 100% |
|
||||
| 2.93 + patch | same | 0% |
|
||||
| 2.93 | no `log-dhcp`, `dhcp-option` for an absent tag | 69% |
|
||||
| 2.93 + patch | same | 0% |
|
||||
|
||||
Status: not reported yet.
|
||||
@@ -0,0 +1,97 @@
|
||||
Subject: [PATCH] proxy DHCP: busy loop when a proxy dhcp-range has a tag
|
||||
|
||||
Hello Simon,
|
||||
|
||||
dnsmasq can be put into a busy loop by two PXE DHCPDISCOVERs from any client
|
||||
on a network with a tagged proxy range. dnsmasq then stops answering DHCP,
|
||||
TFTP and DNS until it is restarted. Reproduced on 2.91 (Debian 13,
|
||||
2.91-1+deb13u2) and on 2.93 built from the release tarball; a one-line patch
|
||||
is below.
|
||||
|
||||
Configuration (the relevant part):
|
||||
|
||||
dhcp-range=set:prov,192.0.2.0,proxy,255.255.255.0
|
||||
pxe-service=tag:prov,x86PC,"boot",pxelinux
|
||||
log-dhcp
|
||||
|
||||
After the second PXE request dnsmasq sits at 100% CPU, makes no system
|
||||
calls, and its sockets fill up. gdb on the stuck process:
|
||||
|
||||
#0 __strcmp_sse42 ()
|
||||
#1 log_tags (netid=..., xid=...) at dhcp-common.c:276
|
||||
#2 dhcp_reply (...) at rfc2131.c:1028 (proxy branch)
|
||||
#3 dhcp_packet (...) at dhcp.c:340
|
||||
#4 main (...) at dnsmasq.c:1299
|
||||
|
||||
The tag list that log_tags() walks is circular. Walking it in gdb on two
|
||||
occasions:
|
||||
|
||||
prov -> eth1 -> eth1 -> eth1 -> ... (iface tag points at itself)
|
||||
prov -> known -> bios -> eth1 -> known -> ... (with dhcp-host and dhcp-match)
|
||||
|
||||
Cause, as far as I can see: in the proxy branch of dhcp_reply(),
|
||||
|
||||
tagif_netid = option_filter(netid, tmp->netid.net ? &tmp->netid : NULL, ...);
|
||||
|
||||
option_filter() walks the context tags to their end and appends the request's
|
||||
tags:
|
||||
|
||||
for (last_tag = context_tags; last_tag->next; last_tag = last_tag->next);
|
||||
last_tag->next = tags;
|
||||
|
||||
tmp->netid is part of the (static) dhcp_context, so after the first request
|
||||
tmp->netid.next still points at the previous request's iface_id, a stack
|
||||
variable of dhcp_reply(). The next request has its iface_id at the same
|
||||
address with next == NULL, so the walk stops on it and sets
|
||||
iface_id.next = &iface_id. The non-proxy path does not have this problem
|
||||
because it assigns context->netid.next before use.
|
||||
|
||||
It is not only log_tags(): without log-dhcp, any walk that looks for a tag
|
||||
that is not in the list never ends. With
|
||||
|
||||
dhcp-option=tag:nosuch,option:ntp-server,192.0.2.9
|
||||
|
||||
added (and log-dhcp removed) 2.93 loops the same way.
|
||||
|
||||
The attached repro.sh shows it in a few seconds (Linux, root): it creates two
|
||||
network namespaces joined by a veth pair, runs the given dnsmasq binary with
|
||||
the configuration above in one and sends two PXE DISCOVERs from the other:
|
||||
|
||||
stock 2.93: dnsmasq CPU over 5 s: 100% RESULT: BUSY LOOP
|
||||
2.93 + patch: dnsmasq CPU over 5 s: 0% RESULT: ok
|
||||
stock 2.93, no log-dhcp, dhcp-option for an absent tag: 69% BUSY LOOP
|
||||
2.93 + patch, same: 0% ok
|
||||
|
||||
We found it on a PXE provisioning system whose rack controllers run dnsmasq
|
||||
in proxy mode next to a site's own DHCP server; the clients were ordinary
|
||||
iPXE ROMs. Our generated configurations now leave the tag off proxy ranges.
|
||||
|
||||
The patch resets the context tag before option_filter() appends to it, which
|
||||
is what the non-proxy path effectively does:
|
||||
|
||||
--- a/src/rfc2131.c
|
||||
+++ b/src/rfc2131.c
|
||||
@@ -977,7 +977,12 @@
|
||||
int redirect4011 = 0;
|
||||
struct dhcp_opt *option;
|
||||
|
||||
- /* OK only dhcp-option-pxe options. */
|
||||
+ /* OK only dhcp-option-pxe options.
|
||||
+ option_filter() appends netid to the end of the context's tag list.
|
||||
+ tmp->netid is static and still links to the previous request's
|
||||
+ (stack) tags here; start it afresh, as the non-proxy path does,
|
||||
+ or the list can become circular and the tag walks never end. */
|
||||
+ tmp->netid.next = NULL;
|
||||
tagif_netid = option_filter(netid, tmp->netid.net ? &tmp->netid : NULL, daemon->dhcp_opts, 2);
|
||||
boot = find_boot(tagif_netid);
|
||||
|
||||
|
||||
option_filter() might be the better place for a general fix (the DHCPv6 path
|
||||
in rfc3315.c passes state->context_tags the same way), but I have only
|
||||
tested the proxy case.
|
||||
|
||||
Since a single client can stop dnsmasq on the whole segment, you may want to
|
||||
treat this as a denial-of-service issue; I have not posted it anywhere else.
|
||||
|
||||
Regards,
|
||||
<name>
|
||||
+57
@@ -0,0 +1,57 @@
|
||||
#!/bin/sh
|
||||
# dnsmasq proxy-DHCP busy loop reproducer. Run as root on Linux.
|
||||
# ./repro.sh [/path/to/dnsmasq]
|
||||
# Creates two network namespaces joined by a veth pair, runs dnsmasq with one
|
||||
# tagged proxy range in one, sends two PXE DHCPDISCOVERs from the other, then
|
||||
# reports dnsmasq's CPU use. A healthy dnsmasq stays near 0%.
|
||||
set -eu
|
||||
DNSMASQ=${1:-dnsmasq}
|
||||
D=$(mktemp -d)
|
||||
cleanup() { [ -f "$D/pid" ] && kill "$(cat "$D/pid")" 2>/dev/null || true; ip netns del dq-srv 2>/dev/null || true; ip netns del dq-cli 2>/dev/null || true; rm -rf "$D"; }
|
||||
trap cleanup EXIT
|
||||
cleanup 2>/dev/null || true; D=$(mktemp -d)
|
||||
ip netns add dq-srv; ip netns add dq-cli
|
||||
ip link add dq0 type veth peer name dq1
|
||||
ip link set dq0 netns dq-srv; ip link set dq1 netns dq-cli
|
||||
ip -n dq-srv addr add 192.0.2.1/24 dev dq0; ip -n dq-srv link set dq0 up; ip -n dq-srv link set lo up
|
||||
ip -n dq-cli link set dq1 up; ip -n dq-cli link set lo up
|
||||
mkdir -p "$D/tftp"
|
||||
cat > "$D/dnsmasq.conf" <<EOF
|
||||
port=0
|
||||
interface=dq0
|
||||
bind-interfaces
|
||||
log-dhcp
|
||||
enable-tftp
|
||||
tftp-root=$D/tftp
|
||||
dhcp-range=set:prov,192.0.2.0,proxy,255.255.255.0
|
||||
pxe-service=tag:prov,x86PC,"boot",pxelinux
|
||||
EOF
|
||||
ip netns exec dq-srv "$DNSMASQ" --conf-file="$D/dnsmasq.conf" --pid-file="$D/pid" --log-facility="$D/log" -u root
|
||||
sleep 1
|
||||
"$DNSMASQ" --version | head -1
|
||||
# two PXE DISCOVERs (vendor class PXEClient, client arch 0 = x86 BIOS), like a PXE/iPXE ROM sends
|
||||
ip netns exec dq-cli python3 - <<'P'
|
||||
import os, socket, struct, time
|
||||
s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
s.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
||||
s.setsockopt(socket.SOL_SOCKET, 25, b"dq1") # SO_BINDTODEVICE
|
||||
s.bind(("0.0.0.0", 68))
|
||||
mac = bytes.fromhex("020000000001")
|
||||
for n in range(2):
|
||||
xid = os.urandom(4)
|
||||
p = bytearray(240)
|
||||
p[0:4] = bytes([1, 1, 6, 0]); p[4:8] = xid
|
||||
struct.pack_into(">H", p, 10, 0x8000)
|
||||
p[28:34] = mac; p[236:240] = bytes([99, 130, 83, 99])
|
||||
vc = b"PXEClient:Arch:00000:UNDI:002001"
|
||||
p += bytes([53, 1, 1, 60, len(vc)]) + vc + bytes([93, 2, 0, 0, 94, 3, 1, 2, 1, 97, 17, 0]) + os.urandom(16) + bytes([255])
|
||||
s.sendto(bytes(p), ("255.255.255.255", 67))
|
||||
time.sleep(1)
|
||||
P
|
||||
sleep 2
|
||||
P=$(cat "$D/pid")
|
||||
a=$(awk '{print $14+$15}' /proc/$P/stat); sleep 5; b=$(awk '{print $14+$15}' /proc/$P/stat)
|
||||
cpu=$(( (b - a) * 100 / (5 * $(getconf CLK_TCK)) ))
|
||||
echo "dnsmasq CPU over 5 s: ${cpu}%"
|
||||
grep -c "proxy" "$D/log" | sed 's/^/proxy replies logged: /'
|
||||
[ "$cpu" -lt 50 ] && echo "RESULT: ok" || echo "RESULT: BUSY LOOP"
|
||||
@@ -3,13 +3,18 @@
|
||||
# Fetches this node's overlay tarball from the rollcall server (identified by
|
||||
# the rollcall.node token on the kernel command line), applies it over / when
|
||||
# it changed, then runs /run/rollcall/post-<stage>.sh if the overlay shipped one.
|
||||
# With rollcall.tls= and rollcall.ca= on the command line it talks HTTPS: the
|
||||
# CA certificate is accepted only by that SHA-256, and there is no fallback
|
||||
# to HTTP (the overlay carries secrets such as munge.key).
|
||||
set -eu
|
||||
stage=${1:-system}
|
||||
SERVER=""; TOKEN=""
|
||||
SERVER=""; TOKEN=""; TLS=""; CA=""
|
||||
for t in $(cat /proc/cmdline); do
|
||||
case $t in
|
||||
rollcall.server=*) SERVER=${t#*=} ;;
|
||||
rollcall.node=*) TOKEN=${t#*=} ;;
|
||||
rollcall.tls=*) TLS=${t#*=} ;;
|
||||
rollcall.ca=*) CA=$(echo "${t#*=}" | tr -d ':' | tr 'A-F' 'a-f') ;;
|
||||
esac
|
||||
done
|
||||
if [ -z "$SERVER" ] || [ -z "$TOKEN" ]; then
|
||||
@@ -17,9 +22,30 @@ if [ -z "$SERVER" ] || [ -z "$TOKEN" ]; then
|
||||
exit 0
|
||||
fi
|
||||
mkdir -p /run/rollcall
|
||||
CURL_CA=""
|
||||
if [ -n "$TLS" ] && [ -n "$CA" ]; then
|
||||
cafile=/run/rollcall/ca.pem
|
||||
if [ ! -s "$cafile" ]; then
|
||||
i=0
|
||||
until curl -fsS --connect-timeout 5 "$SERVER/ca.pem" -o "$cafile.new"; do
|
||||
i=$((i+1)); [ "$i" -ge 30 ] && { echo "rollcall-overlay: cannot fetch the CA certificate" >&2; exit 1; }
|
||||
sleep 5
|
||||
done
|
||||
got=$(openssl x509 -in "$cafile.new" -outform DER | sha256sum | cut -d' ' -f1)
|
||||
if [ "$got" != "$CA" ]; then
|
||||
rm -f "$cafile.new"
|
||||
echo "rollcall-overlay: the CA certificate does not match rollcall.ca; not fetching the overlay" >&2
|
||||
exit 1
|
||||
fi
|
||||
mv -f "$cafile.new" "$cafile"
|
||||
fi
|
||||
SERVER=$TLS
|
||||
CURL_CA="--cacert $cafile"
|
||||
fi
|
||||
tmp=/run/rollcall/$stage.tar.new
|
||||
i=0
|
||||
until curl -fsS --connect-timeout 5 -H "X-Rollcall-Node: $TOKEN" \
|
||||
# shellcheck disable=SC2086 # CURL_CA is two words or none
|
||||
until curl -fsS --connect-timeout 5 $CURL_CA -H "X-Rollcall-Node: $TOKEN" \
|
||||
"$SERVER/api/v1/nodes/self/overlay?stage=$stage" -o "$tmp"; do
|
||||
i=$((i+1))
|
||||
if [ "$i" -ge 60 ]; then echo "rollcall-overlay: server unreachable" >&2; exit 1; fi
|
||||
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/bin/bash
|
||||
# Lab NetBox for Rollcall's NetBox drivers (IPAM, device sync): a Debian 13
|
||||
# container on the lab segment, so tests never write to a real NetBox.
|
||||
# Same install as netops (altyapi hosts/netops/kurulum.sh) without PowerDNS,
|
||||
# ACME and backups; plain HTTP on :80 (the segment is isolated).
|
||||
#
|
||||
# Run as root inside the container; rerunnable. Secrets are generated here:
|
||||
# /etc/lab-netbox/secrets.env (admin password, Rollcall's API token)
|
||||
#
|
||||
# On pve1 (example):
|
||||
# pct create 225 local:vztmpl/debian-13-standard_13.6-1_amd64.tar.zst --hostname lab-netbox \
|
||||
# --cores 2 --memory 2048 --swap 512 --rootfs zfspool:8 --unprivileged 1 \
|
||||
# --net0 name=eth0,bridge=pxenet,ip=10.213.77.6/24,gw=10.213.77.1 --nameserver 10.213.77.1
|
||||
# pct start 225 && pct push 225 install.sh /root/install.sh && pct exec 225 -- bash /root/install.sh
|
||||
set -euo pipefail
|
||||
NETBOX_VERSION=${NETBOX_VERSION:-4.7.4}
|
||||
MYIP=${MYIP:-10.213.77.6}
|
||||
SITE=${SITE:-lab}
|
||||
PREFIXES=${PREFIXES:-"10.213.77.0/24 10.213.78.0/24 10.213.79.0/24 fd00:213:77::/64 fd00:213:78::/64"}
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
|
||||
apt-get update -q
|
||||
apt-get install -yq postgresql valkey-server caddy \
|
||||
python3 python3-venv python3-dev build-essential libxml2-dev libxslt1-dev libffi-dev \
|
||||
libpq-dev libssl-dev zlib1g-dev git curl jq
|
||||
|
||||
install -d -m 700 /etc/lab-netbox
|
||||
S=/etc/lab-netbox/secrets.env
|
||||
if [ ! -f $S ]; then
|
||||
r() { python3 -c "import secrets,sys;print(secrets.token_urlsafe(int(sys.argv[1])))" "$1"; }
|
||||
cat > $S <<EOF
|
||||
NETBOX_DB_PASS=$(r 24)
|
||||
NETBOX_SECRET_KEY=$(r 50)
|
||||
NETBOX_PEPPER=$(r 50)
|
||||
NETBOX_ADMIN_PASS=$(r 18)
|
||||
ROLLCALL_PASS=$(r 18)
|
||||
EOF
|
||||
chmod 600 $S
|
||||
fi
|
||||
. $S
|
||||
|
||||
# ------------------------------------------------------------------ PostgreSQL, Valkey
|
||||
sudo() { runuser -u "$2" -- "${@:3}"; } # "sudo -u postgres ..." without sudo
|
||||
if [ -z "$(sudo -u postgres psql -tAqc "SELECT 1 FROM pg_roles WHERE rolname='netbox'")" ]; then
|
||||
sudo -u postgres psql -qc "CREATE ROLE netbox LOGIN PASSWORD '$NETBOX_DB_PASS'"
|
||||
# UTF-8 whatever the container's locale (C in the Proxmox template)
|
||||
sudo -u postgres psql -qc "CREATE DATABASE netbox OWNER netbox ENCODING 'UTF8' LC_COLLATE 'C.UTF-8' LC_CTYPE 'C.UTF-8' TEMPLATE template0"
|
||||
fi
|
||||
sed -i 's/^# *databases .*/databases 16/' /etc/valkey/valkey.conf || true
|
||||
systemctl enable --now valkey-server
|
||||
|
||||
# ------------------------------------------------------------------ NetBox
|
||||
id netbox >/dev/null 2>&1 || adduser --system --group --home /opt/netbox netbox
|
||||
D=/opt/netbox-$NETBOX_VERSION
|
||||
if [ ! -d $D ]; then
|
||||
curl -fsSL https://github.com/netbox-community/netbox/archive/refs/tags/v$NETBOX_VERSION.tar.gz | tar -xz -C /opt
|
||||
fi
|
||||
[ -L /opt/netbox ] || { [ -d /opt/netbox ] && rmdir /opt/netbox 2>/dev/null || true; }
|
||||
ln -sfn $D /opt/netbox
|
||||
cat > /opt/netbox/netbox/netbox/configuration.py <<EOF
|
||||
# pxestuff lab/netbox/install.sh generates this file
|
||||
ALLOWED_HOSTS = ['$MYIP', 'localhost', '127.0.0.1']
|
||||
DATABASES = {'default': {'NAME': 'netbox', 'USER': 'netbox', 'PASSWORD': '$NETBOX_DB_PASS',
|
||||
'HOST': '127.0.0.1', 'PORT': '', 'CONN_MAX_AGE': 300}}
|
||||
REDIS = {
|
||||
'tasks': {'HOST': '127.0.0.1', 'PORT': 6379, 'USERNAME': '', 'PASSWORD': '', 'DATABASE': 0, 'SSL': False},
|
||||
'caching': {'HOST': '127.0.0.1', 'PORT': 6379, 'USERNAME': '', 'PASSWORD': '', 'DATABASE': 1, 'SSL': False},
|
||||
}
|
||||
SECRET_KEY = '$NETBOX_SECRET_KEY'
|
||||
API_TOKEN_PEPPERS = {1: '$NETBOX_PEPPER'}
|
||||
TIME_ZONE = 'Europe/Stockholm'
|
||||
CSRF_TRUSTED_ORIGINS = ['http://$MYIP']
|
||||
EOF
|
||||
chown -R netbox:netbox $D/netbox/media $D/netbox/reports $D/netbox/scripts 2>/dev/null || true
|
||||
cd /opt/netbox && PYTHON=/usr/bin/python3 ./upgrade.sh >/var/log/netbox-upgrade.log 2>&1 || { tail -40 /var/log/netbox-upgrade.log; exit 1; }
|
||||
chown -R netbox:netbox /opt/netbox/netbox/media
|
||||
cp /opt/netbox/contrib/gunicorn.py /opt/netbox/gunicorn.py
|
||||
sed -i "s/^bind = .*/bind = '127.0.0.1:8001'/" /opt/netbox/gunicorn.py
|
||||
cp /opt/netbox/contrib/netbox.service /opt/netbox/contrib/netbox-rq.service /etc/systemd/system/
|
||||
systemctl daemon-reload
|
||||
systemctl enable --now netbox netbox-rq
|
||||
systemctl restart netbox netbox-rq
|
||||
|
||||
cat > /etc/caddy/Caddyfile <<EOF
|
||||
# pxestuff lab/netbox/install.sh
|
||||
http://$MYIP, http://localhost {
|
||||
handle_path /static/* {
|
||||
root * /opt/netbox/netbox/static
|
||||
file_server
|
||||
}
|
||||
reverse_proxy 127.0.0.1:8001
|
||||
}
|
||||
EOF
|
||||
systemctl enable caddy && systemctl restart caddy
|
||||
|
||||
# ------------------------------------------------------------------ what Rollcall needs
|
||||
# admin and rollcall users, the ownership tag and field, Rollcall's permissions
|
||||
# (as altyapi hosts/netops/seed/rollcall-izinler.py on netops), the lab site
|
||||
# and its prefixes delegated to Rollcall (tagged).
|
||||
NB_ADMIN_PASS=$NETBOX_ADMIN_PASS NB_ROLLCALL_PASS=$ROLLCALL_PASS NB_SITE=$SITE NB_PREFIXES=$PREFIXES \
|
||||
/opt/netbox/venv/bin/python /opt/netbox/netbox/manage.py shell -c '
|
||||
import os
|
||||
from django.contrib.contenttypes.models import ContentType
|
||||
from users.models import User, ObjectPermission
|
||||
from extras.models import Tag, CustomField
|
||||
from dcim.models import Site
|
||||
from ipam.models import Prefix
|
||||
|
||||
def user(name, pw, su):
|
||||
u, _ = User.objects.get_or_create(username=name)
|
||||
u.set_password(pw); u.is_superuser = su; u.is_active = True; u.save()
|
||||
return u
|
||||
|
||||
user("admin", os.environ["NB_ADMIN_PASS"], True)
|
||||
rc = user("rollcall", os.environ["NB_ROLLCALL_PASS"], False)
|
||||
tag, _ = Tag.objects.get_or_create(slug="rollcall", defaults={"name": "rollcall", "color": "2196f3"})
|
||||
cf, _ = CustomField.objects.get_or_create(name="rollcall_machine", defaults={"type": "text", "label": "Rollcall machine"})
|
||||
for m in ("ipam.ipaddress", "dcim.device"):
|
||||
cf.object_types.add(ContentType.objects.get_by_natural_key(*m.split(".")))
|
||||
perms = {
|
||||
"rollcall-ipam-write": (["ipam.ipaddress"], ["view", "add", "change", "delete"]),
|
||||
"rollcall-read": (["ipam.prefix", "ipam.vlan", "ipam.iprange", "extras.tag", "dcim.site"], ["view"]),
|
||||
"rollcall-dcim-write": (["dcim.device", "dcim.interface", "dcim.macaddress"], ["view", "add", "change", "delete"]),
|
||||
"rollcall-dcim-types": (["dcim.manufacturer", "dcim.devicetype", "dcim.devicerole"], ["view", "add"]),
|
||||
"rollcall-dcim-read": (["dcim.location", "dcim.rack"], ["view"]),
|
||||
}
|
||||
for name, (types, actions) in perms.items():
|
||||
p, _ = ObjectPermission.objects.get_or_create(name=name, defaults={"actions": actions})
|
||||
p.actions = actions; p.enabled = True; p.save()
|
||||
p.object_types.set([ContentType.objects.get_by_natural_key(*t.split(".")) for t in types])
|
||||
p.users.add(rc)
|
||||
site, _ = Site.objects.get_or_create(slug=os.environ["NB_SITE"], defaults={"name": os.environ["NB_SITE"].capitalize()})
|
||||
for cidr in os.environ["NB_PREFIXES"].split():
|
||||
p, _ = Prefix.objects.get_or_create(prefix=cidr, defaults={"description": "Rollcall lab", "scope": site})
|
||||
p.tags.add(tag)
|
||||
print("rollcall objects ready")
|
||||
'
|
||||
|
||||
# Rollcall's API token (v2), made once through the API with its password
|
||||
if ! grep -q '^NETBOX_TOKEN_ROLLCALL=' $S; then
|
||||
curl -fsS http://127.0.0.1:8001/api/users/tokens/provision/ -H "Host: $MYIP" \
|
||||
-H 'Content-Type: application/json' -H 'Accept: application/json' \
|
||||
-d "{\"username\":\"rollcall\",\"password\":\"$ROLLCALL_PASS\",\"description\":\"rollcall region\",\"write_enabled\":true}" |
|
||||
python3 -c 'import json,sys;d=json.load(sys.stdin);print("NETBOX_TOKEN_ROLLCALL=nbt_%s.%s"%(d["key"],d["token"]))' >> $S
|
||||
fi
|
||||
|
||||
echo "--- ready"
|
||||
systemctl is-active postgresql valkey-server netbox netbox-rq caddy | paste -sd' '
|
||||
. $S
|
||||
curl -fsS -o /dev/null -w "rollcall token: %{http_code}\n" -H "Authorization: Bearer $NETBOX_TOKEN_ROLLCALL" http://$MYIP/api/ipam/prefixes/
|
||||
@@ -11,6 +11,7 @@ Restart=always
|
||||
RestartSec=3
|
||||
AmbientCapabilities=CAP_NET_BIND_SERVICE
|
||||
User=pve-vbmc
|
||||
StateDirectory=pve-vbmc
|
||||
Group=pve-vbmc
|
||||
NoNewPrivileges=yes
|
||||
ProtectSystem=strict
|
||||
|
||||
+137
-10
@@ -11,9 +11,19 @@ idrac / ilo / generic drivers see what real hardware shows:
|
||||
|
||||
Config: the pve-vbmc config.json; a bmcs entry may carry "redfish": "idrac"|"ilo"|"generic".
|
||||
Basic auth with the same user/password as the IPMI side. Self-signed TLS.
|
||||
|
||||
Firmware (UpdateService), for testing rollcall's firmware jobs: idrac takes
|
||||
multipart uploads and SimpleUpdate, ilo only SimpleUpdate (it downloads the
|
||||
ImageURI itself), generic multipart. A "package" is any file with a line
|
||||
"version=X"; a name with bios/rom/flash in it is the BIOS, anything else the
|
||||
BMC. The BMC version changes when its task completes; the BIOS one at the next
|
||||
reset of the VM, as on real servers. Versions are kept in
|
||||
/var/lib/pve-vbmc/fw-<vmid>.json.
|
||||
"""
|
||||
import argparse
|
||||
import base64
|
||||
import email
|
||||
import email.policy
|
||||
import ipaddress
|
||||
import json
|
||||
import logging
|
||||
@@ -35,11 +45,16 @@ log = logging.getLogger("pve-redfish")
|
||||
PROFILES = {
|
||||
"idrac": {"vendor": "Dell", "manufacturer": "Dell Inc.", "model": "PowerEdge R760 (pve)", "id": "System.Embedded.1",
|
||||
"resets": ["On", "ForceOff", "ForceRestart", "GracefulRestart", "GracefulShutdown", "PushPowerButton", "Nmi", "PowerCycle"],
|
||||
"actioninfo": False, "etag": True},
|
||||
"actioninfo": False, "etag": True, "fw": {"multipart": "/redfish/v1/UpdateService/MultipartUpload", "simple": True,
|
||||
"ids": {"bmc": "Installed-25227", "bios": "Installed-159"},
|
||||
"names": {"bmc": "Integrated Dell Remote Access Controller", "bios": "BIOS"}}},
|
||||
"ilo": {"vendor": "HPE", "manufacturer": "HPE", "model": "ProLiant DL380 Gen11 (pve)", "id": "1",
|
||||
"resets": ["On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"], "actioninfo": False, "etag": False},
|
||||
"resets": ["On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"], "actioninfo": False, "etag": False,
|
||||
"fw": {"multipart": "", "simple": True, "ids": {"bmc": "1", "bios": "2"}, "names": {"bmc": "iLO 6", "bios": "System ROM"}}},
|
||||
"generic": {"vendor": "", "manufacturer": "QEMU", "model": "Standard PC (pve)", "id": "1",
|
||||
"resets": ["On", "ForceOff", "GracefulShutdown", "ForceRestart"], "actioninfo": True, "etag": False},
|
||||
"resets": ["On", "ForceOff", "GracefulShutdown", "ForceRestart"], "actioninfo": True, "etag": False,
|
||||
"fw": {"multipart": "/redfish/v1/UpdateService/update-multipart", "simple": False, "ids": {"bmc": "bmc", "bios": "bios"},
|
||||
"names": {"bmc": "BMC firmware", "bios": "BIOS"}}},
|
||||
}
|
||||
|
||||
|
||||
@@ -82,7 +97,70 @@ class Pve:
|
||||
return order
|
||||
|
||||
|
||||
def make_handler(pve, prof, user, password):
|
||||
class Firmware:
|
||||
"""Installed versions, staged BIOS and update tasks of one emulated BMC."""
|
||||
|
||||
def __init__(self, vmid, prof):
|
||||
self.path = f"/var/lib/pve-vbmc/fw-{vmid}.json"
|
||||
self.prof = prof["fw"]
|
||||
self.lock = threading.Lock()
|
||||
self.tasks = {}
|
||||
try:
|
||||
self.v = json.load(open(self.path))
|
||||
except (OSError, ValueError):
|
||||
self.v = {"bmc": "1.0.0", "bios": "1.0.0", "staged_bios": ""}
|
||||
|
||||
def save(self):
|
||||
try:
|
||||
os.makedirs(os.path.dirname(self.path), exist_ok=True)
|
||||
json.dump(self.v, open(self.path, "w"))
|
||||
except OSError as e: # the versions still hold in memory
|
||||
log.warning("cannot keep firmware state in %s: %s", self.path, e)
|
||||
|
||||
def inventory(self):
|
||||
return [{"@odata.id": f"/redfish/v1/UpdateService/FirmwareInventory/{self.prof['ids'][c]}-{self.v[c]}", "Id": f"{self.prof['ids'][c]}-{self.v[c]}",
|
||||
"Name": self.prof["names"][c], "Version": self.v[c], "Updateable": True} for c in ("bmc", "bios")]
|
||||
|
||||
def start(self, name, data):
|
||||
"""Begin flashing; returns the task id."""
|
||||
comp = "bios" if any(w in name.lower() for w in ("bios", "rom", "flash")) else "bmc"
|
||||
ver = ""
|
||||
for line in data.decode(errors="replace").splitlines():
|
||||
if line.startswith("version="):
|
||||
ver = line.split("=", 1)[1].strip()
|
||||
with self.lock:
|
||||
tid = str(len(self.tasks) + 1)
|
||||
self.tasks[tid] = {"state": "Running", "pct": 0, "msg": f"flashing {comp} {ver}"}
|
||||
threading.Thread(target=self.flash, args=(tid, comp, ver), daemon=True).start()
|
||||
log.info("firmware task %s: %s -> %s (%s)", tid, comp, ver or "?", name)
|
||||
return tid
|
||||
|
||||
def flash(self, tid, comp, ver):
|
||||
for pct in (10, 30, 60, 90):
|
||||
time.sleep(4)
|
||||
self.tasks[tid]["pct"] = pct
|
||||
t = self.tasks[tid]
|
||||
if not ver:
|
||||
t.update(state="Exception", msg="the image is not valid for this system")
|
||||
return
|
||||
with self.lock:
|
||||
if comp == "bmc":
|
||||
self.v["bmc"] = ver
|
||||
t.update(state="Completed", pct=100, msg=f"BMC firmware {ver} installed")
|
||||
else:
|
||||
self.v["staged_bios"] = ver
|
||||
t.update(state="Completed", pct=100, msg=f"BIOS {ver} staged; it applies at the next reboot")
|
||||
self.save()
|
||||
|
||||
def reset(self):
|
||||
with self.lock:
|
||||
if self.v.get("staged_bios"):
|
||||
log.info("reset: BIOS %s applied", self.v["staged_bios"])
|
||||
self.v["bios"], self.v["staged_bios"] = self.v["staged_bios"], ""
|
||||
self.save()
|
||||
|
||||
|
||||
def make_handler(pve, prof, user, password, fw):
|
||||
sysid = prof["id"]
|
||||
syspath = f"/redfish/v1/Systems/{sysid}"
|
||||
state = {"override": "None", "enabled": "Disabled", "etag": 1}
|
||||
@@ -150,6 +228,24 @@ def make_handler(pve, prof, user, password):
|
||||
return self.send(200, self.system(), hdr)
|
||||
if p == syspath + "/ResetActionInfo":
|
||||
return self.send(200, {"Parameters": [{"Name": "ResetType", "Required": True, "AllowableValues": prof["resets"]}]})
|
||||
if p == "/redfish/v1/UpdateService":
|
||||
us = {"@odata.id": p, "ServiceEnabled": True, "FirmwareInventory": {"@odata.id": p + "/FirmwareInventory"}}
|
||||
if fw.prof["multipart"]:
|
||||
us["MultipartHttpPushUri"] = fw.prof["multipart"]
|
||||
if fw.prof["simple"]:
|
||||
us["Actions"] = {"#UpdateService.SimpleUpdate": {"target": p + "/Actions/UpdateService.SimpleUpdate",
|
||||
"TransferProtocol@Redfish.AllowableValues": ["HTTP", "HTTPS"]}}
|
||||
return self.send(200, us)
|
||||
if p == "/redfish/v1/UpdateService/FirmwareInventory":
|
||||
return self.send(200, {"Members": [{"@odata.id": e["@odata.id"]} for e in fw.inventory()]})
|
||||
if p.startswith("/redfish/v1/UpdateService/FirmwareInventory/"):
|
||||
for e in fw.inventory():
|
||||
if e["@odata.id"] == p:
|
||||
return self.send(200, e)
|
||||
if p.startswith("/redfish/v1/TaskService/Tasks/"):
|
||||
t = fw.tasks.get(p.rsplit("/", 1)[1])
|
||||
if t:
|
||||
return self.send(200, {"@odata.id": p, "TaskState": t["state"], "PercentComplete": t["pct"], "Messages": [{"Message": t["msg"]}]})
|
||||
self.error(404, "not found")
|
||||
|
||||
def do_PATCH(self):
|
||||
@@ -170,10 +266,33 @@ def make_handler(pve, prof, user, password):
|
||||
state["etag"] += 1
|
||||
self.send(204)
|
||||
|
||||
def task(self, tid):
|
||||
loc = f"/redfish/v1/TaskService/Tasks/{tid}"
|
||||
self.send(202, {"@odata.id": loc, "@odata.type": "#Task.v1_4_0.Task", "TaskState": "Running"}, {"Location": loc})
|
||||
|
||||
def do_POST(self):
|
||||
if not self.authed():
|
||||
return
|
||||
if self.path.rstrip("/") != syspath + "/Actions/ComputerSystem.Reset":
|
||||
p = self.path.split("?")[0].rstrip("/")
|
||||
if fw.prof["multipart"] and p == fw.prof["multipart"]:
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
if n <= 0:
|
||||
return self.error(411, "Content-Length required")
|
||||
raw = self.rfile.read(n)
|
||||
msg = email.message_from_bytes(b"Content-Type: " + self.headers["Content-Type"].encode() + b"\r\n\r\n" + raw, policy=email.policy.default)
|
||||
for part in msg.iter_parts():
|
||||
if part.get_param("name", header="content-disposition") == "UpdateFile":
|
||||
return self.task(fw.start(part.get_filename() or "image", part.get_payload(decode=True)))
|
||||
return self.error(400, "no UpdateFile part")
|
||||
if fw.prof["simple"] and p == "/redfish/v1/UpdateService/Actions/UpdateService.SimpleUpdate":
|
||||
uri = self.body().get("ImageURI", "")
|
||||
try:
|
||||
r = requests.get(uri, timeout=60)
|
||||
r.raise_for_status()
|
||||
except requests.RequestException as e:
|
||||
return self.error(400, f"cannot download {uri}: {e}")
|
||||
return self.task(fw.start(uri.rsplit("/", 1)[-1], r.content))
|
||||
if p != syspath + "/Actions/ComputerSystem.Reset":
|
||||
return self.error(404, "not found")
|
||||
t = self.body().get("ResetType", "")
|
||||
if t not in prof["resets"]:
|
||||
@@ -190,14 +309,22 @@ def make_handler(pve, prof, user, password):
|
||||
pve.post("status/shutdown", forceStop=1, timeout=60)
|
||||
elif t in ("ForceRestart", "GracefulRestart", "PowerCycle"):
|
||||
pve.post("status/reset" if on else "status/start")
|
||||
if t in ("On", "ForceRestart", "GracefulRestart", "PowerCycle", "PushPowerButton"):
|
||||
fw.reset() # a staged BIOS applies when the host starts again
|
||||
self.send(204)
|
||||
|
||||
return H
|
||||
|
||||
|
||||
def self_signed(addr):
|
||||
d = tempfile.mkdtemp()
|
||||
crt, key = os.path.join(d, "c.pem"), os.path.join(d, "k.pem")
|
||||
def self_signed(addr, vmid):
|
||||
"""A self-signed certificate, kept across restarts like a real BMC's (rm the files for a new one)."""
|
||||
d = "/var/lib/pve-vbmc"
|
||||
crt, key = os.path.join(d, f"tls-{vmid}.crt"), os.path.join(d, f"tls-{vmid}.key")
|
||||
if os.path.exists(crt) and os.path.exists(key):
|
||||
return crt, key
|
||||
if not os.access(d, os.W_OK):
|
||||
d = tempfile.mkdtemp()
|
||||
crt, key = os.path.join(d, "c.pem"), os.path.join(d, "k.pem")
|
||||
subprocess.run(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", "-days", "3650",
|
||||
"-subj", "/CN=pve-redfish", "-addext", f"subjectAltName=IP:{addr}", "-keyout", key, "-out", crt],
|
||||
check=True, capture_output=True)
|
||||
@@ -223,11 +350,11 @@ def main():
|
||||
addr = entry["address"]
|
||||
user = entry.get("user", cfg.get("user", "admin"))
|
||||
password = entry.get("password", cfg.get("password", "password"))
|
||||
handler = make_handler(Pve(cfg, args.vmid), prof, user, password)
|
||||
handler = make_handler(Pve(cfg, args.vmid), prof, user, password, Firmware(args.vmid, prof))
|
||||
ip = ipaddress.ip_address(addr)
|
||||
cls = Server6 if ip.version == 6 else ThreadingHTTPServer
|
||||
srv = cls((addr, int(entry.get("redfish_port", 443))), handler)
|
||||
crt, key = self_signed(addr)
|
||||
crt, key = self_signed(addr, args.vmid)
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
ctx.load_cert_chain(crt, key)
|
||||
srv.socket = ctx.wrap_socket(srv.socket, server_side=True)
|
||||
|
||||
Reference in new issue
Block a user