From c8465c235ac4c6c85aac66c8b30345f812f9e101 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 28 Oct 2026 07:44:13 +0100 Subject: [PATCH] Region as certificate authority, BMC power drivers, IPv6 CA (llmgw-style) - region creates its CA (ECDSA P-256) and serves API/UI/images over HTTPS on tls_listen with a certificate for every name/address it has (reissued on change or a month before expiry); admin_http policy (allow|loopback|deny) for the admin API over plain HTTP; /ca.pem - rack controllers enroll with one-time join codes (CA fingerprint + secret + expiry + address): rollcall controller join-code, rollcall rack join; mutual TLS from then on, identity from the certificate (CN rack:NAME, serial on record), removal revokes, renewal a month before expiry; the shared rack_secret stays as a legacy fallback - clients: CA from the server config, ROLLCALL_CA, or trusted on first connect in the console after showing the fingerprint (kept in client.json); deployed machines trust it via cloud-init ca_certs (allowed on RHEL-family images), overlays get .CA; rollcall ca show|issue Power drivers - ipmi (cipher/privilege/interface), redfish (system discovery, ResetType from AllowableValues or ActionInfo, boot override with If-Match), idrac, ilo, supermicro, xcc, openbmc, proxmox (API token), manual; executed on the rack controller; Redfish auto-detection after commissioning; operator settings survive recommissioning; rollcall bmc, BMC editing in web UI and console - lab: pve_redfish.py Redfish emulator per Proxmox VM (iDRAC, iLO, generic profiles) IPv6 - IPv6 subnets and IPAM (netip), dual-stack default layout (all subnets of the boot VLAN), per-family default routes, accept-ra off where addresses are planned, dnsmasq RA/DHCPv6 with router lifetime 0 unless the rack is the gateway, AAAA host records from IPAM, IPv6 listen addresses and URLs, rack interfaces and region certificate with IPv6, agent reports IPv6 Tests for CA/join/pinning/CSR signing, Redfish (iDRAC, iLO fallbacks, probe), Proxmox, manual, IPv6 candidates, dual-stack plans and dnsmasq IPv6. Verified in the lab: rack2 enrolled and running on mutual TLS only, revocation and single-use codes; iLO auto-detected through rack2, iDRAC and Proxmox drivers, wrong password not saved; Debian 13 (bond+VLAN), Rocky 9 and AlmaLinux 9 dual-stack with Redfish-driven deployments; CA trusted on Debian and AlmaLinux; admin CLI over HTTPS on IPv4 and IPv6. Co-Authored-By: Claude Opus 5.5 --- CHANGELOG.md | 10 + README.md | 99 ++++- agent/rollcall-agent.py | 2 + cmd/rollcall/ca.go | 606 ++++++++++++++++++++++++++++++ cmd/rollcall/ca_test.go | 103 +++++ cmd/rollcall/client.go | 158 +++++++- cmd/rollcall/config.go | 27 +- cmd/rollcall/deploy.go | 21 +- cmd/rollcall/ipmi.go | 242 ++++++++++-- cmd/rollcall/main.go | 19 +- cmd/rollcall/netapi.go | 51 ++- cmd/rollcall/network.go | 279 ++++++++++---- cmd/rollcall/network_test.go | 65 +++- cmd/rollcall/power.go | 535 ++++++++++++++++++++++++++ cmd/rollcall/power_test.go | 190 ++++++++++ cmd/rollcall/rackctl.go | 122 +++--- cmd/rollcall/rackd.go | 263 ++++++++++++- cmd/rollcall/server.go | 90 ++++- cmd/rollcall/stateless.go | 15 +- cmd/rollcall/store.go | 9 + cmd/rollcall/tui.go | 139 ++++++- cmd/rollcall/web/app.js | 46 ++- cmd/rollcall/web/index.html | 1 + deploy/config.example.json | 6 +- deploy/install-rack.sh | 20 +- deploy/rack.example.json | 9 +- lab/pve-vbmc/install.sh | 10 +- lab/pve-vbmc/pve-redfish@.service | 21 ++ lab/pve-vbmc/pve_redfish.py | 239 ++++++++++++ 29 files changed, 3156 insertions(+), 241 deletions(-) create mode 100644 cmd/rollcall/ca.go create mode 100644 cmd/rollcall/ca_test.go create mode 100644 cmd/rollcall/power.go create mode 100644 cmd/rollcall/power_test.go create mode 100644 lab/pve-vbmc/pve-redfish@.service create mode 100644 lab/pve-vbmc/pve_redfish.py diff --git a/CHANGELOG.md b/CHANGELOG.md index edd8cd9..f60dfb7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,16 @@ publish a tag that has no section here. if dnsmasq does not come back. - Web UI: Network page (rack controllers, subnets with ranges and usage, layouts), network tab per machine, layout action; console: Network page, network section, layout action. +- Certificate authority on the region (llmgw-style): HTTPS listener with an auto-renewed region + certificate, `admin_http` policy, `/ca.pem`; rack controllers enroll with one-time join codes + (`rollcall controller join-code`, `rollcall rack join`) and use mutual TLS, renewing their + certificate themselves; removal revokes; deployed machines and overlays trust the CA; + `rollcall ca show|issue`; the console trusts a CA on first connect after showing its fingerprint. +- Power drivers: ipmi, redfish, idrac, ilo, supermicro, xcc, openbmc, proxmox, manual, run on the + rack controller; Redfish auto-detection after commissioning; `rollcall bmc`, BMC editing in the + web UI and console. Lab Redfish emulator with iDRAC/iLO/generic profiles. +- IPv6: subnets, IPAM, dual-stack default layout, per-family default routes, dnsmasq RA/DHCPv6, + AAAA host records, IPv6 listen addresses and URLs, IPv6 in the commissioning inventory. - `rollcall version`; CD-built binary carries `git describe`. - Release notes from `CHANGELOG.md`. diff --git a/README.md b/README.md index ad7d9d0..99dca2f 100644 --- a/README.md +++ b/README.md @@ -380,14 +380,11 @@ controller (`rack_name`, default its hostname), which is all a single-server installation needs. Remote ones: ```sh -# region: config.json gets "rack_secret": "" -# rack host (Debian/Ubuntu), from deploy/ with the binary next to it: -cat > /etc/rollcall/rack.json <", - "url": "http://10.213.78.1", "listen": "10.213.78.1:80"} -EOF -sh install-rack.sh # dnsmasq, syslinux, cache dirs, rollcall-rack.service -rollcall controllers # on the region: it shows up online +rollcall controller join-code -name hall-b -addr 10.0.0.234:443 # on the region: one word, one use, 60 min +# on the rack host (Debian/Ubuntu), from deploy/ with the binary next to it: +./rollcall rack join rollcall-join1-... -url http://10.213.78.1 # writes /etc/rollcall/rack.json + certificate +sh install-rack.sh # dnsmasq, syslinux, cache dirs, rollcall-rack.service +rollcall controllers # on the region: it shows up online rollcall vlan add hall-b 0 -dhcp -rack hall-b ``` @@ -399,9 +396,89 @@ hand-written DHCP configuration for the same interface is the usual cause: The lab runs both: `pxe-server` (embedded, 10.213.77.0/24) and `pxe-rack2` (VM 213, remote, 10.213.78.0/24 on SDN vnet `pxenet2`), with node002 behind -pxe-rack2 and its virtual BMC on pxe-rack2. Rack ↔ region traffic is plain -HTTP in the lab; BMC passwords travel on it for queued commands, so put the -region behind TLS on real networks. +pxe-rack2 and its virtual BMC on pxe-rack2. Rack ↔ region traffic is mutual +TLS (next section), so the BMC passwords of queued commands are encrypted. + +## Certificate authority and TLS + +The region is the installation's certificate authority, as the llmgw control +plane is: `ca_dir` (default `/etc/rollcall/ca`) holds `ca.pem` and +`ca-key.pem` (ECDSA P-256, ten years), created on first start. + +* **HTTPS.** The region serves API, UI and images on `tls_listen` (default + `:443`) with a certificate for every name and address it has plus + `tls_names`; it is reissued when they change or a month before it ends. + `admin_http` decides what plain HTTP may still do with the admin API: + `allow` (default, for compatibility), `loopback` (the CLI on the region + host) or `deny`. Nodes keep booting over HTTP: firmware and the live image + cannot check a certificate before they have one. +* **Rack controllers** join with a code from `rollcall controller join-code`: + one word carrying the first half of the CA's SHA-256, a one-time secret, an + expiry and the region's address. `rollcall rack join` checks the region + against the fingerprint, sends a CSR with the secret and gets a + client+server certificate (`rack:`, a year). From then on every rack + call is mutual TLS and the region identifies the rack by its certificate; + `rollcall controller rm` revokes it at once. Racks renew a month before + expiry. The old `rack_secret` is still accepted when configured. +* **Clients.** On the region host the CLI reads the CA from `ca_dir`; + elsewhere `ROLLCALL_CA=ca.pem`, or the console shows the CA fingerprint on + first connect and keeps the CA in `~/.config/rollcall/client.json` once you + confirm it (`rollcall ca show` prints it on the region). `/ca.pem` serves + the certificate; the web login links to it. +* **Machines** trust it: deployments add it with cloud-init `ca_certs` + (allowed on RHEL-family images, where cloud-init marks the module + unverified), stateless overlays get it as `.CA`. `rollcall ca issue NAME + -host h1,h2 [-client]` issues leaves for anything else (Slurm REST, a + registry, a web front end). + +## Power drivers + +Every machine's BMC record names a driver; commands run on the rack controller +that reaches the BMC network. + +| Driver | For | +|---|---| +| `ipmi` | IPMI 2.0 over LAN (ipmitool lanplus), any vendor; settings `cipher` (17 on newer BMCs), `privilege`, `interface` | +| `redfish` | DMTF Redfish, any vendor: system discovered, ResetType chosen from what the BMC allows, boot override once (with If-Match where the BMC wants it) | +| `idrac` | Dell iDRAC 8/9 (Redfish, `System.Embedded.1`) | +| `ilo` | HPE iLO 4 (2.30+), 5, 6 (Redfish; no PowerCycle, so ForceRestart; PushPowerButton for soft off on older firmware) | +| `supermicro`, `xcc`, `openbmc` | Supermicro, Lenovo XClarity, OpenBMC (Redfish) | +| `proxmox` | Proxmox VE VMs: API token (user = token id, password = secret), settings `node`, `vmid` | +| `manual` | no power control; actions say so | + +After commissioning, a BMC still on IPMI is asked whether it speaks Redfish +with the same account; if it does, the driver for its vendor takes over +(event `bmc driver`). An operator's choice is never overridden. + +```sh +rollcall bmc drivers +rollcall bmc m-1234 -driver idrac -address 10.0.5.21 -user root -password - -verify < pw +rollcall bmc m-1234 -driver redfish -set system=/redfish/v1/Systems/1 -set verify_tls=true +rollcall bmc m-1234 -probe # detect Redfish again +``` + +The lab has `lab/pve-vbmc/pve_redfish.py`, a Redfish service per Proxmox VM +with iDRAC, iLO and generic profiles, next to the IPMI emulator. + +## IPv6 + +Subnets can be IPv6 (`rollcall subnet add fd00:50::/64 ...`). IPAM hands out +addresses from static ranges (walked from their start), deployments get them +in `network-config` with a separate default route per family, and the default +layout links every subnet of the boot NIC's VLAN, so an IPv6 prefix next to +the IPv4 one makes machines dual-stack. Interfaces with planned addresses +ignore router advertisements. dnsmasq serves router advertisements and +DHCPv6 on IPv6 subnets with dhcp on; the rack offers itself as default router +only when it is the subnet's gateway. Host files carry AAAA records, rack +controllers report their IPv6 addresses, the region certificate covers them, +and region, racks and clients work over IPv6 (`https://[fd00::1]`). + +PXE stays IPv4: syslinux cannot netboot over IPv6, so the provisioning VLAN +needs an IPv4 subnet for commissioning and deployment. + +Verified in the lab: Debian 13 (with a bond and VLAN), Rocky 9 and AlmaLinux 9 +dual-stack on two segments, AAAA lookups through dnsmasq, the admin CLI over +`https://[fdc3:...]`. ## Lab diff --git a/agent/rollcall-agent.py b/agent/rollcall-agent.py index 1a4bd1e..6bf1a02 100644 --- a/agent/rollcall-agent.py +++ b/agent/rollcall-agent.py @@ -262,6 +262,8 @@ def network(): "numa_node": readf(f"{sysd}/device/numa_node") or None, "ipv4": [a["local"] + "/" + str(a["prefixlen"]) for a in addrs.get(ifn, {}).get("addr_info", []) if a.get("family") == "inet"], + "ipv6": [a["local"] + "/" + str(a["prefixlen"]) for a in addrs.get(ifn, {}).get("addr_info", []) + if a.get("family") == "inet6" and a.get("scope") == "global"], } if l.get("link_type") == "ether": nic.update(ethtool_info(ifn)) diff --git a/cmd/rollcall/ca.go b/cmd/rollcall/ca.go new file mode 100644 index 0000000..b71de2a --- /dev/null +++ b/cmd/rollcall/ca.go @@ -0,0 +1,606 @@ +package main + +// The region is the certificate authority of its installation, the way the +// llmgw control plane is: one root every part trusts, leaves issued from it, +// identity in the subject. +// +// - the region serves its API and UI over HTTPS with a certificate from it; +// - rack controllers join with a one-time code (CA fingerprint, secret, +// expiry, address in one word) and get a client+server certificate whose +// common name is their name; from then on they talk to the region over +// mutual TLS and the shared rack_secret is not needed; +// - deployed machines trust it (cloud-init ca_certs), stateless overlays get +// it as .CA, and `rollcall ca issue` hands out leaves for anything else. +// +// Nodes still boot over plain HTTP: PXE firmware and the live image have no +// way to check a certificate before they have one. + +import ( + "context" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/binary" + "encoding/hex" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "log" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "sort" + "strings" + "time" +) + +const ( + caValidity = 10 * 365 * 24 * time.Hour + leafValidity = 365 * 24 * time.Hour + backdate = time.Hour + renewBefore = 30 * 24 * time.Hour + joinPrefix = "rollcall-join1-" +) + +type CA struct { + Cert *x509.Certificate + Key *ecdsa.PrivateKey + PEM []byte +} + +func randomSerial() (*big.Int, error) { + return rand.Int(rand.Reader, new(big.Int).Lsh(big.NewInt(1), 128)) +} + +func pemEncode(kind string, der []byte) []byte { + return pem.EncodeToMemory(&pem.Block{Type: kind, Bytes: der}) +} + +func newCA(cn string) (*CA, error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, err + } + serial, err := randomSerial() + if err != nil { + return nil, err + } + tmpl := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: cn, Organization: []string{"rollcall"}}, + NotBefore: time.Now().Add(-backdate), NotAfter: time.Now().Add(caValidity), + KeyUsage: x509.KeyUsageCertSign | x509.KeyUsageCRLSign, BasicConstraintsValid: true, IsCA: true, MaxPathLenZero: true} + der, err := x509.CreateCertificate(rand.Reader, tmpl, tmpl, &key.PublicKey, key) + if err != nil { + return nil, err + } + cert, err := x509.ParseCertificate(der) + if err != nil { + return nil, err + } + return &CA{Cert: cert, Key: key, PEM: pemEncode("CERTIFICATE", der)}, nil +} + +func parseCertPEM(b []byte) (*x509.Certificate, error) { + blk, _ := pem.Decode(b) + if blk == nil { + return nil, errors.New("not PEM") + } + return x509.ParseCertificate(blk.Bytes) +} + +func parseKeyPEM(b []byte) (*ecdsa.PrivateKey, error) { + blk, _ := pem.Decode(b) + if blk == nil { + return nil, errors.New("not PEM") + } + return x509.ParseECPrivateKey(blk.Bytes) +} + +// loadOrCreateCA keeps the authority in dir as ca.pem / ca-key.pem. +func loadOrCreateCA(dir, cn string) (*CA, error) { + cp, kp := filepath.Join(dir, "ca.pem"), filepath.Join(dir, "ca-key.pem") + if cb, err := os.ReadFile(cp); err == nil { + kb, err := os.ReadFile(kp) + if err != nil { + return nil, err + } + cert, err := parseCertPEM(cb) + if err != nil { + return nil, fmt.Errorf("%s: %w", cp, err) + } + key, err := parseKeyPEM(kb) + if err != nil { + return nil, fmt.Errorf("%s: %w", kp, err) + } + return &CA{Cert: cert, Key: key, PEM: cb}, nil + } + ca, err := newCA(cn) + if err != nil { + return nil, err + } + if err := os.MkdirAll(dir, 0o750); err != nil { + return nil, err + } + kder, _ := x509.MarshalECPrivateKey(ca.Key) + if err := os.WriteFile(kp, pemEncode("EC PRIVATE KEY", kder), 0o600); err != nil { + return nil, err + } + if err := os.WriteFile(cp, ca.PEM, 0o644); err != nil { + return nil, err + } + log.Printf("created certificate authority %q in %s (fingerprint %s)", cn, dir, ca.Fingerprint()) + return ca, nil +} + +func certFingerprint(c *x509.Certificate) [32]byte { return sha256.Sum256(c.Raw) } + +// Fingerprint is the SHA-256 of the CA certificate, as people compare it. +func (ca *CA) Fingerprint() string { return fpString(certFingerprint(ca.Cert)) } + +func fpString(sum [32]byte) string { + h := hex.EncodeToString(sum[:]) + var p []string + for i := 0; i < len(h); i += 2 { + p = append(p, strings.ToUpper(h[i:i+2])) + } + return "sha256:" + strings.Join(p, ":") +} + +func splitHosts(hosts []string) (ips []net.IP, dns []string) { + seen := map[string]bool{} + for _, h := range hosts { + h = strings.Trim(strings.TrimSpace(h), "[]") + if h == "" || seen[h] { + continue + } + seen[h] = true + if ip := net.ParseIP(h); ip != nil { + ips = append(ips, ip) + } else { + dns = append(dns, h) + } + } + return +} + +// sign issues a certificate for pub with the given name, hosts and usages. +func (ca *CA) sign(pub crypto.PublicKey, cn string, hosts []string, usage ...x509.ExtKeyUsage) ([]byte, *x509.Certificate, error) { + serial, err := randomSerial() + if err != nil { + return nil, nil, err + } + tmpl := &x509.Certificate{SerialNumber: serial, Subject: pkix.Name{CommonName: cn, Organization: []string{"rollcall"}}, + NotBefore: time.Now().Add(-backdate), NotAfter: time.Now().Add(leafValidity), + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment, ExtKeyUsage: usage} + tmpl.IPAddresses, tmpl.DNSNames = splitHosts(hosts) + der, err := x509.CreateCertificate(rand.Reader, tmpl, ca.Cert, pub, ca.Key) + if err != nil { + return nil, nil, err + } + cert, err := x509.ParseCertificate(der) + return pemEncode("CERTIFICATE", der), cert, err +} + +// issue makes a key and a certificate for it. +func (ca *CA) issue(cn string, hosts []string, usage ...x509.ExtKeyUsage) (certPEM, keyPEM []byte, err error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, nil, err + } + certPEM, _, err = ca.sign(&key.PublicKey, cn, hosts, usage...) + if err != nil { + return nil, nil, err + } + kder, _ := x509.MarshalECPrivateKey(key) + return certPEM, pemEncode("EC PRIVATE KEY", kder), nil +} + +// hostAddresses: names and addresses this machine answers on (no link-local). +func hostAddresses() []string { + h, _ := os.Hostname() + out := []string{"localhost", "127.0.0.1", "::1"} + if h != "" { + out = append(out, h) + } + ifs, _ := net.Interfaces() + for _, i := range ifs { + addrs, _ := i.Addrs() + for _, a := range addrs { + if ipn, ok := a.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() { + out = append(out, ipn.IP.String()) + } + } + } + return out +} + +// regionCert keeps dir/region.pem valid for every name and address the region has. +func regionCert(ca *CA, dir string, extra []string) (tls.Certificate, error) { + cp, kp := filepath.Join(dir, "region.pem"), filepath.Join(dir, "region-key.pem") + want := append(hostAddresses(), extra...) + if cb, err := os.ReadFile(cp); err == nil { + if cert, err := parseCertPEM(cb); err == nil && time.Until(cert.NotAfter) > renewBefore && cert.CheckSignatureFrom(ca.Cert) == nil && coversHosts(cert, want) { + if pair, err := tls.LoadX509KeyPair(cp, kp); err == nil { + pair.Certificate = append(pair.Certificate, ca.Cert.Raw) + return pair, nil + } + } + } + h, _ := os.Hostname() + certPEM, keyPEM, err := ca.issue("rollcall region "+h, want, x509.ExtKeyUsageServerAuth) + if err != nil { + return tls.Certificate{}, err + } + if err := os.WriteFile(kp, keyPEM, 0o600); err != nil { + return tls.Certificate{}, err + } + if err := os.WriteFile(cp, certPEM, 0o644); err != nil { + return tls.Certificate{}, err + } + log.Printf("issued the region's TLS certificate for %s", strings.Join(want, " ")) + pair, err := tls.X509KeyPair(certPEM, keyPEM) + if err != nil { + return pair, err + } + pair.Certificate = append(pair.Certificate, ca.Cert.Raw) // clients that pin the CA find it in the chain + return pair, nil +} + +func coversHosts(c *x509.Certificate, hosts []string) bool { + for _, h := range hosts { + if c.VerifyHostname(strings.Trim(h, "[]")) != nil { + return false + } + } + return true +} + +// ---------------------------------------------------------------- join codes + +type joinCode struct { + Fingerprint [16]byte // first half of the CA certificate's SHA-256 + Secret [16]byte + Expires int64 + Addr string // host:port of the region's HTTPS listener +} + +func (c joinCode) String() string { + b := make([]byte, 0, 36+len(c.Addr)) + b = append(b, c.Fingerprint[:]...) + b = append(b, c.Secret[:]...) + b = binary.BigEndian.AppendUint32(b, uint32(c.Expires)) + b = append(b, c.Addr...) + return joinPrefix + base64.RawURLEncoding.EncodeToString(b) +} + +func parseJoinCode(s string) (joinCode, error) { + raw, ok := strings.CutPrefix(strings.TrimSpace(s), joinPrefix) + if !ok { + return joinCode{}, errors.New("not a join code: it starts " + joinPrefix) + } + b, err := base64.RawURLEncoding.DecodeString(raw) + if err != nil || len(b) < 36+3 { + return joinCode{}, errors.New("the join code is damaged: copy it whole") + } + var c joinCode + copy(c.Fingerprint[:], b[:16]) + copy(c.Secret[:], b[16:32]) + c.Expires = int64(binary.BigEndian.Uint32(b[32:36])) + c.Addr = string(b[36:]) + if _, _, err := net.SplitHostPort(c.Addr); err != nil { + return joinCode{}, errors.New("the join code is damaged: copy it whole") + } + if time.Now().Unix() > c.Expires { + return joinCode{}, fmt.Errorf("the join code expired at %s; ask for a new one", time.Unix(c.Expires, 0).Format(time.RFC3339)) + } + return c, nil +} + +// pinnedTLS trusts a server whose chain carries the CA with this fingerprint. +func pinnedTLS(fp [16]byte) *tls.Config { + return &tls.Config{MinVersion: tls.VersionTLS12, InsecureSkipVerify: true, //nolint:gosec // verified below against the pinned CA + VerifyPeerCertificate: func(raw [][]byte, _ [][]*x509.Certificate) error { + var leaf *x509.Certificate + for i, r := range raw { + c, err := x509.ParseCertificate(r) + if err != nil { + return err + } + if i == 0 { + leaf = c + } + sum := sha256.Sum256(r) + if c.IsCA && string(sum[:16]) == string(fp[:]) { + pool := x509.NewCertPool() + pool.AddCert(c) + _, err := leaf.Verify(x509.VerifyOptions{Roots: pool}) + return err + } + } + return errors.New("the server's certificate is not from the authority this code is for") + }} +} + +func (s *Store) PutJoinToken(ctx context.Context, hash, name string, expires time.Time) error { + _, err := s.pool.Exec(ctx, `INSERT INTO join_tokens(hash, name, expires) VALUES ($1,nullif($2,''),$3)`, hash, name, expires) + return err +} + +// UseJoinToken consumes a token; it returns the name it was bound to. +func (s *Store) UseJoinToken(ctx context.Context, hash string) (string, error) { + var name string + err := s.pool.QueryRow(ctx, `UPDATE join_tokens SET used=now() WHERE hash=$1 AND used IS NULL AND expires > now() RETURNING coalesce(name,'')`, hash).Scan(&name) + if err != nil { + return "", errors.New("unknown, used or expired join code") + } + return name, nil +} + +func (s *Store) SetRackCert(ctx context.Context, name, serial string, expires time.Time) error { + _, err := s.pool.Exec(ctx, `UPDATE rack_controllers SET cert_serial=$2, cert_expires=$3 WHERE name=$1`, name, serial, expires) + return err +} + +func (s *Store) RackCertSerial(ctx context.Context, name string) string { + var serial string + _ = s.pool.QueryRow(ctx, `SELECT coalesce(cert_serial,'') FROM rack_controllers WHERE name=$1`, name).Scan(&serial) + return serial +} + +// ---------------------------------------------------------------- HTTP + +// peerRack is the rack controller a mutual-TLS request comes from, if any: +// a verified client certificate whose serial is the one on record. +func (a *App) peerRack(r *http.Request) string { + if r.TLS == nil || len(r.TLS.VerifiedChains) == 0 || len(r.TLS.PeerCertificates) == 0 { + return "" + } + c := r.TLS.PeerCertificates[0] + name := strings.TrimPrefix(c.Subject.CommonName, "rack:") + if name == c.Subject.CommonName { // only rack certificates + return "" + } + if a.store.RackCertSerial(r.Context(), name) != c.SerialNumber.Text(16) { + return "" + } + return name +} + +func (a *App) handleCAPEM(w http.ResponseWriter, r *http.Request) { + if a.ca == nil { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/x-pem-file") + w.Header().Set("Content-Disposition", `attachment; filename="rollcall-ca.pem"`) + w.Header().Set("X-Rollcall-CA-Fingerprint", a.ca.Fingerprint()) + _, _ = w.Write(a.ca.PEM) +} + +// handleJoinCode makes a one-time code for a rack controller (admin). +func (a *App) handleJoinCode(w http.ResponseWriter, r *http.Request) { + if a.ca == nil || a.cfg.TLSListen == "" { + fail(w, 409, errors.New("the region has no TLS listener (tls_listen)")) + return + } + var req struct { + Name string `json:"name"` + TTL int `json:"ttl_minutes"` + Addr string `json:"addr"` + } + _ = json.NewDecoder(r.Body).Decode(&req) + if req.TTL <= 0 { + req.TTL = 60 + } + addr := req.Addr + if addr == "" { + host := r.Host + if h, _, err := net.SplitHostPort(host); err == nil { + host = h + } + if ip := net.ParseIP(strings.Trim(host, "[]")); (ip != nil && ip.IsLoopback()) || host == "localhost" { + fail(w, 400, errors.New("say where rack controllers reach the region's HTTPS: addr (CLI -addr host:443)")) + return + } + _, port, _ := net.SplitHostPort(a.cfg.TLSListen) + addr = net.JoinHostPort(strings.Trim(host, "[]"), port) + } + var c joinCode + sum := certFingerprint(a.ca.Cert) + copy(c.Fingerprint[:], sum[:16]) + _, _ = rand.Read(c.Secret[:]) + c.Expires = time.Now().Add(time.Duration(req.TTL) * time.Minute).Unix() + c.Addr = addr + h := sha256.Sum256(c.Secret[:]) + if err := a.store.PutJoinToken(r.Context(), hex.EncodeToString(h[:]), req.Name, time.Unix(c.Expires, 0)); err != nil { + fail(w, 500, err) + return + } + writeJSON(w, 200, obj{"code": c.String(), "expires": time.Unix(c.Expires, 0), "addr": addr, "name": req.Name, "ca_fingerprint": a.ca.Fingerprint()}) +} + +type enrollRequest struct { + Secret string `json:"secret"` // base64 of the code's secret + Name string `json:"name"` + CSR string `json:"csr"` + Rack RackController `json:"rack"` +} + +func (a *App) signRackCSR(csrPEM, name string, rc RackController) ([]byte, *x509.Certificate, error) { + blk, _ := pem.Decode([]byte(csrPEM)) + if blk == nil { + return nil, nil, errors.New("csr is not PEM") + } + csr, err := x509.ParseCertificateRequest(blk.Bytes) + if err != nil { + return nil, nil, err + } + if err := csr.CheckSignature(); err != nil { + return nil, nil, err + } + hosts := []string{name} + if u := strings.TrimPrefix(strings.TrimPrefix(rc.URL, "http://"), "https://"); u != "" { + if h, _, err := net.SplitHostPort(u); err == nil { + hosts = append(hosts, h) + } else { + hosts = append(hosts, strings.Trim(u, "/[]")) + } + } + for _, i := range rc.Interfaces { + for _, ad := range i.Addrs { + hosts = append(hosts, stripPrefix(ad)) + } + } + return a.ca.sign(csr.PublicKey, "rack:"+name, hosts, x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth) +} + +// handleRackEnroll: a rack controller presents a join code's secret and a CSR +// over TLS; it gets a certificate and the CA. No admin token, no client cert. +func (a *App) handleRackEnroll(w http.ResponseWriter, r *http.Request) { + if r.TLS == nil || a.ca == nil { + fail(w, 400, errors.New("enrollment needs HTTPS")) + return + } + var req enrollRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil { + fail(w, 400, err) + return + } + sec, err := base64.RawURLEncoding.DecodeString(req.Secret) + if err != nil { + fail(w, 400, err) + return + } + h := sha256.Sum256(sec) + bound, err := a.store.UseJoinToken(r.Context(), hex.EncodeToString(h[:])) + if err != nil { + fail(w, 401, err) + return + } + name := req.Name + if bound != "" { + name = bound + } + if !tagNameRe.MatchString(name) || name == a.cfg.RackName { + fail(w, 400, fmt.Errorf("bad rack controller name %q", name)) + return + } + certPEM, cert, err := a.signRackCSR(req.CSR, name, req.Rack) + if err != nil { + fail(w, 400, err) + return + } + rc := req.Rack + rc.Name, rc.Kind = name, "remote" + if err := a.store.PutRackController(r.Context(), rc); err != nil { + fail(w, 500, err) + return + } + if err := a.store.SetRackCert(r.Context(), name, cert.SerialNumber.Text(16), cert.NotAfter); err != nil { + fail(w, 500, err) + return + } + a.reloadRacks(r.Context()) + a.hub.bump() + log.Printf("rack controller %s enrolled from %s, certificate until %s", name, r.RemoteAddr, cert.NotAfter.Format("2006-01-02")) + writeJSON(w, 200, obj{"name": name, "cert": string(certPEM), "ca": string(a.ca.PEM)}) +} + +// handleRackRenew issues a new certificate to a rack controller that still has a valid one. +func (a *App) handleRackRenew(w http.ResponseWriter, r *http.Request, rack string) { + if a.peerRack(r) != rack { + fail(w, 401, errors.New("renewal needs the current certificate")) + return + } + var req enrollRequest + if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 1<<20)).Decode(&req); err != nil { + fail(w, 400, err) + return + } + rc := a.racksSnapshot()[rack] + certPEM, cert, err := a.signRackCSR(req.CSR, rack, rc) + if err != nil { + fail(w, 400, err) + return + } + if err := a.store.SetRackCert(r.Context(), rack, cert.SerialNumber.Text(16), cert.NotAfter); err != nil { + fail(w, 500, err) + return + } + log.Printf("rack controller %s renewed its certificate, until %s", rack, cert.NotAfter.Format("2006-01-02")) + writeJSON(w, 200, obj{"name": rack, "cert": string(certPEM), "ca": string(a.ca.PEM)}) +} + +// ---------------------------------------------------------------- CLI: rollcall ca (on the region host) + +func caCmd(args []string) error { + cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json")) + if err != nil { + return err + } + if len(args) == 0 || args[0] == "show" { + cb, err := os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")) + if err != nil { + return fmt.Errorf("no CA yet (it is created when rollcall serve starts): %w", err) + } + c, err := parseCertPEM(cb) + if err != nil { + return err + } + fmt.Printf("%s\n subject %s\n valid until %s\n fingerprint %s\n", filepath.Join(cfg.CADir, "ca.pem"), c.Subject.CommonName, + c.NotAfter.Format("2006-01-02"), fpString(certFingerprint(c))) + if rb, err := os.ReadFile(filepath.Join(cfg.CADir, "region.pem")); err == nil { + if rc, err := parseCertPEM(rb); err == nil { + names := append([]string{}, rc.DNSNames...) + for _, ip := range rc.IPAddresses { + names = append(names, ip.String()) + } + sort.Strings(names) + fmt.Printf("region certificate until %s for %s\n", rc.NotAfter.Format("2006-01-02"), strings.Join(names, " ")) + } + } + return nil + } + if args[0] != "issue" || len(args) < 2 { + usage() + } + fs := newFlags("ca issue") + hosts := fs.String("host", "", "comma-separated DNS names and addresses") + client := fs.Bool("client", false, "client certificate (identifies its holder) instead of a server one") + out := fs.String("out", ".", "directory for NAME.pem and NAME-key.pem") + _ = fs.Parse(args[2:]) + ca, err := loadOrCreateCA(cfg.CADir, "rollcall CA") + if err != nil { + return err + } + var hs []string + if *hosts != "" { + hs = strings.Split(*hosts, ",") + } + usage := x509.ExtKeyUsageServerAuth + if *client { + usage = x509.ExtKeyUsageClientAuth + } else if len(hs) == 0 { + return errors.New("a server certificate needs -host") + } + certPEM, keyPEM, err := ca.issue(args[1], hs, usage) + if err != nil { + return err + } + if err := os.WriteFile(filepath.Join(*out, args[1]+"-key.pem"), keyPEM, 0o600); err != nil { + return err + } + if err := os.WriteFile(filepath.Join(*out, args[1]+".pem"), append(certPEM, ca.PEM...), 0o644); err != nil { + return err + } + fmt.Printf("wrote %s.pem (with the CA) and %s-key.pem in %s, valid until %s\n", args[1], args[1], *out, time.Now().Add(leafValidity).Format("2006-01-02")) + return nil +} diff --git a/cmd/rollcall/ca_test.go b/cmd/rollcall/ca_test.go new file mode 100644 index 0000000..0ed1abf --- /dev/null +++ b/cmd/rollcall/ca_test.go @@ -0,0 +1,103 @@ +package main + +import ( + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestCAAndRegionCert(t *testing.T) { + dir := t.TempDir() + ca, err := loadOrCreateCA(dir, "test CA") + if err != nil { + t.Fatal(err) + } + again, err := loadOrCreateCA(dir, "test CA") + if err != nil || again.Fingerprint() != ca.Fingerprint() { + t.Fatalf("reload gives another CA: %v", err) + } + pair, err := regionCert(ca, dir, []string{"rollcall.example", "fd00:213:77::1"}) + if err != nil { + t.Fatal(err) + } + leaf, _ := x509.ParseCertificate(pair.Certificate[0]) + for _, h := range []string{"rollcall.example", "fd00:213:77::1", "127.0.0.1", "localhost"} { + if err := leaf.VerifyHostname(h); err != nil { + t.Errorf("region certificate does not cover %s: %v", h, err) + } + } + if len(pair.Certificate) != 2 { + t.Errorf("the chain should carry the CA for pinning clients") + } + // unchanged names: the same certificate is reused + p2, _ := regionCert(ca, dir, []string{"rollcall.example", "fd00:213:77::1"}) + if string(p2.Certificate[0]) != string(pair.Certificate[0]) { + t.Errorf("certificate reissued without a reason") + } +} + +func TestJoinCodeAndPinnedTLS(t *testing.T) { + dir := t.TempDir() + ca, _ := loadOrCreateCA(dir, "test CA") + pair, _ := regionCert(ca, dir, nil) + srv := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte("ok")) })) + srv.TLS = &tls.Config{Certificates: []tls.Certificate{pair}} + srv.StartTLS() + defer srv.Close() + + var c joinCode + sum := certFingerprint(ca.Cert) + copy(c.Fingerprint[:], sum[:16]) + c.Secret[0] = 7 + c.Expires = time.Now().Add(time.Hour).Unix() + c.Addr = strings.TrimPrefix(srv.URL, "https://") + got, err := parseJoinCode(c.String()) + if err != nil || got != c { + t.Fatalf("round trip: %+v %v", got, err) + } + ok := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(c.Fingerprint)}} + if resp, err := ok.Get(srv.URL); err != nil { + t.Fatalf("pinned CA rejected: %v", err) + } else { + resp.Body.Close() + } + other, _ := newCA("another") + osum := sha256.Sum256(other.Cert.Raw) + var wrong [16]byte + copy(wrong[:], osum[:16]) + bad := &http.Client{Transport: &http.Transport{TLSClientConfig: pinnedTLS(wrong)}} + if _, err := bad.Get(srv.URL); err == nil { + t.Errorf("a server from another authority was accepted") + } + expired := c + expired.Expires = time.Now().Add(-time.Minute).Unix() + if _, err := parseJoinCode(expired.String()); err == nil { + t.Errorf("an expired code was accepted") + } +} + +func TestSignRackCSR(t *testing.T) { + ca, _ := newCA("test CA") + a := &App{ca: ca, cfg: &Config{}} + _, csr, err := newCSR("rack:hall-b") + if err != nil { + t.Fatal(err) + } + _, cert, err := a.signRackCSR(csr, "hall-b", RackController{URL: "http://10.213.78.1", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"fd00:213:78::1/64"}}}}) + if err != nil { + t.Fatal(err) + } + pool := x509.NewCertPool() + pool.AddCert(ca.Cert) + if _, err := cert.Verify(x509.VerifyOptions{Roots: pool, KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth}}); err != nil { + t.Errorf("rack certificate is no client certificate: %v", err) + } + if cert.Subject.CommonName != "rack:hall-b" || cert.VerifyHostname("10.213.78.1") != nil || cert.VerifyHostname("fd00:213:78::1") != nil { + t.Errorf("subject %s, SANs %v %v", cert.Subject.CommonName, cert.IPAddresses, cert.DNSNames) + } +} diff --git a/cmd/rollcall/client.go b/cmd/rollcall/client.go index 4536748..c457142 100644 --- a/cmd/rollcall/client.go +++ b/cmd/rollcall/client.go @@ -3,13 +3,17 @@ package main import ( "archive/tar" "bytes" + "crypto/tls" + "crypto/x509" "encoding/json" "flag" "fmt" "io" + "net" "net/http" "os" "path/filepath" + "sort" "strings" "time" ) @@ -17,14 +21,46 @@ import ( type client struct { base, token string http *http.Client + ca string +} + +// clientTLS: the region's CA from the server config (root on the region), the +// client config (trusted on first connect), or ROLLCALL_CA; else the system roots. +func clientTLS(caPEM string) *tls.Config { + pem := []byte(caPEM) + if f := os.Getenv("ROLLCALL_CA"); f != "" { + if b, err := os.ReadFile(f); err == nil { + pem = b + } + } + if len(pem) == 0 { + if cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json")); err == nil { + pem, _ = os.ReadFile(filepath.Join(cfg.CADir, "ca.pem")) + } + } + tc := &tls.Config{MinVersion: tls.VersionTLS12} + if len(pem) > 0 { + pool, err := x509.SystemCertPool() + if err != nil || pool == nil { + pool = x509.NewCertPool() + } + pool.AppendCertsFromPEM(pem) + tc.RootCAs = pool + } + return tc +} + +func newHTTPClient(caPEM string) *http.Client { + return &http.Client{Timeout: 90 * time.Second, Transport: &http.Transport{TLSClientConfig: clientTLS(caPEM), Proxy: http.ProxyFromEnvironment}} } func newClient() (*client, error) { base, token := os.Getenv("ROLLCALL_URL"), os.Getenv("ROLLCALL_TOKEN") + caPEM := "" if base == "" || token == "" { if cfg, err := loadConfig(envOr("ROLLCALL_CONFIG", "/etc/rollcall/config.json")); err == nil { if base == "" { - base = "http://" + strings.Replace(cfg.Listen, "0.0.0.0", "127.0.0.1", 1) + base = "http://" + localListen(cfg.Listen) } if token == "" { token = cfg.AdminToken @@ -39,12 +75,18 @@ func newClient() (*client, error) { if token == "" { token = cc.Token } + caPEM = cc.CA } } if base == "" || token == "" { return nil, fmt.Errorf("set ROLLCALL_URL and ROLLCALL_TOKEN, make the server config readable, or run rollcall in a terminal to connect") } - return &client{base: strings.TrimRight(base, "/"), token: token, http: &http.Client{Timeout: 90 * time.Second}}, nil + if caPEM == "" { + if cc, ok := loadClientConf(); ok && strings.TrimRight(cc.URL, "/") == strings.TrimRight(base, "/") { + caPEM = cc.CA + } + } + return &client{base: strings.TrimRight(base, "/"), token: token, http: newHTTPClient(caPEM), ca: caPEM}, nil } func (c *client) do(method, path string, body any, out any) error { @@ -225,6 +267,8 @@ func runClient(cmd string, args []string) error { return c.tagCmd(cmd, args) case "location": return c.locationCmd(args) + case "bmc": + return c.bmcCmd(args) case "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp": return c.netCmd(cmd, args) default: @@ -813,3 +857,113 @@ func asList(v any) []any { } return []any{v} } + +func (c *client) bmcCmd(args []string) error { + if len(args) < 1 { + usage() + } + if args[0] == "drivers" { + var ds []powerDriver + if err := c.do("GET", "/api/v1/power-drivers", nil, &ds); err != nil { + return err + } + for _, d := range ds { + fmt.Printf("%-11s %-8s %s\n", d.Name, d.Family, d.Description) + } + return nil + } + fs := newFlags("bmc") + driver := fs.String("driver", "", "power driver (rollcall bmc drivers)") + address := fs.String("address", "", "BMC address (host, IP or https://host:port)") + port := fs.Int("port", 0, "port (0 = the driver's default)") + user := fs.String("user", "", "account (proxmox: token id user@realm!name)") + password := fs.String("password", "", "password (proxmox: token secret); - reads it from stdin") + verify := fs.Bool("verify", false, "check with a status call before saving") + probe := fs.Bool("probe", false, "re-run Redfish detection with the stored account") + var sets multiFlag + fs.Var(&sets, "set", "driver setting KEY=VALUE, repeatable (system, verify_tls, cipher, privilege, interface, node, vmid); KEY= removes it") + _ = fs.Parse(args[1:]) + id := args[0] + req := obj{} + set := func(k, v string) { + if v != "" { + req[k] = v + } + } + set("driver", *driver) + set("address", *address) + set("user", *user) + if *port != 0 { + req["port"] = *port + } + if *password == "-" { + b, err := io.ReadAll(os.Stdin) + if err != nil { + return err + } + req["password"] = strings.TrimRight(string(b), "\r\n") + } else if *password != "" { + req["password"] = *password + } + if len(sets) > 0 { + ex := map[string]string{} + for _, s := range sets { + k, v, _ := strings.Cut(s, "=") + ex[k] = v + } + req["extra"] = ex + } + var d obj + switch { + case *probe: + if err := c.do("POST", "/api/v1/machines/"+id+"/bmc", obj{"probe": true}, &d); err != nil { + return err + } + case len(req) == 0: + if err := c.do("GET", "/api/v1/machines/"+id+"/bmc", nil, &d); err != nil { + return err + } + default: + req["verify"] = *verify + if err := c.do("POST", "/api/v1/machines/"+id+"/bmc", req, &d); err != nil { + return err + } + } + b, _ := d["bmc"].(obj) + drv := str(b["driver"]) + if drv == "" { + drv = "ipmi" + } + fmt.Printf("%s driver %s address %s port %s user %s source %s%s\n", id, drv, str(b["address"]), orDash(str(b["port"])), str(b["user"]), str(b["source"]), + map[bool]string{true: " (set by operator)", false: ""}[b["driver_set"] == true]) + if ex, ok := b["extra"].(obj); ok && len(ex) > 0 { + var kv []string + for k, v := range ex { + kv = append(kv, k+"="+str(v)) + } + sort.Strings(kv) + fmt.Printf(" settings: %s\n", strings.Join(kv, " ")) + } + if rf, ok := b["redfish"].(obj); ok { + fmt.Printf(" redfish %s · %s %s · %s\n", str(rf["redfish"]), str(rf["manufacturer"]), str(rf["model"]), str(rf["system"])) + } + if out := str(d["output"]); out != "" { + fmt.Println(" " + out) + } + return nil +} + +// localListen turns a listen address into one this host can dial. +func localListen(listen string) string { + host, port, err := net.SplitHostPort(listen) + if err != nil { + return listen + } + switch host { + case "", "0.0.0.0": + host = "127.0.0.1" + case "::": + host = "::1" + } + return net.JoinHostPort(host, port) +} diff --git a/cmd/rollcall/config.go b/cmd/rollcall/config.go index 6033ab4..db0de56 100644 --- a/cmd/rollcall/config.go +++ b/cmd/rollcall/config.go @@ -3,6 +3,7 @@ package main import ( "encoding/json" "fmt" + "net" "os" "path/filepath" "strings" @@ -36,7 +37,13 @@ type Config struct { // region/rack: this server runs an embedded rack controller named RackName; // remote rack controllers (rollcall rack) authenticate with RackSecret RackName string `json:"rack_name"` - RackSecret string `json:"rack_secret"` + RackSecret string `json:"rack_secret"` // legacy shared secret; rack controllers join with a code and mutual TLS now + + // certificate authority and HTTPS + CADir string `json:"ca_dir"` // ca.pem, ca-key.pem, region.pem, region-key.pem (default /etc/rollcall/ca) + TLSListen string `json:"tls_listen"` // HTTPS listener, default ":443"; "off" disables + TLSNames []string `json:"tls_names"` // extra names/addresses for the region certificate + AdminHTTP string `json:"admin_http"` // admin API over plain HTTP: allow (default) | loopback | deny } // BMCHint maps a machine MAC to a known BMC (virtual lab nodes). @@ -64,11 +71,11 @@ func loadConfig(path string) (*Config, error) { } } if c.ServerURL == "" { - host, _, _ := strings.Cut(c.Listen, ":") - if host == "" || host == "0.0.0.0" { + host, _, err := net.SplitHostPort(c.Listen) + if err != nil || host == "" || host == "0.0.0.0" || host == "::" { return nil, fmt.Errorf("%s: server_url is required when listen is not a specific address", path) } - c.ServerURL = "http://" + c.Listen + c.ServerURL = "http://" + c.Listen // host:port, IPv6 already bracketed ([::1]:80) } c.ServerURL = strings.TrimRight(c.ServerURL, "/") if c.ImagesDir == "" { @@ -86,5 +93,17 @@ func loadConfig(path string) (*Config, error) { if c.RackName == "" { c.RackName, _ = os.Hostname() } + if c.CADir == "" { + c.CADir = "/etc/rollcall/ca" + } + switch c.TLSListen { + case "": + c.TLSListen = ":443" + case "off", "none": + c.TLSListen = "" + } + if c.AdminHTTP == "" { + c.AdminHTTP = "allow" + } return c, nil } diff --git a/cmd/rollcall/deploy.go b/cmd/rollcall/deploy.go index d06c8f2..9d79aad 100644 --- a/cmd/rollcall/deploy.go +++ b/cmd/rollcall/deploy.go @@ -289,15 +289,28 @@ func (a *App) renderSeed(j *Job, sp DeploySpec, token string) []seedFile { b.WriteString("final_message: \"rollcall: deployed $hostname up after $UPTIME s\"\n") ud = b.String() } - if merr == nil { - if frags, err := a.tagFragments(context.Background(), m); err == nil && len(frags) > 0 { - ud = multipartUserData(ud, frags) + var frags []TagDef + if a.ca != nil { + var cc strings.Builder + cc.WriteString("#cloud-config\n# rollcall: trust the installation's certificate authority\nca_certs:\n trusted:\n - |\n") + for _, l := range strings.Split(strings.TrimSpace(string(a.ca.PEM)), "\n") { + cc.WriteString(" " + l + "\n") } + frags = append(frags, TagDef{Name: "rollcall-ca", CloudConfig: cc.String()}) + } + if merr == nil { + if tf, err := a.tagFragments(context.Background(), m); err == nil { + frags = append(frags, tf...) + } + } + if len(frags) > 0 { + ud = multipartUserData(ud, frags) } files := []seedFile{ {Path: "var/lib/cloud/seed/nocloud/meta-data", Mode: "0600", Content: meta}, {Path: "var/lib/cloud/seed/nocloud/user-data", Mode: "0600", Content: ud}, - {Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\n"}, + // ca_certs is "not verified" on some distros (Rocky, Alma) and skipped unless allowed + {Path: "etc/cloud/cloud.cfg.d/99-rollcall.cfg", Mode: "0644", Content: "# written by rollcall at deploy time\ndatasource_list: [ NoCloud, None ]\nunverified_modules: [ ca_certs ]\n"}, } if sp.NetworkConfig != "" { files = append(files, seedFile{Path: "var/lib/cloud/seed/nocloud/network-config", Mode: "0600", Content: sp.NetworkConfig}) diff --git a/cmd/rollcall/ipmi.go b/cmd/rollcall/ipmi.go index 09a165c..89f34e1 100644 --- a/cmd/rollcall/ipmi.go +++ b/cmd/rollcall/ipmi.go @@ -1,50 +1,244 @@ package main +// A machine's BMC: the record commissioning (or an operator) leaves in +// machines.bmc, the sealed password, and the API that sets and probes it. + import ( + "context" "encoding/json" + "errors" "fmt" + "log" + "net/http" + "strings" ) -// bmcCreds is what the server needs to talk to a machine's BMC. +// bmcCreds is what a power driver needs. type bmcCreds struct { - Address string - Port int - User string - Password string -} - -var powerActions = map[string][]string{ - "status": {"chassis", "power", "status"}, - "on": {"chassis", "power", "on"}, - "off": {"chassis", "power", "off"}, - "soft": {"chassis", "power", "soft"}, - "cycle": {"chassis", "power", "cycle"}, - "reset": {"chassis", "power", "reset"}, + Driver string `json:"driver"` + Address string `json:"address"` + Port int `json:"port"` + User string `json:"user"` + Password string `json:"password"` + Extra map[string]string `json:"extra"` } var bootDevices = map[string]bool{"pxe": true, "disk": true, "cdrom": true, "bios": true, "none": true, "safe": true} +type bmcRecord struct { + Driver string `json:"driver"` + Address string `json:"address"` + Port int `json:"port"` + User string `json:"user"` + Extra map[string]string `json:"extra"` +} + func (a *App) credsFor(m *Machine) (*bmcCreds, error) { if m.BMC == nil || m.bmcSecret == nil { return nil, fmt.Errorf("no BMC credentials stored") } - var b struct { - Address string `json:"address"` - Port int `json:"port"` - User string `json:"user"` - } + var b bmcRecord if err := json.Unmarshal(m.BMC, &b); err != nil { return nil, err } - if b.Address == "" || b.User == "" { - return nil, fmt.Errorf("BMC record has no address/user") + if b.Driver == "manual" { + return &bmcCreds{Driver: "manual"}, nil + } + if b.Address == "" { + return nil, fmt.Errorf("BMC record has no address") } pw, err := a.sealer.Open(m.bmcSecret, m.ID) if err != nil { return nil, fmt.Errorf("decrypt BMC secret: %w", err) } - if b.Port == 0 { - b.Port = 623 + if b.Driver == "" { + b.Driver = "ipmi" } - return &bmcCreds{Address: b.Address, Port: b.Port, User: b.User, Password: pw}, nil + if b.Extra == nil { + b.Extra = map[string]string{} + } + return &bmcCreds{Driver: b.Driver, Address: b.Address, Port: b.Port, User: b.User, Password: pw, Extra: b.Extra}, nil +} + +var errNoBMC = errors.New("no BMC credentials stored") + +// power runs a power operation for a machine on the rack controller that can reach its BMC. +func (a *App) power(ctx context.Context, m *Machine, op, arg string) (string, error) { + c, err := a.credsFor(m) + if err != nil { + return "", fmt.Errorf("%w: %v", errNoBMC, err) + } + return a.powerVia(ctx, a.rackOf(m), c, op, arg) +} + +func (a *App) powerVia(ctx context.Context, rack string, c *bmcCreds, op, arg string) (string, error) { + if rack == a.cfg.RackName || rack == "" { + return powerRun(ctx, a.cfg.IPMITool, c, op, arg) + } + r, ok := a.racksSnapshot()[rack] + if !ok || !rackOnline(r, a.cfg.RackName) { + return "", fmt.Errorf("rack controller %s is offline", rack) + } + return a.hub.dispatch(ctx, rack, &rackJob{Kind: "power", Driver: c.Driver, Address: c.Address, Port: c.Port, User: c.User, + Password: c.Password, Extra: c.Extra, Op: op, Arg: arg}) +} + +func (s *Store) SetBMC(ctx context.Context, id string, bmc []byte, secret []byte) error { + if secret != nil { + _, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2, bmc_secret=$3 WHERE id=$1`, id, bmc, secret) + return err + } + _, err := s.pool.Exec(ctx, `UPDATE machines SET bmc=$2 WHERE id=$1`, id, bmc) + return err +} + +// probeBMC: after commissioning, a BMC still on plain IPMI is asked whether it +// speaks Redfish with the same account; if it does, the vendor's driver takes over. +// An operator's explicit choice (driver_set) is left alone. +func (a *App) probeBMC(m *Machine) { + if m.BMC == nil || m.bmcSecret == nil { + return + } + var b obj + if json.Unmarshal(m.BMC, &b) != nil || b["driver_set"] == true || str(b["address"]) == "" { + return + } + if d := str(b["driver"]); d != "" && d != "ipmi" { + return + } + c, err := a.credsFor(m) + if err != nil { + return + } + probe := *c + probe.Driver, probe.Port = "redfish", 0 + ctx, cancel := context.WithTimeout(context.Background(), 90e9) + defer cancel() + out, err := a.powerVia(ctx, a.rackOf(m), &probe, "probe", "") + if err != nil { + _ = a.store.AddEvent(ctx, m.ID, "bmc probe", "staying on ipmi: "+oneLine(err.Error())) + return + } + var res obj + if json.Unmarshal([]byte(out), &res) != nil || str(res["driver"]) == "" { + return + } + extra, _ := b["extra"].(map[string]any) + if extra == nil { + extra = map[string]any{} + } + extra["system"] = str(res["system"]) + b["driver"], b["extra"], b["redfish"] = str(res["driver"]), extra, res + nb, _ := json.Marshal(b) + if err := a.store.SetBMC(ctx, m.ID, nb, nil); err != nil { + log.Printf("probe %s: %v", m.ID, err) + return + } + _ = a.store.AddEvent(ctx, m.ID, "bmc driver", fmt.Sprintf("%s (Redfish %s, %s %s)", str(res["driver"]), str(res["redfish"]), str(res["manufacturer"]), str(res["model"]))) + log.Printf("%s: BMC speaks Redfish %s (%s) -> driver %s", m.ID, str(res["redfish"]), str(res["vendor"]), str(res["driver"])) +} + +// handleBMC: GET shows the record (no secret); POST sets driver, address, +// account and extra settings, checked with a status call first when verify is set. +func (a *App) handleBMC(w http.ResponseWriter, r *http.Request) { + m, err := a.store.Get(r.Context(), r.PathValue("id")) + if err != nil { + fail(w, 500, err) + return + } + var b obj + _ = json.Unmarshal(m.BMC, &b) + if b == nil { + b = obj{} + } + if r.Method == http.MethodGet { + writeJSON(w, 200, obj{"id": m.ID, "bmc": b, "has_secret": m.bmcSecret != nil, "drivers": powerDrivers}) + return + } + var req struct { + Driver *string `json:"driver"` + Address *string `json:"address"` + Port *int `json:"port"` + User *string `json:"user"` + Password *string `json:"password"` + Extra map[string]string `json:"extra"` + Verify bool `json:"verify"` + Probe bool `json:"probe"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + fail(w, 400, err) + return + } + if req.Probe { // re-run Redfish detection + b["driver_set"] = false + b["driver"] = "ipmi" + nb, _ := json.Marshal(b) + _ = a.store.SetBMC(r.Context(), m.ID, nb, nil) + m, _ = a.store.Get(r.Context(), m.ID) + a.probeBMC(m) + m, _ = a.store.Get(r.Context(), m.ID) + _ = json.Unmarshal(m.BMC, &b) + writeJSON(w, 200, obj{"id": m.ID, "bmc": b}) + return + } + if req.Driver != nil { + if _, ok := driverByName(*req.Driver); !ok { + fail(w, 400, fmt.Errorf("unknown driver %q", *req.Driver)) + return + } + b["driver"] = *req.Driver + } + if req.Address != nil { + b["address"] = strings.TrimSpace(*req.Address) + } + if req.Port != nil { + b["port"] = *req.Port + } + if req.User != nil { + b["user"] = *req.User + } + if req.Extra != nil { + ex, _ := b["extra"].(map[string]any) + if ex == nil { + ex = map[string]any{} + } + for k, v := range req.Extra { + if v == "" { + delete(ex, k) + } else { + ex[k] = v + } + } + b["extra"] = ex + } + b["driver_set"] = true + if str(b["source"]) == "" { + b["source"] = "operator" + } + secret := m.bmcSecret + if req.Password != nil { + if secret, err = a.sealer.Seal(*req.Password, m.ID); err != nil { + fail(w, 500, err) + return + } + } + nb, _ := json.Marshal(b) + trial := *m + trial.BMC, trial.bmcSecret = nb, secret + out := "" + if req.Verify { + out, err = a.power(r.Context(), &trial, "status", "") + if err != nil { + fail(w, 409, fmt.Errorf("not saved, the BMC did not answer: %v %s", err, out)) + return + } + b["configured"], b["verified_oob"] = true, true + nb, _ = json.Marshal(b) + } + if err := a.store.SetBMC(r.Context(), m.ID, nb, secret); err != nil { + fail(w, 500, err) + return + } + _ = a.store.AddEvent(r.Context(), m.ID, "bmc set", fmt.Sprintf("driver=%s address=%s user=%s", str(b["driver"]), str(b["address"]), str(b["user"]))) + writeJSON(w, 200, obj{"id": m.ID, "bmc": b, "output": out}) } diff --git a/cmd/rollcall/main.go b/cmd/rollcall/main.go index ab486c8..1faf024 100644 --- a/cmd/rollcall/main.go +++ b/cmd/rollcall/main.go @@ -28,12 +28,15 @@ func usage() { rollcall tui [-dump WxH -page P -machine ID -plain] rollcall serve [-config PATH] region: API, database, web UI, embedded rack controller rollcall rack [-config /etc/rollcall/rack.json] a rack controller next to the machines + rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] enroll a rack controller (writes rack.json + certificate) + rollcall ca show | issue NAME -host h1,h2 [-client] [-out DIR] the installation's certificate authority (region host) rollcall version rollcall keygen rollcall list rollcall show ID [-json] rollcall events ID rollcall power ID status|on|off|soft|cycle|reset + rollcall bmc ID [-driver D] [-address A] [-user U] [-password P|-] [-set K=V] [-verify] [-probe] | bmc drivers rollcall bootdev ID pxe|disk|cdrom|bios|none rollcall recommission ID [-now] rollcall delete ID @@ -60,7 +63,7 @@ func usage() { rollcall ip list [-subnet CIDR] | reserve IP [-name N] | release IP rollcall layout list | show NAME | add NAME FILE.json | rm NAME rollcall net show ID | set ID [-layout L] [-rack R] [-static IFACE=IP] [-unstatic IP] - rollcall controllers | controller rm NAME | dhcp CONTROLLER + rollcall controllers | controller rm NAME | controller join-code [-name N] [-ttl 60] [-addr HOST:443] | dhcp CONTROLLER (deploy, power and location take tag:NAME instead of ID to act on every machine with that tag) env: ROLLCALL_URL (default from config listen), ROLLCALL_TOKEN, ROLLCALL_CONFIG; away from the server the console saves URL and token in ~/.config/rollcall/client.json @@ -86,9 +89,21 @@ func main() { cmd, args := os.Args[1], os.Args[2:] switch cmd { case "rack": + if len(args) > 0 && args[0] == "join" { + if err := rackJoin(args[1:]); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } + return + } if err := runRack(args); err != nil { log.Fatal(err) } + case "ca": + if err := caCmd(args); err != nil { + fmt.Fprintln(os.Stderr, "error:", err) + os.Exit(1) + } case "tui", "console": if err := tuiDump(args); err != nil { fmt.Fprintln(os.Stderr, "error:", err) @@ -108,7 +123,7 @@ func main() { case "list", "show", "events", "power", "bootdev", "recommission", "delete", "images", "profile", "assign", "unassign", "overlay", "render", "deploy", "release", "jobs", "job", "tag", "tags", "location", - "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp": + "bmc", "fabric", "vlan", "subnet", "range", "ip", "layout", "net", "controllers", "controller", "dhcp": if err := runClient(cmd, args); err != nil { fmt.Fprintln(os.Stderr, "error:", err) os.Exit(1) diff --git a/cmd/rollcall/netapi.go b/cmd/rollcall/netapi.go index 39da35d..2a8c564 100644 --- a/cmd/rollcall/netapi.go +++ b/cmd/rollcall/netapi.go @@ -116,12 +116,18 @@ func (a *App) handleSubnets(w http.ResponseWriter, r *http.Request) { fail(w, 400, err) return } - ip, n, err := net.ParseCIDR(sn.CIDR) - if err != nil || ip.To4() == nil { - fail(w, 400, errors.New("cidr must be an IPv4 network, e.g. 10.0.0.0/24")) + _, n, err := net.ParseCIDR(sn.CIDR) + if err != nil { + fail(w, 400, errors.New("cidr must be a network, e.g. 10.0.0.0/24 or fd00:1::/64")) return } sn.CIDR = n.String() + if familyOf(sn.CIDR) == 6 { + if ones, _ := n.Mask.Size(); ones > 126 { + fail(w, 400, errors.New("an IPv6 subnet needs room: /126 or shorter (/64 is usual)")) + return + } + } if sn.Gateway != "" && !sn.contains(sn.Gateway) { fail(w, 400, fmt.Errorf("gateway %s is outside %s", sn.Gateway, sn.CIDR)) return @@ -202,16 +208,17 @@ func (a *App) handleRanges(w http.ResponseWriter, r *http.Request) { fail(w, 404, fmt.Errorf("subnet %s: %w", rg.Subnet, errSubnetUnknown)) return } - lo, ok1 := ip4(rg.Start) - hi, ok2 := ip4(rg.End) - if !ok1 || !ok2 || hi < lo || !sn.contains(rg.Start) || !sn.contains(rg.End) { + lo, ok1 := parseAddr(rg.Start) + hi, ok2 := parseAddr(rg.End) + if !ok1 || !ok2 || hi.Less(lo) || !sn.contains(rg.Start) || !sn.contains(rg.End) { fail(w, 400, fmt.Errorf("range %s-%s is not inside %s", rg.Start, rg.End, sn.CIDR)) return } + rg.Start, rg.End = lo.String(), hi.String() for _, o := range sn.Ranges { - ol, _ := ip4(o.Start) - oh, _ := ip4(o.End) - if lo <= oh && ol <= hi { + ol, _ := parseAddr(o.Start) + oh, _ := parseAddr(o.End) + if !oh.Less(lo) && !hi.Less(ol) { fail(w, 409, fmt.Errorf("overlaps the %s range %s-%s", o.Kind, o.Start, o.End)) return } @@ -534,7 +541,7 @@ func (c *client) netCmd(cmd string, args []string) error { if err := c.do("GET", "/api/v1/subnets", nil, &ss); err != nil { return err } - fmt.Printf("%-18s %-12s %-14s %-12s %-15s %-12s %9s %s\n", "SUBNET", "PURPOSE", "FABRIC/VLAN", "DHCP RACK", "GATEWAY", "DOMAIN", "USED", "RANGES") + fmt.Printf("%-24s %-12s %-14s %-12s %-15s %-12s %9s %s\n", "SUBNET", "PURPOSE", "FABRIC/VLAN", "DHCP RACK", "GATEWAY", "DOMAIN", "USED", "RANGES") for _, s := range ss { dhcp := "-" if s.DHCP { @@ -547,7 +554,11 @@ func (c *client) netCmd(cmd string, args []string) error { for _, r := range s.Ranges { rs = append(rs, fmt.Sprintf("%s:%s-%s", r.Kind, r.Start, lastOctets(r.End))) } - fmt.Printf("%-18s %-12s %-14s %-12s %-15s %-12s %4d/%-4d %s\n", s.CIDR, s.Purpose, fmt.Sprintf("%s/%d", orDash(s.Fabric), s.VID), dhcp, orDash(s.Gateway), orDash(s.Domain), s.Used, s.Size, strings.Join(rs, " ")) + size := strconv.Itoa(s.Size) + if s.Size == 0 { + size = "∞" + } + fmt.Printf("%-24s %-12s %-14s %-12s %-15s %-12s %4d/%-4s %s\n", s.CIDR, s.Purpose, fmt.Sprintf("%s/%d", orDash(s.Fabric), s.VID), dhcp, orDash(s.Gateway), orDash(s.Domain), s.Used, size, strings.Join(rs, " ")) } case "show": need(2) @@ -743,6 +754,19 @@ func (c *client) netCmd(cmd string, args []string) error { case cmd == "controller" && sub == "rm": need(2) return c.do("DELETE", "/api/v1/controllers/"+args[1], nil, nil) + case cmd == "controller" && sub == "join-code": + fs := newFlags("controller join-code") + name := fs.String("name", "", "bind the code to this rack controller name") + ttl := fs.Int("ttl", 60, "minutes the code is good for") + addr := fs.String("addr", "", "host:port the rack reaches the region's HTTPS at (default: how you reached it)") + _ = fs.Parse(args[1:]) + var d obj + if err := c.do("POST", "/api/v1/controllers/join", obj{"name": *name, "ttl_minutes": *ttl, "addr": *addr}, &d); err != nil { + return err + } + fmt.Printf("%s\n\none use, until %s; region %s, CA %s\non the rack host: rollcall rack join -url http://\n", + str(d["code"]), str(d["expires"]), str(d["addr"]), str(d["ca_fingerprint"])) + return nil default: var rs []RackController if err := c.do("GET", "/api/v1/controllers", nil, &rs); err != nil { @@ -774,6 +798,11 @@ func (m *multiFlag) String() string { return strings.Join(*m, ",") } func (m *multiFlag) Set(v string) error { *m = append(*m, v); return nil } func lastOctets(ip string) string { + if strings.Contains(ip, ":") { + if i := strings.LastIndex(ip, ":"); i >= 0 { + return ip[i:] + } + } if i := strings.LastIndex(ip, "."); i >= 0 { return ip[i+1:] } diff --git a/cmd/rollcall/network.go b/cmd/rollcall/network.go index 461b47b..d16ef36 100644 --- a/cmd/rollcall/network.go +++ b/cmd/rollcall/network.go @@ -13,6 +13,7 @@ import ( "errors" "fmt" "net" + "net/netip" "regexp" "sort" "strconv" @@ -63,7 +64,8 @@ type Subnet struct { Comment string `json:"comment"` Ranges []IPRange `json:"ranges"` Used int `json:"used"` - Size int `json:"size"` + Size int `json:"size"` // host addresses; 0 when too many to count (IPv6) + Family int `json:"family"` // 4 or 6 } type IPAddr struct { @@ -192,9 +194,17 @@ func (s *Store) Subnets(ctx context.Context) ([]Subnet, error) { if sn.DNS == nil { sn.DNS = []string{} } + sn.Family = familyOf(sn.CIDR) if _, n, err := net.ParseCIDR(sn.CIDR); err == nil { ones, bits := n.Mask.Size() - sn.Size = max(0, 1<<(bits-ones)-2) + switch { + case bits-ones > 30: + sn.Size = 0 + case sn.Family == 6: + sn.Size = 1<<(bits-ones) - 1 + default: + sn.Size = max(0, 1<<(bits-ones)-2) + } } sn.Ranges = []IPRange{} out = append(out, sn) @@ -374,15 +384,31 @@ func u2ip(u uint32) string { return b.String() } +// parseAddr reads an IPv4 or IPv6 address (brackets and a /prefix are ignored). +func parseAddr(s string) (netip.Addr, bool) { + a, err := netip.ParseAddr(strings.Trim(stripPrefix(strings.TrimSpace(s)), "[]")) + if err != nil { + return netip.Addr{}, false + } + return a.Unmap(), true +} + func ipLess(a, b string) bool { - x, ok1 := ip4(a) - y, ok2 := ip4(b) + x, ok1 := parseAddr(a) + y, ok2 := parseAddr(b) if ok1 && ok2 { - return x < y + return x.Less(y) } return a < b } +func familyOf(cidr string) int { + if p, err := netip.ParsePrefix(cidr); err == nil && p.Addr().Is6() && !p.Addr().Is4In6() { + return 6 + } + return 4 +} + func stripPrefix(s string) string { ip, _, _ := strings.Cut(s, "/"); return ip } func (sn Subnet) contains(ip string) bool { @@ -426,24 +452,22 @@ func subnetByRef(subnets []Subnet, ref string) *Subnet { // freeCandidates lists the addresses IPAM may hand out in a subnet, in order: // inside static ranges when the subnet has any, never in dynamic or reserved -// ranges, never the network, broadcast, gateway, a rack controller or a used one. +// ranges, never the network (or IPv4 broadcast), the gateway, a rack +// controller or a used one. IPv6 subnets are walked from the start of the +// static ranges (or the subnet) for a bounded number of steps. func freeCandidates(sn Subnet, used map[string]bool, avoid []string, limit int) []string { - _, n, err := net.ParseCIDR(sn.CIDR) - if err != nil || n.IP.To4() == nil { + pfx, err := netip.ParsePrefix(sn.CIDR) + if err != nil { return nil } - ones, bits := n.Mask.Size() - if bits-ones > 16 { // a /15 or larger: refuse to walk it - ones = bits - 16 - } - base := binary.BigEndian.Uint32(n.IP.To4()) - last := base | (1<<(bits-ones) - 1) - type span struct{ lo, hi uint32 } + pfx = pfx.Masked() + base := pfx.Addr() + type span struct{ lo, hi netip.Addr } var static, blocked []span for _, r := range sn.Ranges { - lo, ok1 := ip4(r.Start) - hi, ok2 := ip4(r.End) - if !ok1 || !ok2 { + lo, ok1 := parseAddr(r.Start) + hi, ok2 := parseAddr(r.End) + if !ok1 || !ok2 || hi.Less(lo) { continue } if r.Kind == "static" { @@ -452,35 +476,58 @@ func freeCandidates(sn Subnet, used map[string]bool, avoid []string, limit int) blocked = append(blocked, span{lo, hi}) } } - skip := map[uint32]bool{base: true, last: true} - if g, ok := ip4(sn.Gateway); ok { + skip := map[netip.Addr]bool{base: true} + if base.Is4() { + last := base.As4() + bits := pfx.Bits() + v := binary.BigEndian.Uint32(last[:]) | (1<<(32-bits) - 1) + var b [4]byte + binary.BigEndian.PutUint32(b[:], v) + skip[netip.AddrFrom4(b)] = true + } + if g, ok := parseAddr(sn.Gateway); ok { skip[g] = true } for _, a := range avoid { - if u, ok := ip4(stripPrefix(a)); ok { + if u, ok := parseAddr(a); ok { skip[u] = true } } - in := func(x uint32, ss []span) bool { + in := func(x netip.Addr, ss []span) bool { for _, s := range ss { - if x >= s.lo && x <= s.hi { + if !x.Less(s.lo) && !s.hi.Less(x) { return true } } return false } var out []string - for x := base + 1; x < last; x++ { - if skip[x] || in(x, blocked) || (len(static) > 0 && !in(x, static)) { - continue + try := func(x netip.Addr) bool { + if skip[x] || in(x, blocked) || used[x.String()] { + return false } - ip := u2ip(x) - if used[ip] { - continue + out = append(out, x.String()) + return limit > 0 && len(out) >= limit + } + const maxSteps = 1 << 16 + if len(static) > 0 { + sort.Slice(static, func(i, j int) bool { return static[i].lo.Less(static[j].lo) }) + for _, s := range static { + steps := 0 + for x := s.lo; pfx.Contains(x) && !s.hi.Less(x) && steps < maxSteps; x = x.Next() { + steps++ + if try(x) { + return out + } + } } - out = append(out, ip) - if limit > 0 && len(out) >= limit { - break + return out + } + steps := 0 + for x := base.Next(); x.IsValid() && pfx.Contains(x) && steps < maxSteps; x = x.Next() { + steps++ + if try(x) { + return out } } return out @@ -636,6 +683,7 @@ func selectNICs(nics []invNIC, boot *invNIC, sel string) []invNIC { // ---------------------------------------------------------------- plan type PlanLink struct { + Family int `json:"family"` Subnet string `json:"subnet"` Mode string `json:"mode"` IP string `json:"ip"` @@ -667,12 +715,20 @@ type NetPlan struct { member map[string]invNIC } -func defaultLayout(bootSubnet *Subnet) *NetLayout { - link := LayoutLink{Subnet: "boot", Mode: "auto", Gateway: true} +// defaultLayout: the boot NIC on its subnet, plus every other subnet of the same +// VLAN (an IPv6 prefix next to the IPv4 one makes the machine dual-stack). +func defaultLayout(bootSubnet *Subnet, subnets []Subnet) *NetLayout { + links := []LayoutLink{{Subnet: "boot", Mode: "auto", Gateway: true}} if bootSubnet == nil { - link = LayoutLink{Subnet: "boot", Mode: "dhcp"} + links = []LayoutLink{{Subnet: "boot", Mode: "dhcp"}} + } else if bootSubnet.VLANID != 0 { + for _, s := range subnets { + if s.VLANID == bootSubnet.VLANID && s.CIDR != bootSubnet.CIDR { + links = append(links, LayoutLink{Subnet: s.CIDR, Mode: "auto"}) + } + } } - return &NetLayout{Name: "(default)", Interfaces: []LayoutIface{{Type: "physical", Select: "boot", Links: []LayoutLink{link}}}} + return &NetLayout{Name: "(default)", Interfaces: []LayoutIface{{Type: "physical", Select: "boot", Links: links}}} } // resolvePlan turns a layout into concrete interfaces for one machine. alloc is @@ -689,7 +745,7 @@ func resolvePlan(l *NetLayout, nics []invNIC, boot *invNIC, subnets []Subnet, ha } } if l == nil { - l = defaultLayout(bootSubnet) + l = defaultLayout(bootSubnet, subnets) } p := &NetPlan{Layout: l.Name, Warnings: []string{}, member: map[string]invNIC{}} used := map[string]bool{} @@ -796,13 +852,13 @@ func resolvePlan(l *NetLayout, nics []invNIC, boot *invNIC, subnets []Subnet, ha } if sn == nil { if mode == "dhcp" || mode == "link_up" { - pi.Links = append(pi.Links, PlanLink{Mode: mode}) + pi.Links = append(pi.Links, PlanLink{Mode: mode, Family: 4}) continue } p.Warnings = append(p.Warnings, fmt.Sprintf("%s: subnet %q is not defined", pi.Name, lk.Subnet)) continue } - pl := PlanLink{Subnet: sn.CIDR, Mode: mode, Prefix: sn.bits(), Gateway: sn.Gateway, Default: lk.Gateway, DNS: sn.DNS, Domain: sn.Domain} + pl := PlanLink{Family: familyOf(sn.CIDR), Subnet: sn.CIDR, Mode: mode, Prefix: sn.bits(), Gateway: sn.Gateway, Default: lk.Gateway, DNS: sn.DNS, Domain: sn.Domain} if pi.MTU == 0 && sn.MTU > 0 && sn.MTU != 1500 { pi.MTU = sn.MTU } @@ -830,14 +886,18 @@ func resolvePlan(l *NetLayout, nics []invNIC, boot *invNIC, subnets []Subnet, ha } p.Ifaces = append(p.Ifaces, pi) } - // one default route: the flagged link, else the first addressed link with a gateway on the boot NIC, else anywhere - hasDefault := false - for _, pi := range p.Ifaces { - for _, pl := range pi.Links { - hasDefault = hasDefault || pl.Default + // one default route per family: the flagged link, else the first addressed + // link with a gateway on the boot NIC, else anywhere + for _, fam := range []int{4, 6} { + hasDefault := false + for _, pi := range p.Ifaces { + for _, pl := range pi.Links { + hasDefault = hasDefault || (pl.Default && pl.Family == fam) + } + } + if hasDefault { + continue } - } - if !hasDefault { pickDefault := func(onBoot bool) bool { for i := range p.Ifaces { carriesBoot := false @@ -849,7 +909,7 @@ func resolvePlan(l *NetLayout, nics []invNIC, boot *invNIC, subnets []Subnet, ha } for j := range p.Ifaces[i].Links { pl := &p.Ifaces[i].Links[j] - if pl.Gateway != "" && (pl.Mode == "auto" || pl.Mode == "static") { + if pl.Family == fam && pl.Gateway != "" && (pl.Mode == "auto" || pl.Mode == "static") { pl.Default = true return true } @@ -864,13 +924,19 @@ func resolvePlan(l *NetLayout, nics []invNIC, boot *invNIC, subnets []Subnet, ha return p } -// primaryIPs returns the address carrying the default route and the first IB address. +// primaryIPs returns the address carrying the default route (IPv4 first) and the first IB address. func (p *NetPlan) primaryIPs() (mgmt, ib string) { + for _, fam := range []int{4, 6} { + for _, pi := range p.Ifaces { + for _, pl := range pi.Links { + if pl.Default && pl.IP != "" && mgmt == "" && pl.Family == fam { + mgmt = pl.IP + } + } + } + } for _, pi := range p.Ifaces { for _, pl := range pi.Links { - if pl.Default && pl.IP != "" && mgmt == "" { - mgmt = pl.IP - } if pi.IB && pl.IP != "" && ib == "" { ib = pl.IP } @@ -897,18 +963,27 @@ func networkConfig(p *NetPlan) map[string]any { e["mtu"] = pi.MTU } var addrs, dns, search []string - dhcp := false + var routes []map[string]any + dhcp4, dhcp6 := false, false for _, pl := range pi.Links { switch pl.Mode { case "dhcp": - dhcp = true + if pl.Family == 6 { + dhcp6 = true + } else { + dhcp4 = true + } case "auto", "static": if pl.IP == "" { continue } addrs = append(addrs, fmt.Sprintf("%s/%d", pl.IP, pl.Prefix)) if pl.Default && pl.Gateway != "" { - e["routes"] = []map[string]any{{"to": "0.0.0.0/0", "via": pl.Gateway}} + to := "0.0.0.0/0" + if pl.Family == 6 { + to = "::/0" + } + routes = append(routes, map[string]any{"to": to, "via": pl.Gateway}) } for _, d := range pl.DNS { if !hasTag(dns, d) { @@ -920,10 +995,20 @@ func networkConfig(p *NetPlan) map[string]any { } } } - e["dhcp4"] = dhcp + e["dhcp4"] = dhcp4 + if dhcp6 { + e["dhcp6"] = true + } else { + // addresses come from the plan (or none at all): router advertisements must not add + // SLAAC/DHCPv6 addresses or a second default route + e["accept-ra"] = false + } if len(addrs) > 0 { e["addresses"] = addrs } + if len(routes) > 0 { + e["routes"] = routes + } if len(dns) > 0 || len(search) > 0 { ns := map[string]any{} if len(dns) > 0 { @@ -961,6 +1046,7 @@ func networkConfig(p *NetPlan) map[string]any { for _, n := range pi.nics { e := physical(n, nil) e["dhcp4"] = false + e["accept-ra"] = false if pi.MTU > 0 { e["mtu"] = pi.MTU } @@ -1162,47 +1248,108 @@ func dnsmasqConfig(rc RackController, subnets []Subnet, dnsDir string) (string, if !shared { b.WriteString("dhcp-authoritative\n") } + // IPv6: router advertisements carry the prefix; the rack only offers + // itself as the default router when it is the subnet's gateway + v6ifaces := map[string]bool{} + routerOn := map[string]bool{} + for _, sn := range served { + if familyOf(sn.CIDR) != 6 { + continue + } + for _, ri := range rc.Interfaces { + for _, ad := range ri.Addrs { + if sn.contains(ad) { + v6ifaces[ri.Name] = true + if stripPrefix(ad) == sn.Gateway { + routerOn[ri.Name] = true + } + } + } + } + } + if len(v6ifaces) > 0 { + b.WriteString("enable-ra\n") + for _, i := range sortedKeys(v6ifaces) { + if !routerOn[i] { + fmt.Fprintf(&b, "ra-param=%s,10,0\n", i) + } + } + } for _, d := range domains { fmt.Fprintf(&b, "local=/%s/\n", d) } for i, sn := range served { tag := fmt.Sprintf("s%d", i) + v6 := familyOf(sn.CIDR) == 6 fmt.Fprintf(&b, "\n# %s · %s vlan %d · %s\n", sn.CIDR, orDash(sn.Fabric), sn.VID, sn.Purpose) if sn.Domain != "" { fmt.Fprintf(&b, "domain=%s,%s\n", sn.Domain, sn.CIDR) } + mask := sn.netmask() + if v6 { + mask = strconv.Itoa(sn.bits()) + } dyn := 0 for _, r := range sn.Ranges { if r.Kind != "dynamic" { continue } - lo, ok1 := ip4(r.Start) - hi, ok2 := ip4(r.End) - if !ok1 || !ok2 || hi < lo { + lo, ok1 := parseAddr(r.Start) + hi, ok2 := parseAddr(r.End) + if !ok1 || !ok2 || hi.Less(lo) { continue } // two racks on one VLAN: each serves its own half of the pool - if sn.SecondaryRack != "" { - mid := lo + (hi-lo)/2 + if sn.SecondaryRack != "" && lo.Is4() { + l4, h4 := lo.As4(), hi.As4() + a, z := binary.BigEndian.Uint32(l4[:]), binary.BigEndian.Uint32(h4[:]) + mid := a + (z-a)/2 if rc.Name == sn.PrimaryRack { - hi = mid + z = mid } else { - lo = mid + 1 + a = mid + 1 } + var bl, bh [4]byte + binary.BigEndian.PutUint32(bl[:], a) + binary.BigEndian.PutUint32(bh[:], z) + lo, hi = netip.AddrFrom4(bl), netip.AddrFrom4(bh) } - fmt.Fprintf(&b, "dhcp-range=set:%s,%s,%s,%s,12h\n", tag, u2ip(lo), u2ip(hi), sn.netmask()) + fmt.Fprintf(&b, "dhcp-range=set:%s,%s,%s,%s,12h\n", tag, lo, hi, mask) dyn++ } if dyn == 0 { // static leases only (known machines), no pool for strangers _, n, _ := net.ParseCIDR(sn.CIDR) - fmt.Fprintf(&b, "dhcp-range=set:%s,%s,static,%s,12h\n", tag, n.IP.String(), sn.netmask()) + fmt.Fprintf(&b, "dhcp-range=set:%s,%s,static,%s,12h\n", tag, n.IP.String(), mask) + } + if v6 { + if len(sn.DNS) > 0 { + var ds []string + for _, d := range sn.DNS { + if a, ok := parseAddr(d); ok && a.Is6() { + ds = append(ds, "["+a.String()+"]") + } + } + if len(ds) > 0 { + fmt.Fprintf(&b, "dhcp-option=tag:%s,option6:dns-server,%s\n", tag, strings.Join(ds, ",")) + } + } + if sn.Domain != "" { + fmt.Fprintf(&b, "dhcp-option=tag:%s,option6:domain-search,%s\n", tag, sn.Domain) + } + continue } if sn.Gateway != "" { fmt.Fprintf(&b, "dhcp-option=tag:%s,option:router,%s\n", tag, sn.Gateway) } - if len(sn.DNS) > 0 { - fmt.Fprintf(&b, "dhcp-option=tag:%s,option:dns-server,%s\n", tag, strings.Join(sn.DNS, ",")) + var ds []string + for _, d := range sn.DNS { + if a, ok := parseAddr(d); ok && a.Is4() { + ds = append(ds, a.String()) + } + } + if len(ds) > 0 { + fmt.Fprintf(&b, "dhcp-option=tag:%s,option:dns-server,%s\n", tag, strings.Join(ds, ",")) } if sn.MTU > 0 && sn.MTU != 1500 { fmt.Fprintf(&b, "dhcp-option=tag:%s,option:mtu,%d\n", tag, sn.MTU) diff --git a/cmd/rollcall/network_test.go b/cmd/rollcall/network_test.go index 9ee399a..9792a0d 100644 --- a/cmd/rollcall/network_test.go +++ b/cmd/rollcall/network_test.go @@ -49,7 +49,7 @@ func TestDefaultPlanAndNetworkConfig(t *testing.T) { b, _ := json.Marshal(networkConfig(p)) s := string(b) for _, want := range []string{`"macaddress":"aa:bb:cc:00:00:01"`, `"set-name":"eno1"`, `"addresses":["10.0.0.21/24"]`, - `"routes":[{"to":"0.0.0.0/0","via":"10.0.0.1"}]`, `"search":["lab.local"]`, `"dhcp4":false`} { + `"routes":[{"to":"0.0.0.0/0","via":"10.0.0.1"}]`, `"search":["lab.local"]`, `"dhcp4":false`, `"accept-ra":false`} { if !strings.Contains(s, want) { t.Errorf("network-config lacks %s:\n%s", want, s) } @@ -97,7 +97,7 @@ func TestBondVLANLayout(t *testing.T) { b, _ := json.Marshal(networkConfig(p)) s := string(b) for _, want := range []string{`"bonds":{"bond0":`, `"interfaces":["ens1f0","ens1f1"]`, `"mode":"802.3ad"`, `"vlans":{"bond0.200":`, - `"link":"bond0"`, `"id":200`, `"10.20.0.5/24"`, `"mtu":9000`, `"ibp1s0":{"addresses":["10.30.0.5/24"]`} { + `"link":"bond0"`, `"id":200`, `"10.20.0.5/24"`, `"mtu":9000`, `"ibp1s0":{`, `"addresses":["10.30.0.5/24"]`} { if !strings.Contains(s, want) { t.Errorf("network-config lacks %s:\n%s", want, s) } @@ -163,3 +163,64 @@ func TestDnsmasqConfig(t *testing.T) { t.Errorf("a rack that serves nothing must not get a config") } } + +func v6Subnet() Subnet { + return Subnet{CIDR: "fd00:213:77::/64", Gateway: "fd00:213:77::1", DNS: []string{"fd00:213:77::1"}, Domain: "lab.local", Fabric: "f0", VLANID: 1, + DHCP: true, PrimaryRack: "r1", Purpose: "provisioning", + Ranges: []IPRange{{Kind: "static", Start: "fd00:213:77::50", End: "fd00:213:77::99"}, {Kind: "dynamic", Start: "fd00:213:77::100", End: "fd00:213:77::1ff"}}} +} + +func TestIPv6Candidates(t *testing.T) { + got := freeCandidates(v6Subnet(), map[string]bool{"fd00:213:77::50": true}, []string{"fd00:213:77::51/64"}, 2) + if strings.Join(got, " ") != "fd00:213:77::52 fd00:213:77::53" { + t.Errorf("v6 candidates %v", got) + } + open := Subnet{CIDR: "fd00:1::/64", Gateway: "fd00:1::1", Ranges: []IPRange{{Kind: "reserved", Start: "fd00:1::2", End: "fd00:1::ff"}}} + if got := freeCandidates(open, nil, nil, 1); len(got) != 1 || got[0] != "fd00:1::100" { + t.Errorf("v6 without static range %v", got) + } + if familyOf("fd00:1::/64") != 6 || familyOf("10.0.0.0/24") != 4 { + t.Error("familyOf") + } +} + +func TestDualStackDefaultPlan(t *testing.T) { + v4 := labSubnet() + v4.VLANID = 1 + nics := []invNIC{{Name: "eno1", MAC: "aa:bb:cc:00:00:01", Link: true, IPv4: []string{"10.0.0.150/24"}}} + p := resolvePlan(nil, nics, &nics[0], []Subnet{v4, v6Subnet()}, nil, allocFrom("10.0.0.21", "fd00:213:77::50")) + if len(p.Ifaces) != 1 || len(p.Ifaces[0].Links) != 2 { + t.Fatalf("plan %+v", p) + } + b, _ := json.Marshal(networkConfig(p)) + s := string(b) + for _, want := range []string{`"10.0.0.21/24"`, `"fd00:213:77::50/64"`, `{"to":"0.0.0.0/0","via":"10.0.0.1"}`, `{"to":"::/0","via":"fd00:213:77::1"}`, + `"addresses":["10.0.0.1","fd00:213:77::1"]`} { + if !strings.Contains(s, want) { + t.Errorf("network-config lacks %s:\n%s", want, s) + } + } + if mgmt, _ := p.primaryIPs(); mgmt != "10.0.0.21" { + t.Errorf("mgmt prefers IPv4, got %s", mgmt) + } +} + +func TestDnsmasqIPv6(t *testing.T) { + rc := RackController{Name: "r1", Interfaces: []RackIface{{Name: "eth1", Addrs: []string{"10.0.0.1/24", "fd00:213:77::2/64"}}}} + conf, _ := dnsmasqConfig(rc, []Subnet{labSubnet(), v6Subnet()}, "/d") + for _, want := range []string{"enable-ra", "ra-param=eth1,10,0", "dhcp-range=set:s1,fd00:213:77::100,fd00:213:77::1ff,64,12h", + "dhcp-option=tag:s1,option6:dns-server,[fd00:213:77::1]", "dhcp-option=tag:s1,option6:domain-search,lab.local", "dhcp-option=tag:s0,option:dns-server,10.0.0.1\n"} { + if !strings.Contains(conf, want) { + t.Errorf("dnsmasq conf lacks %q:\n%s", want, conf) + } + } + if strings.Contains(conf, "tag:s1,option:router") { + t.Errorf("no IPv4 router option on an IPv6 subnet:\n%s", conf) + } + // the rack is the IPv6 gateway: it advertises itself as router (no lifetime 0) + rc.Interfaces[0].Addrs[1] = "fd00:213:77::1/64" + conf, _ = dnsmasqConfig(rc, []Subnet{labSubnet(), v6Subnet()}, "/d") + if strings.Contains(conf, "ra-param=eth1,10,0") { + t.Errorf("router lifetime 0 although the rack is the gateway:\n%s", conf) + } +} diff --git a/cmd/rollcall/power.go b/cmd/rollcall/power.go new file mode 100644 index 0000000..efa5545 --- /dev/null +++ b/cmd/rollcall/power.go @@ -0,0 +1,535 @@ +package main + +// Power drivers. A machine's BMC record names a driver; the command runs on +// the rack controller that reaches the BMC network (the embedded one runs it +// in-process). Every driver answers the same operations: +// +// status on off soft cycle reset bootdev(pxe|disk|bios|cdrom|none) probe +// +// and reports power state the way ipmitool does ("Chassis Power is on"), so +// callers do not care which one ran. + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os/exec" + "sort" + "strconv" + "strings" + "time" +) + +type powerDriver struct { + Name string `json:"name"` + Description string `json:"description"` + Family string `json:"family"` // ipmi | redfish | api | none + DefaultPort int `json:"default_port"` +} + +var powerDrivers = []powerDriver{ + {"ipmi", "IPMI 2.0 over LAN (ipmitool lanplus); any vendor", "ipmi", 623}, + {"redfish", "DMTF Redfish, any vendor", "redfish", 443}, + {"idrac", "Dell iDRAC 8/9 (Redfish)", "redfish", 443}, + {"ilo", "HPE iLO 4 (2.30+), 5, 6 (Redfish)", "redfish", 443}, + {"supermicro", "Supermicro X11 and later (Redfish)", "redfish", 443}, + {"xcc", "Lenovo XClarity Controller (Redfish)", "redfish", 443}, + {"openbmc", "OpenBMC (Redfish)", "redfish", 443}, + {"proxmox", "Proxmox VE virtual machine (API token: user = token id, password = secret; extra node, vmid)", "api", 8006}, + {"manual", "no power control: someone presses the button", "none", 0}, +} + +func driverByName(n string) (powerDriver, bool) { + if n == "" { + n = "ipmi" + } + for _, d := range powerDrivers { + if d.Name == n { + return d, true + } + } + return powerDriver{}, false +} + +var powerOps = map[string]bool{"status": true, "on": true, "off": true, "soft": true, "cycle": true, "reset": true, "bootdev": true, "probe": true} + +var errManualPower = errors.New("this machine has manual power control: do it by hand") + +// powerRun executes one operation with a driver. +func powerRun(ctx context.Context, ipmitool string, c *bmcCreds, op, arg string) (string, error) { + if !powerOps[op] { + return "", fmt.Errorf("unknown power operation %q", op) + } + if op == "bootdev" && !bootDevices[arg] { + return "", fmt.Errorf("boot device must be one of pxe, disk, cdrom, bios, none") + } + ctx, cancel := context.WithTimeout(ctx, 90*time.Second) + defer cancel() + d, ok := driverByName(c.Driver) + if !ok { + return "", fmt.Errorf("unknown power driver %q", c.Driver) + } + switch d.Family { + case "ipmi": + return ipmiOp(ctx, ipmitool, c, op, arg) + case "redfish": + if op == "probe" { + return redfishProbe(ctx, c) + } + return redfishOp(ctx, c, op, arg) + case "api": + return proxmoxOp(ctx, c, op, arg) + default: + if op == "status" { + return "Chassis Power is unknown (manual power control)", nil + } + return "", errManualPower + } +} + +// ---------------------------------------------------------------- IPMI + +func ipmiOp(ctx context.Context, tool string, c *bmcCreds, op, arg string) (string, error) { + var args []string + switch op { + case "bootdev": + args = []string{"chassis", "bootdev", arg} + case "probe": + args = []string{"mc", "info"} + default: + args = []string{"chassis", "power", op} + } + return ipmiExec(ctx, tool, c, args...) +} + +func ipmiExec(ctx context.Context, tool string, c *bmcCreds, args ...string) (string, error) { + if tool == "" { + tool = "ipmitool" + } + iface := c.Extra["interface"] + if iface == "" { + iface = "lanplus" + } + port := c.Port + if port == 0 || port == 443 { + port = 623 + } + full := []string{"-I", iface, "-H", c.Address, "-p", strconv.Itoa(port), "-U", c.User, "-E"} + if v := c.Extra["cipher"]; v != "" { // 17 on newer BMCs that dropped suite 3 + full = append(full, "-C", v) + } + if v := c.Extra["privilege"]; v != "" { + full = append(full, "-L", v) + } + cmd := exec.CommandContext(ctx, tool, append(full, args...)...) + cmd.Env = []string{"IPMI_PASSWORD=" + c.Password, "PATH=/usr/sbin:/usr/bin:/sbin:/bin"} + var out bytes.Buffer + cmd.Stdout, cmd.Stderr = &out, &out + err := cmd.Run() + return strings.TrimSpace(out.String()), err +} + +// ---------------------------------------------------------------- Redfish + +type rfClient struct { + base, user, pass string + hc *http.Client + etag string +} + +func bmcHTTPClient(c *bmcCreds) *http.Client { + // BMCs ship self-signed certificates; checking them is opt-in (extra verify_tls=true) + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: c.Extra["verify_tls"] != "true"}, //nolint:gosec + DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext} + return &http.Client{Timeout: 45 * time.Second, Transport: tr} +} + +// bmcBase builds https://host[:port] from the record; IPMI's port is not Redfish's. +func bmcBase(c *bmcCreds, scheme string, defPort int) string { + a := strings.TrimRight(c.Address, "/") + if strings.Contains(a, "://") { + return a + } + port := c.Port + if port == 623 || port == 0 { + port = defPort + } + host := a + if strings.Contains(host, ":") && !strings.HasPrefix(host, "[") { + host = "[" + host + "]" + } + if (scheme == "https" && port == 443) || (scheme == "http" && port == 80) { + return scheme + "://" + host + } + return fmt.Sprintf("%s://%s:%d", scheme, host, port) +} + +func newRF(c *bmcCreds) *rfClient { + return &rfClient{base: bmcBase(c, "https", 443), user: c.User, pass: c.Password, hc: bmcHTTPClient(c)} +} + +func (r *rfClient) do(ctx context.Context, method, path string, body, out any) (int, error) { + var rd io.Reader + if body != nil { + b, _ := json.Marshal(body) + rd = bytes.NewReader(b) + } + req, err := http.NewRequestWithContext(ctx, method, r.base+path, rd) + if err != nil { + return 0, err + } + if r.user != "" { + req.SetBasicAuth(r.user, r.pass) + } + req.Header.Set("Accept", "application/json") + if body != nil { + req.Header.Set("Content-Type", "application/json") + } + if method == "PATCH" && r.etag != "" { + req.Header.Set("If-Match", r.etag) // some services (Supermicro, newer iDRAC) insist + } + resp, err := r.hc.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(io.LimitReader(resp.Body, 4<<20)) + if method == "GET" { + if et := resp.Header.Get("ETag"); et != "" { + r.etag = et + } + } + if resp.StatusCode >= 300 { + return resp.StatusCode, fmt.Errorf("%s %s: %s %s", method, path, resp.Status, redfishMessage(data)) + } + if out != nil && len(data) > 0 { + if err := json.Unmarshal(data, out); err != nil { + return resp.StatusCode, fmt.Errorf("%s %s: %w", method, path, err) + } + } + return resp.StatusCode, nil +} + +// redfishMessage pulls the human message out of a Redfish error body. +func redfishMessage(data []byte) string { + var e struct { + Error struct { + Message string `json:"message"` + Ext []struct { + Message string `json:"Message"` + } `json:"@Message.ExtendedInfo"` + } `json:"error"` + } + if json.Unmarshal(data, &e) == nil { + if len(e.Error.Ext) > 0 && e.Error.Ext[0].Message != "" { + return e.Error.Ext[0].Message + } + if e.Error.Message != "" { + return e.Error.Message + } + } + return oneLine(string(data)) +} + +// redfishSystemPath: extra system=…, else the vendor's usual one if it exists, else the first member. +func (r *rfClient) system(ctx context.Context, c *bmcCreds) (string, obj, error) { + var cands []string + if s := c.Extra["system"]; s != "" { + cands = append(cands, s) + } + switch c.Driver { + case "idrac": + cands = append(cands, "/redfish/v1/Systems/System.Embedded.1") + case "ilo", "xcc", "supermicro": + cands = append(cands, "/redfish/v1/Systems/1") + } + for _, p := range cands { + var sys obj + if _, err := r.do(ctx, "GET", p, nil, &sys); err == nil { + return p, sys, nil + } else if c.Extra["system"] != "" { + return "", nil, err + } + } + var coll struct { + Members []struct { + ID string `json:"@odata.id"` + } `json:"Members"` + } + if _, err := r.do(ctx, "GET", "/redfish/v1/Systems", nil, &coll); err != nil { + return "", nil, err + } + if len(coll.Members) == 0 { + return "", nil, errors.New("the BMC lists no systems") + } + p := coll.Members[0].ID + var sys obj + _, err := r.do(ctx, "GET", p, nil, &sys) + return p, sys, err +} + +// resetTypes are the ResetType values the system accepts. +func (r *rfClient) resetTypes(ctx context.Context, sys obj) []string { + act, _ := dig(sys, "Actions", "#ComputerSystem.Reset").(obj) + var out []string + for _, v := range toList(act["ResetType@Redfish.AllowableValues"]) { + out = append(out, str(v)) + } + if len(out) == 0 { + if info := str(act["@Redfish.ActionInfo"]); info != "" { + var ai struct { + Parameters []struct { + Name string `json:"Name"` + AllowableValues []string `json:"AllowableValues"` + } `json:"Parameters"` + } + if _, err := r.do(ctx, "GET", info, nil, &ai); err == nil { + for _, p := range ai.Parameters { + if p.Name == "ResetType" { + out = p.AllowableValues + } + } + } + } + } + return out +} + +func pickReset(allowed []string, prefs ...string) string { + if len(allowed) == 0 { + return prefs[0] + } + for _, p := range prefs { + if hasTag(allowed, p) { + return p + } + } + return "" +} + +var redfishBootTargets = map[string]string{"pxe": "Pxe", "disk": "Hdd", "bios": "BiosSetup", "cdrom": "Cd", "none": "None", "safe": "None"} + +func redfishOp(ctx context.Context, c *bmcCreds, op, arg string) (string, error) { + r := newRF(c) + path, sys, err := r.system(ctx, c) + if err != nil { + return "", err + } + state := strings.ToLower(str(sys["PowerState"])) // On | Off | PoweringOn | PoweringOff + on := state == "on" || state == "poweringon" + reset := func(prefs ...string) (string, error) { + t := pickReset(r.resetTypes(ctx, sys), prefs...) + if t == "" { + return "", fmt.Errorf("the BMC accepts none of %v", prefs) + } + target := str(dig(sys, "Actions", "#ComputerSystem.Reset", "target")) + if target == "" { + target = path + "/Actions/ComputerSystem.Reset" + } + if _, err := r.do(ctx, "POST", target, obj{"ResetType": t}, nil); err != nil { + return "", err + } + return "Chassis Power Control: " + t + " (redfish)", nil + } + switch op { + case "status": + if on { + return "Chassis Power is on", nil + } + return "Chassis Power is off", nil + case "on": + if on { + return "Chassis Power is on already", nil + } + return reset("On", "PushPowerButton") + case "off": + if !on { + return "Chassis Power is off already", nil + } + return reset("ForceOff", "PushPowerButton") + case "soft": + if !on { + return "Chassis Power is off already", nil + } + return reset("GracefulShutdown", "PushPowerButton") + case "cycle": + if !on { + return reset("On", "PushPowerButton") + } + return reset("PowerCycle", "ForceRestart", "GracefulRestart") + case "reset": + if !on { + return reset("On", "PushPowerButton") + } + return reset("ForceRestart", "PowerCycle", "GracefulRestart") + case "bootdev": + boot := obj{"BootSourceOverrideTarget": redfishBootTargets[arg], "BootSourceOverrideEnabled": "Once"} + if arg == "none" || arg == "safe" { + boot = obj{"BootSourceOverrideEnabled": "Disabled"} + } + if _, err := r.do(ctx, "PATCH", path, obj{"Boot": boot}, nil); err != nil { + return "", err + } + return "Set Boot Device to " + arg + " (redfish " + str(boot["BootSourceOverrideTarget"]) + ", once)", nil + } + return "", fmt.Errorf("redfish: %s is not supported", op) +} + +// redfishProbe asks whether the BMC speaks Redfish with these credentials and +// suggests the driver for its vendor. The answer is JSON. +func redfishProbe(ctx context.Context, c *bmcCreds) (string, error) { + r := newRF(c) + var root obj + if _, err := r.do(ctx, "GET", "/redfish/v1/", nil, &root); err != nil { + return "", fmt.Errorf("no Redfish service: %w", err) + } + res := obj{"redfish": str(root["RedfishVersion"]), "vendor": str(root["Vendor"])} + if res["vendor"] == "" { + for k := range asObj(root["Oem"]) { + res["vendor"] = k + } + } + path, sys, err := r.system(ctx, &bmcCreds{Driver: "redfish", Extra: c.Extra, User: c.User, Password: c.Password, Address: c.Address, Port: c.Port}) + if err != nil { + return "", fmt.Errorf("Redfish %s answers, but not to these credentials: %w", res["redfish"], err) + } + res["system"], res["manufacturer"], res["model"], res["power"] = path, str(sys["Manufacturer"]), str(sys["Model"]), str(sys["PowerState"]) + v := strings.ToLower(str(res["vendor"]) + " " + str(res["manufacturer"])) + switch { + case strings.Contains(v, "dell"): + res["driver"] = "idrac" + case strings.Contains(v, "hpe"), strings.Contains(v, "hewlett"), strings.Contains(v, "hp "): + res["driver"] = "ilo" + case strings.Contains(v, "supermicro"): + res["driver"] = "supermicro" + case strings.Contains(v, "lenovo"): + res["driver"] = "xcc" + case strings.Contains(v, "openbmc"): + res["driver"] = "openbmc" + default: + res["driver"] = "redfish" + } + b, _ := json.Marshal(res) + return string(b), nil +} + +func asObj(v any) obj { + o, _ := v.(obj) + return o +} + +// ---------------------------------------------------------------- Proxmox VE + +func proxmoxOp(ctx context.Context, c *bmcCreds, op, arg string) (string, error) { + node, vmid := c.Extra["node"], c.Extra["vmid"] + if node == "" || vmid == "" { + return "", errors.New("proxmox driver needs extra node and vmid") + } + base := bmcBase(c, "https", 8006) + "/api2/json/nodes/" + url.PathEscape(node) + "/qemu/" + url.PathEscape(vmid) + hc := bmcHTTPClient(c) + call := func(method, path string, form url.Values) (obj, error) { + var rd io.Reader + if form != nil { + rd = strings.NewReader(form.Encode()) + } + req, err := http.NewRequestWithContext(ctx, method, base+path, rd) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "PVEAPIToken="+c.User+"="+c.Password) + if form != nil { + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + } + resp, err := hc.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode >= 300 { + return nil, fmt.Errorf("proxmox %s %s: %s %s", method, path, resp.Status, oneLine(string(data))) + } + var out obj + _ = json.Unmarshal(data, &out) + return out, nil + } + cur, err := call("GET", "/status/current", nil) + if err != nil { + return "", err + } + on := str(dig(cur, "data", "status")) == "running" + act := func(what string) (string, error) { + if _, err := call("POST", "/status/"+what, url.Values{}); err != nil { + return "", err + } + return "Chassis Power Control: " + what + " (proxmox)", nil + } + switch op { + case "status", "probe": + if on { + return "Chassis Power is on", nil + } + return "Chassis Power is off", nil + case "on": + if on { + return "Chassis Power is on already", nil + } + return act("start") + case "off": + if !on { + return "Chassis Power is off already", nil + } + return act("stop") + case "soft": + return act("shutdown") + case "cycle", "reset": + if !on { + return act("start") + } + return act("reset") + case "bootdev": + cfg, err := call("GET", "/config", nil) + if err != nil { + return "", err + } + d, _ := cfg["data"].(obj) + var disks, nets, cds []string + keys := make([]string, 0, len(d)) + for k := range d { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := str(d[k]) + switch { + case strings.HasPrefix(k, "net") && len(k) > 3 && k[3] >= '0' && k[3] <= '9': + nets = append(nets, k) + case strings.Contains(v, "media=cdrom"): + cds = append(cds, k) + case hasTag([]string{"scsi0", "virtio0", "sata0", "ide0", "nvme0"}, k): + disks = append(disks, k) + } + } + var order []string + switch arg { + case "pxe": + order = append(append(append(order, nets...), disks...), cds...) + case "cdrom": + order = append(append(append(order, cds...), disks...), nets...) + default: + order = append(append(append(order, disks...), cds...), nets...) + } + if _, err := call("PUT", "/config", url.Values{"boot": {"order=" + strings.Join(order, ";")}}); err != nil { + return "", err + } + return "Set Boot Device to " + arg + " (proxmox boot order " + strings.Join(order, ";") + ")", nil + } + return "", fmt.Errorf("proxmox: %s is not supported", op) +} diff --git a/cmd/rollcall/power_test.go b/cmd/rollcall/power_test.go new file mode 100644 index 0000000..e0ed83e --- /dev/null +++ b/cmd/rollcall/power_test.go @@ -0,0 +1,190 @@ +package main + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "sync" + "testing" +) + +// fakeRedfish is a tiny Redfish service in the shape of one vendor. +type fakeRedfish struct { + mu sync.Mutex + vendor string + system string + allowed []string + power string + resets []string + boot map[string]any + ifMatch []string +} + +func (f *fakeRedfish) handler(t *testing.T) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + f.mu.Lock() + defer f.mu.Unlock() + u, p, ok := r.BasicAuth() + if r.URL.Path != "/redfish/v1/" && (!ok || u != "admin" || p != "secret") { + w.WriteHeader(401) + _, _ = w.Write([]byte(`{"error":{"message":"bad credentials"}}`)) + return + } + switch { + case r.URL.Path == "/redfish/v1/": + _ = json.NewEncoder(w).Encode(obj{"RedfishVersion": "1.11.0", "Vendor": f.vendor, "Systems": obj{"@odata.id": "/redfish/v1/Systems"}}) + case r.URL.Path == "/redfish/v1/Systems": + _ = json.NewEncoder(w).Encode(obj{"Members": []obj{{"@odata.id": f.system}}}) + case r.URL.Path == f.system && r.Method == "GET": + w.Header().Set("ETag", `W/"42"`) + _ = json.NewEncoder(w).Encode(obj{"PowerState": f.power, "Manufacturer": f.vendor, "Model": "Test 1", + "Actions": obj{"#ComputerSystem.Reset": obj{"target": f.system + "/Actions/ComputerSystem.Reset", "ResetType@Redfish.AllowableValues": f.allowed}}}) + case r.URL.Path == f.system && r.Method == "PATCH": + f.ifMatch = append(f.ifMatch, r.Header.Get("If-Match")) + var body obj + _ = json.NewDecoder(r.Body).Decode(&body) + f.boot, _ = body["Boot"].(map[string]any) + w.WriteHeader(204) + case r.URL.Path == f.system+"/Actions/ComputerSystem.Reset": + var body struct{ ResetType string } + _ = json.NewDecoder(r.Body).Decode(&body) + if !hasTag(f.allowed, body.ResetType) { + w.WriteHeader(400) + _, _ = w.Write([]byte(`{"error":{"@Message.ExtendedInfo":[{"Message":"ResetType not supported"}]}}`)) + return + } + f.resets = append(f.resets, body.ResetType) + switch body.ResetType { + case "On": + f.power = "On" + case "ForceOff", "GracefulShutdown": + f.power = "Off" + } + w.WriteHeader(204) + default: + http.NotFound(w, r) + } + }) +} + +func creds(addr, driver string) *bmcCreds { + u, _ := url.Parse(addr) + return &bmcCreds{Driver: driver, Address: "https://" + u.Host, User: "admin", Password: "secret", Extra: map[string]string{}} +} + +func TestRedfishIDRAC(t *testing.T) { + f := &fakeRedfish{vendor: "Dell", system: "/redfish/v1/Systems/System.Embedded.1", power: "Off", + allowed: []string{"On", "ForceOff", "ForceRestart", "GracefulRestart", "GracefulShutdown", "PushPowerButton", "Nmi", "PowerCycle"}} + srv := httptest.NewTLSServer(f.handler(t)) + defer srv.Close() + c := creds(srv.URL, "idrac") + ctx := context.Background() + if out, _ := powerRun(ctx, "", c, "status", ""); out != "Chassis Power is off" { + t.Errorf("status %q", out) + } + if _, err := powerRun(ctx, "", c, "cycle", ""); err != nil { // off: cycle means on + t.Fatal(err) + } + if _, err := powerRun(ctx, "", c, "cycle", ""); err != nil { + t.Fatal(err) + } + if _, err := powerRun(ctx, "", c, "soft", ""); err != nil { + t.Fatal(err) + } + if got := strings.Join(f.resets, ","); got != "On,PowerCycle,GracefulShutdown" { + t.Errorf("resets %s", got) + } + if _, err := powerRun(ctx, "", c, "bootdev", "pxe"); err != nil { + t.Fatal(err) + } + if f.boot["BootSourceOverrideTarget"] != "Pxe" || f.boot["BootSourceOverrideEnabled"] != "Once" || f.ifMatch[0] != `W/"42"` { + t.Errorf("boot override %v if-match %v", f.boot, f.ifMatch) + } + out, err := powerRun(ctx, "", creds(srv.URL, "redfish"), "probe", "") + if err != nil || !strings.Contains(out, `"driver":"idrac"`) || !strings.Contains(out, "System.Embedded.1") { + t.Errorf("probe %s %v", out, err) + } + bad := creds(srv.URL, "redfish") + bad.Password = "nope" + if _, err := powerRun(ctx, "", bad, "probe", ""); err == nil || !strings.Contains(err.Error(), "credentials") { + t.Errorf("probe with a wrong password must fail clearly: %v", err) + } +} + +// iLO has no PowerCycle and, on older firmware, no GracefulShutdown. +func TestRedfishILOFallbacks(t *testing.T) { + f := &fakeRedfish{vendor: "HPE", system: "/redfish/v1/Systems/1", power: "On", allowed: []string{"On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"}} + srv := httptest.NewTLSServer(f.handler(t)) + defer srv.Close() + c := creds(srv.URL, "ilo") + ctx := context.Background() + for _, op := range []string{"cycle", "soft"} { + if _, err := powerRun(ctx, "", c, op, ""); err != nil { + t.Fatalf("%s: %v", op, err) + } + } + if got := strings.Join(f.resets, ","); got != "ForceRestart,PushPowerButton" { + t.Errorf("resets %s", got) + } + out, _ := powerRun(ctx, "", creds(srv.URL, "redfish"), "probe", "") + if !strings.Contains(out, `"driver":"ilo"`) { + t.Errorf("probe %s", out) + } +} + +func TestProxmoxDriver(t *testing.T) { + var mu sync.Mutex + status, posts, boot := "stopped", []string{}, "" + srv := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + defer mu.Unlock() + if r.Header.Get("Authorization") != "PVEAPIToken=root@pam!rc=s3cret" { + w.WriteHeader(401) + return + } + base := "/api2/json/nodes/pve1/qemu/211" + switch { + case r.URL.Path == base+"/status/current": + _ = json.NewEncoder(w).Encode(obj{"data": obj{"status": status}}) + return + case r.URL.Path == base+"/config" && r.Method == "GET": + _ = json.NewEncoder(w).Encode(obj{"data": obj{"scsi0": "zfspool:vm-211-disk-0,size=30G", "net0": "virtio=BC:24:11:8E:1E:6C", "ide2": "local:iso/x.iso,media=cdrom"}}) + return + case r.URL.Path == base+"/config" && r.Method == "PUT": + _ = r.ParseForm() + boot = r.Form.Get("boot") + case strings.HasPrefix(r.URL.Path, base+"/status/"): + posts = append(posts, strings.TrimPrefix(r.URL.Path, base+"/status/")) + status = "running" + } + _, _ = w.Write([]byte(`{"data":null}`)) + })) + defer srv.Close() + u, _ := url.Parse(srv.URL) + c := &bmcCreds{Driver: "proxmox", Address: "https://" + u.Host, User: "root@pam!rc", Password: "s3cret", Extra: map[string]string{"node": "pve1", "vmid": "211"}} + ctx := context.Background() + if out, _ := powerRun(ctx, "", c, "status", ""); out != "Chassis Power is off" { + t.Errorf("status %q", out) + } + _, _ = powerRun(ctx, "", c, "cycle", "") + _, _ = powerRun(ctx, "", c, "cycle", "") + if strings.Join(posts, ",") != "start,reset" { + t.Errorf("posts %v", posts) + } + if _, err := powerRun(ctx, "", c, "bootdev", "pxe"); err != nil || boot != "order=net0;scsi0;ide2" { + t.Errorf("boot order %q %v", boot, err) + } +} + +func TestManualDriver(t *testing.T) { + c := &bmcCreds{Driver: "manual"} + if out, err := powerRun(context.Background(), "", c, "status", ""); err != nil || !strings.Contains(out, "unknown") { + t.Errorf("status %q %v", out, err) + } + if _, err := powerRun(context.Background(), "", c, "on", ""); err != errManualPower { + t.Errorf("on: %v", err) + } +} diff --git a/cmd/rollcall/rackctl.go b/cmd/rollcall/rackctl.go index 2b0e2b1..f872d5c 100644 --- a/cmd/rollcall/rackctl.go +++ b/cmd/rollcall/rackctl.go @@ -13,7 +13,6 @@ package main // what a single-server installation uses. import ( - "bytes" "context" "crypto/subtle" "encoding/json" @@ -23,7 +22,6 @@ import ( "net" "net/http" "os" - "os/exec" "path/filepath" "sort" "strconv" @@ -64,13 +62,16 @@ type rackState struct { } type rackJob struct { - ID string `json:"id"` - Kind string `json:"kind"` // ipmi - Address string `json:"address"` - Port int `json:"port"` - User string `json:"user"` - Password string `json:"password"` - Args []string `json:"args"` + ID string `json:"id"` + Kind string `json:"kind"` // power + Driver string `json:"driver"` + Address string `json:"address"` + Port int `json:"port"` + User string `json:"user"` + Password string `json:"password"` + Extra map[string]string `json:"extra"` + Op string `json:"op"` + Arg string `json:"arg"` done chan rackResult } @@ -405,40 +406,6 @@ func (a *App) rackAddresses() []string { // ---------------------------------------------------------------- BMC through the machine's rack -var errNoBMC = errors.New("no BMC credentials stored") - -func ipmiExec(ctx context.Context, tool string, c *bmcCreds, args ...string) (string, error) { - ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - if tool == "" { - tool = "ipmitool" - } - full := append([]string{"-I", "lanplus", "-H", c.Address, "-p", strconv.Itoa(c.Port), "-U", c.User, "-E"}, args...) - cmd := exec.CommandContext(ctx, tool, full...) - cmd.Env = []string{"IPMI_PASSWORD=" + c.Password, "PATH=/usr/sbin:/usr/bin:/sbin:/bin"} - var out bytes.Buffer - cmd.Stdout, cmd.Stderr = &out, &out - err := cmd.Run() - return strings.TrimSpace(out.String()), err -} - -// bmc runs an ipmitool command for a machine, on the rack controller that can reach its BMC. -func (a *App) bmc(ctx context.Context, m *Machine, args ...string) (string, error) { - c, err := a.credsFor(m) - if err != nil { - return "", fmt.Errorf("%w: %v", errNoBMC, err) - } - rack := a.rackOf(m) - if rack == a.cfg.RackName { - return ipmiExec(ctx, a.cfg.IPMITool, c, args...) - } - r, ok := a.racksSnapshot()[rack] - if !ok || !rackOnline(r, a.cfg.RackName) { - return "", fmt.Errorf("rack controller %s is offline", rack) - } - return a.hub.dispatch(ctx, rack, &rackJob{Kind: "ipmi", Address: c.Address, Port: c.Port, User: c.User, Password: c.Password, Args: args}) -} - // ---------------------------------------------------------------- desired state const defaultPXETmpl = `# generated by rollcall for rack controller %s @@ -503,12 +470,21 @@ func (a *App) rackState(ctx context.Context, name string) (*rackState, error) { return st, nil } -// hostsFiles: static DHCP leases for the machines this rack serves, and names for every machine. +// hostsFiles: static DHCP leases for the machines this rack serves, and names +// for every machine: the addresses on the interface carrying the management +// address get the plain name (A and AAAA), others name-, IB name-ib. func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (string, string, error) { ms, err := a.store.List(ctx) if err != nil { return "", "", err } + all, _ := a.store.IPs(ctx, "", "") + byMachine := map[string][]IPAddr{} + for _, x := range all { + if x.MachineID != "" { + byMachine[x.MachineID] = append(byMachine[x.MachineID], x) + } + } var dhcp, hosts strings.Builder dhcp.WriteString("# generated by rollcall\n") hosts.WriteString("# generated by rollcall\n") @@ -518,24 +494,49 @@ func (a *App) hostsFiles(ctx context.Context, rack string, subnets []Subnet) (st } return str(a.cfg.ClusterVars["domain"]) } + line := func(ip, name string) { + if d := domainOf(ip); d != "" { + fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name) + } else { + fmt.Fprintf(&hosts, "%s %s\n", ip, name) + } + } sort.Slice(ms, func(i, j int) bool { return deref(ms[i].NodeName) < deref(ms[j].NodeName) }) for _, m := range ms { if m.NodeName == nil || len(m.MACs) == 0 { continue } - name := *m.NodeName - if ip := deref(m.MgmtIP); ip != "" { - if a.rackOf(m) == rack { - fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], ip, name) - } - if d := domainOf(ip); d != "" { - fmt.Fprintf(&hosts, "%s %s.%s %s\n", ip, name, d, name) - } else { - fmt.Fprintf(&hosts, "%s %s\n", ip, name) + name, mgmt, ibip := *m.NodeName, deref(m.MgmtIP), deref(m.IBIP) + mine := a.rackOf(m) == rack + mgmtIface := "" + for _, x := range byMachine[m.ID] { + if x.IP == mgmt { + mgmtIface = x.Iface } } - if ip := deref(m.IBIP); ip != "" { - fmt.Fprintf(&hosts, "%s %s-ib\n", ip, name) + if mgmt != "" { + line(mgmt, name) + if mine && familyOf(mgmt+"/32") == 4 { + fmt.Fprintf(&dhcp, "%s,%s,%s\n", m.MACs[0], mgmt, name) + } + } + for _, x := range byMachine[m.ID] { + if x.IP == mgmt || x.IP == ibip { + continue + } + v6 := strings.Contains(x.IP, ":") + switch { + case x.Iface == mgmtIface && mgmtIface != "": + line(x.IP, name) + if v6 && mine { + fmt.Fprintf(&dhcp, "%s,[%s],%s\n", m.MACs[0], x.IP, name) + } + case x.Iface != "": + line(x.IP, name+"-"+strings.NewReplacer(".", "-", ":", "-").Replace(x.Iface)) + } + } + if ibip != "" { + fmt.Fprintf(&hosts, "%s %s-ib\n", ibip, name) } } return dhcp.String(), hosts.String(), nil @@ -641,7 +642,7 @@ func localInterfaces() []RackIface { ri := RackIface{Name: i.Name, MAC: i.HardwareAddr.String()} addrs, _ := i.Addrs() for _, ad := range addrs { - if ipn, ok := ad.(*net.IPNet); ok && ipn.IP.To4() != nil { + if ipn, ok := ad.(*net.IPNet); ok && !ipn.IP.IsLinkLocalUnicast() && !ipn.IP.IsLoopback() { ri.Addrs = append(ri.Addrs, ipn.String()) } } @@ -741,9 +742,13 @@ func (a *App) exportDnsmasq(ctx context.Context) error { func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack string)) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if name := a.peerRack(r); name != "" { + next(w, r, name) + return + } sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack") if a.cfg.RackSecret == "" || subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) != 1 || name == "" { - fail(w, 401, errors.New("rack controller secret required")) + fail(w, 401, errors.New("rack controller certificate (or legacy secret) required")) return } next(w, r, name) @@ -752,6 +757,9 @@ func (a *App) rackAuth(next func(w http.ResponseWriter, r *http.Request, rack st // fromRack names the rack controller a node request came through (it proxies them). func (a *App) fromRack(r *http.Request) string { + if name := a.peerRack(r); name != "" { + return name + } sec, name := r.Header.Get("X-Rollcall-Rack-Secret"), r.Header.Get("X-Rollcall-Rack") if a.cfg.RackSecret != "" && name != "" && subtle.ConstantTimeCompare([]byte(sec), []byte(a.cfg.RackSecret)) == 1 { return name diff --git a/cmd/rollcall/rackd.go b/cmd/rollcall/rackd.go index 21a2bdb..88e2a37 100644 --- a/cmd/rollcall/rackd.go +++ b/cmd/rollcall/rackd.go @@ -10,15 +10,24 @@ package main import ( "bytes" "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" "encoding/json" "errors" "fmt" "io" "log" + "net" "net/http" "net/http/httputil" "net/url" "os" + "os/user" "path" "path/filepath" "strconv" @@ -28,16 +37,19 @@ import ( ) type RackConfig struct { - Name string `json:"name"` // default: hostname - Region string `json:"region"` // the region's URL, e.g. http://10.0.0.234 - Secret string `json:"secret"` // the region's rack_secret - URL string `json:"url"` // how machines reach this rack, e.g. http://10.213.78.1 - Listen string `json:"listen"` // default ":80" - TFTPRoot string `json:"tftp_root"` // default /srv/tftp - DnsmasqDir string `json:"dnsmasq_dir"` // default /var/lib/rollcall/dnsmasq - CacheDir string `json:"cache_dir"` // default /var/cache/rollcall - IPMITool string `json:"ipmitool"` // default ipmitool - DHCPInterfaces []string `json:"dhcp_interfaces"` // default: the interfaces with an address in a served subnet + Name string `json:"name,omitempty"` // default: hostname + Region string `json:"region,omitempty"` // the region's URL, e.g. http://10.0.0.234 + Secret string `json:"secret,omitempty"` // the region's rack_secret + URL string `json:"url,omitempty"` // how machines reach this rack, e.g. http://10.213.78.1 + Listen string `json:"listen,omitempty"` // default ":80" + TFTPRoot string `json:"tftp_root,omitempty"` // default /srv/tftp + DnsmasqDir string `json:"dnsmasq_dir,omitempty"` // default /var/lib/rollcall/dnsmasq + CacheDir string `json:"cache_dir,omitempty"` // default /var/cache/rollcall + IPMITool string `json:"ipmitool,omitempty"` // default ipmitool + DHCPInterfaces []string `json:"dhcp_interfaces,omitempty"` // default: the interfaces with an address in a served subnet + CAFile string `json:"ca_file,omitempty"` // set by rollcall rack join: the region's CA, + CertFile string `json:"cert_file,omitempty"` // this rack controller's certificate + KeyFile string `json:"key_file,omitempty"` // and key; with them the secret is not needed } type rackd struct { @@ -45,7 +57,11 @@ type rackd struct { region *url.URL hc *http.Client // long polls fast *http.Client + big *http.Client // image downloads, no timeout gen int64 + tlsc *tls.Config + certMu sync.Mutex + cert *tls.Certificate headMu sync.Mutex heads map[string]headInfo @@ -86,14 +102,206 @@ func loadRackConfig(p string) (RackConfig, error) { c.IPMITool = "ipmitool" } c.Region, c.URL = strings.TrimRight(c.Region, "/"), strings.TrimRight(c.URL, "/") - for _, k := range []struct{ v, n string }{{c.Region, "region"}, {c.Secret, "secret"}, {c.URL, "url"}} { + for _, k := range []struct{ v, n string }{{c.Region, "region"}, {c.URL, "url"}} { if k.v == "" { return c, fmt.Errorf("%s: %s is required", p, k.n) } } + if c.Secret == "" && c.CertFile == "" { + return c, fmt.Errorf("%s: not enrolled: run rollcall rack join CODE (or set the legacy secret)", p) + } return c, nil } +// clients builds the HTTP clients; with a certificate they speak mutual TLS +// and pick up a renewed certificate without being rebuilt. +func (r *rackd) clients() error { + r.tlsc = &tls.Config{MinVersion: tls.VersionTLS12} + if r.cfg.CertFile != "" { + cab, err := os.ReadFile(r.cfg.CAFile) + if err != nil { + return err + } + pool := x509.NewCertPool() + if !pool.AppendCertsFromPEM(cab) { + return fmt.Errorf("%s: no certificate", r.cfg.CAFile) + } + pair, err := tls.LoadX509KeyPair(r.cfg.CertFile, r.cfg.KeyFile) + if err != nil { + return err + } + pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0]) + r.cert = &pair + r.tlsc.RootCAs = pool + r.tlsc.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + r.certMu.Lock() + defer r.certMu.Unlock() + return r.cert, nil + } + } + tr := func() *http.Transport { + return &http.Transport{TLSClientConfig: r.tlsc, Proxy: http.ProxyFromEnvironment, MaxIdleConnsPerHost: 8, + DialContext: (&net.Dialer{Timeout: 10 * time.Second}).DialContext} + } + r.hc = &http.Client{Timeout: 45 * time.Second, Transport: tr()} + r.fast = &http.Client{Timeout: 30 * time.Second, Transport: tr()} + r.big = &http.Client{Transport: tr()} + return nil +} + +// renewLoop asks for a new certificate a month before the current one ends. +func (r *rackd) renewLoop() { + for { + r.certMu.Lock() + leaf := r.cert.Leaf + r.certMu.Unlock() + if leaf != nil && time.Until(leaf.NotAfter) < renewBefore { + if err := r.renew(); err != nil { + log.Printf("certificate renewal: %v", err) + } + } + time.Sleep(12 * time.Hour) + } +} + +func (r *rackd) renew() error { + key, csr, err := newCSR("rack:" + r.cfg.Name) + if err != nil { + return err + } + var resp struct { + Cert string `json:"cert"` + } + if _, err := r.call(r.fast, "POST", "/api/v1/rack/renew", enrollRequest{CSR: csr}, &resp); err != nil { + return err + } + pair, err := tls.X509KeyPair([]byte(resp.Cert), key) + if err != nil { + return err + } + pair.Leaf, _ = x509.ParseCertificate(pair.Certificate[0]) + if err := os.WriteFile(r.cfg.KeyFile, key, 0o640); err != nil { + return err + } + if err := os.WriteFile(r.cfg.CertFile, []byte(resp.Cert), 0o644); err != nil { + return err + } + r.certMu.Lock() + r.cert = &pair + r.certMu.Unlock() + log.Printf("renewed the rack certificate, valid until %s", pair.Leaf.NotAfter.Format("2006-01-02")) + return nil +} + +func newCSR(cn string) (keyPEM []byte, csrPEM string, err error) { + key, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + if err != nil { + return nil, "", err + } + der, err := x509.CreateCertificateRequest(rand.Reader, &x509.CertificateRequest{Subject: pkix.Name{CommonName: cn}}, key) + if err != nil { + return nil, "", err + } + kder, _ := x509.MarshalECPrivateKey(key) + return pemEncode("EC PRIVATE KEY", kder), string(pemEncode("CERTIFICATE REQUEST", der)), nil +} + +// rackJoin enrolls this host as a rack controller with a code from +// `rollcall controller join-code` and writes rack.json and the certificate. +func rackJoin(args []string) error { + if len(args) < 1 || strings.HasPrefix(args[0], "-") { + return errors.New("usage: rollcall rack join CODE -url http://ADDR [-name N] [-listen ADDR:80] [-config /etc/rollcall/rack.json]") + } + code, err := parseJoinCode(args[0]) + if err != nil { + return err + } + fs := newFlags("rack join") + host, _ := os.Hostname() + name := fs.String("name", host, "rack controller name") + u := fs.String("url", "", "how machines reach this rack controller, e.g. http://10.213.78.1 (required)") + listen := fs.String("listen", "", "listen address (default: the url's host, port 80)") + cfgPath := fs.String("config", "/etc/rollcall/rack.json", "configuration to write") + dir := fs.String("dir", "/etc/rollcall/rack", "where the certificate goes") + _ = fs.Parse(args[1:]) + if *u == "" { + return errors.New("-url is required: the address machines on this segment reach this host at") + } + cfg := RackConfig{} + if b, err := os.ReadFile(*cfgPath); err == nil { + _ = json.Unmarshal(b, &cfg) + } + cfg.Name, cfg.URL, cfg.Region, cfg.Secret = *name, strings.TrimRight(*u, "/"), "https://"+code.Addr, "" + if *listen != "" { + cfg.Listen = *listen + } else if cfg.Listen == "" { + if pu, err := url.Parse(cfg.URL); err == nil { + cfg.Listen = net.JoinHostPort(pu.Hostname(), "80") + } + } + cfg.CAFile, cfg.CertFile, cfg.KeyFile = filepath.Join(*dir, "ca.pem"), filepath.Join(*dir, "rack.pem"), filepath.Join(*dir, "rack-key.pem") + if cfg.TFTPRoot == "" { + cfg.TFTPRoot = "/srv/tftp" + } + key, csr, err := newCSR("rack:" + cfg.Name) + if err != nil { + return err + } + req := enrollRequest{Secret: base64.RawURLEncoding.EncodeToString(code.Secret[:]), Name: cfg.Name, CSR: csr, + Rack: RackController{URL: cfg.URL, Version: version, Interfaces: localInterfaces(), DHCPInterfaces: cfg.DHCPInterfaces, TFTPRoot: cfg.TFTPRoot}} + body, _ := json.Marshal(req) + hc := &http.Client{Timeout: 30 * time.Second, Transport: &http.Transport{TLSClientConfig: pinnedTLS(code.Fingerprint)}} + resp, err := hc.Post(cfg.Region+"/api/v1/rack/enroll", "application/json", bytes.NewReader(body)) + if err != nil { + return fmt.Errorf("cannot reach the region at %s: %w", cfg.Region, err) + } + defer resp.Body.Close() + data, _ := io.ReadAll(resp.Body) + if resp.StatusCode != 200 { + var e apiError + _ = json.Unmarshal(data, &e) + return fmt.Errorf("the region refused: %s %s", resp.Status, e.Error) + } + var out struct { + Name, Cert, CA string + } + if err := json.Unmarshal(data, &out); err != nil { + return err + } + cfg.Name = out.Name + if err := os.MkdirAll(*dir, 0o750); err != nil { + return err + } + for _, f := range []struct { + p string + b []byte + mode os.FileMode + }{{cfg.CAFile, []byte(out.CA), 0o644}, {cfg.CertFile, []byte(out.Cert), 0o644}, {cfg.KeyFile, key, 0o640}} { + if err := os.WriteFile(f.p, f.b, f.mode); err != nil { + return err + } + } + cb, _ := json.MarshalIndent(cfg, "", " ") + if err := os.MkdirAll(filepath.Dir(*cfgPath), 0o755); err != nil { + return err + } + if err := os.WriteFile(*cfgPath, append(cb, '\n'), 0o640); err != nil { + return err + } + // the service runs as user rollcall: give its group the key and the configuration + if g, err := user.LookupGroup("rollcall"); err == nil { + if gid, err := strconv.Atoi(g.Gid); err == nil { + for _, p := range []string{*dir, cfg.CAFile, cfg.CertFile, cfg.KeyFile, *cfgPath} { + _ = os.Chown(p, 0, gid) + } + } + } + ca, _ := parseCertPEM([]byte(out.CA)) + fmt.Printf("joined the region at %s as rack controller %s\n CA %s\n wrote %s and %s\nnext: sh install-rack.sh (first time) or systemctl restart rollcall-rack\n", + cfg.Region, cfg.Name, fpString(certFingerprint(ca)), *cfgPath, *dir) + return nil +} + func runRack(args []string) error { fs := newFlags("rack") cfgPath := fs.String("config", "/etc/rollcall/rack.json", "rack controller configuration") @@ -106,7 +314,15 @@ func runRack(args []string) error { if err != nil { return fmt.Errorf("region: %w", err) } - r := &rackd{cfg: cfg, region: ru, hc: &http.Client{Timeout: 45 * time.Second}, fast: &http.Client{Timeout: 30 * time.Second}, heads: map[string]headInfo{}} + r := &rackd{cfg: cfg, region: ru, heads: map[string]headInfo{}} + if err := r.clients(); err != nil { + return fmt.Errorf("tls: %w", err) + } + if r.cert != nil { + go r.renewLoop() + } else { + log.Printf("warning: using the legacy shared secret; enroll with rollcall rack join for mutual TLS") + } for _, d := range []string{filepath.Join(cfg.TFTPRoot, "pxelinux.cfg"), cfg.CacheDir, filepath.Join(cfg.DnsmasqDir, "conf.d")} { if err := os.MkdirAll(d, 0o755); err != nil { return err @@ -122,9 +338,13 @@ func runRack(args []string) error { Rewrite: func(pr *httputil.ProxyRequest) { pr.SetURL(ru) pr.SetXForwarded() + pr.Out.Header.Del("X-Rollcall-Rack-Secret") pr.Out.Header.Set("X-Rollcall-Rack", cfg.Name) - pr.Out.Header.Set("X-Rollcall-Rack-Secret", cfg.Secret) + if cfg.Secret != "" { + pr.Out.Header.Set("X-Rollcall-Rack-Secret", cfg.Secret) + } }, + Transport: r.big.Transport, FlushInterval: 100 * time.Millisecond, } for _, p := range []string{"/api/v1/commission", "/api/v1/bmc-hint", "/api/v1/jobs/self", "/api/v1/jobs/self/progress", @@ -150,7 +370,9 @@ func (r *rackd) req(ctx context.Context, method, p string, body any) (*http.Requ return nil, err } req.Header.Set("X-Rollcall-Rack", r.cfg.Name) - req.Header.Set("X-Rollcall-Rack-Secret", r.cfg.Secret) + if r.cfg.Secret != "" { + req.Header.Set("X-Rollcall-Rack-Secret", r.cfg.Secret) + } req.Header.Set("Content-Type", "application/json") return req, nil } @@ -243,19 +465,20 @@ func (r *rackd) workLoop() { go func(j rackJob) { res := rackResult{ID: j.ID} switch j.Kind { - case "ipmi": - out, err := ipmiExec(context.Background(), r.cfg.IPMITool, &bmcCreds{Address: j.Address, Port: j.Port, User: j.User, Password: j.Password}, j.Args...) + case "power": + out, err := powerRun(context.Background(), r.cfg.IPMITool, &bmcCreds{Driver: j.Driver, Address: j.Address, Port: j.Port, + User: j.User, Password: j.Password, Extra: j.Extra}, j.Op, j.Arg) res.Output = out if err != nil { res.Error = err.Error() if out != "" { - res.Error = out + res.Error = out + " (" + err.Error() + ")" } } default: - res.Error = "unknown job kind " + j.Kind + res.Error = "unknown job kind " + j.Kind + " (update this rack controller)" } - log.Printf("job %s %s %s: %s", j.ID, j.Kind, strings.Join(j.Args, " "), oneLine(res.Output+" "+res.Error)) + log.Printf("job %s %s %s %s %s: %s", j.ID, j.Kind, j.Driver, j.Op, j.Arg, oneLine(res.Output+" "+res.Error)) if _, err := r.call(r.fast, "POST", "/api/v1/rack/result", res, nil); err != nil { log.Printf("result %s: %v", j.ID, err) } @@ -318,7 +541,7 @@ func (r *rackd) fill(rel string, w http.ResponseWriter) error { if !ok { return errors.New("bad path") } - resp, err := http.Get(r.cfg.Region + rel) // no timeout: images are large + resp, err := r.big.Get(r.cfg.Region + rel) // no timeout: images are large if err != nil { return err } diff --git a/cmd/rollcall/server.go b/cmd/rollcall/server.go index c8c0b31..ee979b8 100644 --- a/cmd/rollcall/server.go +++ b/cmd/rollcall/server.go @@ -3,6 +3,8 @@ package main import ( "context" "crypto/subtle" + "crypto/tls" + "crypto/x509" "embed" "encoding/json" "errors" @@ -10,7 +12,9 @@ import ( "io" "io/fs" "log" + "net" "net/http" + "net/url" "regexp" "strings" "sync" @@ -25,6 +29,7 @@ type App struct { store *Store sealer *Sealer hub *rackHub + ca *CA netMu sync.RWMutex racks map[string]RackController @@ -59,8 +64,29 @@ func (a *App) isAdmin(r *http.Request) bool { return strings.HasPrefix(h, "Bearer ") && tokenEqual(strings.TrimPrefix(h, "Bearer "), a.cfg.AdminToken) } +// adminOverHTTP applies admin_http to requests that did not come over TLS. +func (a *App) adminOverHTTP(r *http.Request) error { + if r.TLS != nil || a.cfg.AdminHTTP == "allow" { + return nil + } + host, _, _ := net.SplitHostPort(r.RemoteAddr) + if ip := net.ParseIP(host); a.cfg.AdminHTTP == "loopback" && ip != nil && ip.IsLoopback() { + return nil + } + _, port, _ := net.SplitHostPort(a.cfg.TLSListen) + h := r.Host + if hh, _, err := net.SplitHostPort(h); err == nil { + h = hh + } + return fmt.Errorf("the admin API is HTTPS-only here: https://%s (trust the CA from /ca.pem)", net.JoinHostPort(strings.Trim(h, "[]"), port)) +} + func (a *App) adminAuth(next http.HandlerFunc) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { + if err := a.adminOverHTTP(r); err != nil { + fail(w, 403, err) + return + } if !a.isAdmin(r) { w.Header().Set("WWW-Authenticate", `Bearer realm="rollcall"`) fail(w, http.StatusUnauthorized, errors.New("admin token required")) @@ -111,6 +137,9 @@ func (a *App) routes() http.Handler { mux.HandleFunc("GET /api/v1/machines/{id}/power", a.adminAuth(a.handlePower)) mux.HandleFunc("POST /api/v1/machines/{id}/power", a.adminAuth(a.handlePower)) mux.HandleFunc("POST /api/v1/machines/{id}/bootdev", a.adminAuth(a.handleBootdev)) + mux.HandleFunc("GET /api/v1/machines/{id}/bmc", a.adminAuth(a.handleBMC)) + mux.HandleFunc("POST /api/v1/machines/{id}/bmc", a.adminAuth(a.handleBMC)) + mux.HandleFunc("GET /api/v1/power-drivers", a.adminAuth(func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, powerDrivers) })) mux.HandleFunc("GET /api/v1/whoami", a.adminAuth(func(w http.ResponseWriter, r *http.Request) { writeJSON(w, 200, map[string]any{"ok": true}) })) @@ -173,6 +202,10 @@ func (a *App) routes() http.Handler { mux.HandleFunc("GET /api/v1/controllers", a.adminAuth(a.handleControllers)) mux.HandleFunc("DELETE /api/v1/controllers/{name}", a.adminAuth(a.handleControllerDelete)) mux.HandleFunc("GET /api/v1/controllers/{name}/dhcp", a.adminAuth(a.handleRackDHCP)) + mux.HandleFunc("GET /ca.pem", a.handleCAPEM) + mux.HandleFunc("POST /api/v1/controllers/join", a.adminAuth(a.handleJoinCode)) + mux.HandleFunc("POST /api/v1/rack/enroll", a.handleRackEnroll) // join-code secret inside, TLS only + mux.HandleFunc("POST /api/v1/rack/renew", a.rackAuth(a.handleRackRenew)) mux.HandleFunc("POST /api/v1/rack/register", a.rackAuth(a.handleRackRegister)) mux.HandleFunc("GET /api/v1/rack/state", a.rackAuth(a.handleRackState)) mux.HandleFunc("GET /api/v1/rack/work", a.rackAuth(a.handleRackWork)) @@ -327,7 +360,11 @@ func summarize(m *Machine) Summary { if m.BMC != nil { var b obj if json.Unmarshal(m.BMC, &b) == nil { - s.BMC = &obj{"source": b["source"], "address": b["address"], "user": b["user"], + drv := str(b["driver"]) + if drv == "" { + drv = "ipmi" + } + s.BMC = &obj{"source": b["source"], "address": b["address"], "user": b["user"], "driver": drv, "configured": b["configured"], "verified_oob": b["verified_oob"]} } } @@ -429,6 +466,21 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) { } m.Inventory, _ = json.Marshal(inv) + // a BMC an operator configured (driver_set) survives commissioning; what the agent + // found is kept next to it for reference + if old, err := a.store.Get(r.Context(), m.ID); err == nil && old.BMC != nil { + var ob obj + if json.Unmarshal(old.BMC, &ob) == nil && ob["driver_set"] == true { + var found obj + _ = json.Unmarshal(m.BMC, &found) + delete(found, "password") + ob["commissioning"] = found + m.BMC, _ = json.Marshal(ob) + secret = old.bmcSecret + detail += " (operator BMC settings kept)" + } + } + isNew, err := a.store.UpsertCommissioned(r.Context(), m, secret) if err != nil { fail(w, 500, err) @@ -436,6 +488,9 @@ func (a *App) handleCommission(w http.ResponseWriter, r *http.Request) { } _ = a.store.AddEvent(r.Context(), m.ID, "commissioned", detail) _ = a.store.SetRackController(r.Context(), m.ID, a.fromRack(r)) + if full, err := a.store.Get(r.Context(), m.ID); err == nil { + go a.probeBMC(full) + } if full, err := a.store.Get(r.Context(), m.ID); err == nil { m = full // carries profile / node token when this is a re-commission of a stateless node } @@ -558,12 +613,11 @@ func (a *App) handlePower(w http.ResponseWriter, r *http.Request) { } action = req.Action } - args, ok := powerActions[action] - if !ok { + if !powerOps[action] || action == "bootdev" || action == "probe" { fail(w, 400, fmt.Errorf("unknown power action %q", action)) return } - out, err := a.bmc(r.Context(), m, args...) + out, err := a.power(r.Context(), m, action, "") if errors.Is(err, errNoBMC) { fail(w, 409, err) return @@ -587,7 +641,7 @@ func (a *App) handleBootdev(w http.ResponseWriter, r *http.Request) { fail(w, 400, fmt.Errorf("device must be one of pxe, disk, cdrom, bios, none, safe")) return } - out, err := a.bmc(r.Context(), m, "chassis", "bootdev", req.Device) + out, err := a.power(r.Context(), m, "bootdev", req.Device) if errors.Is(err, errNoBMC) { fail(w, 409, err) return @@ -612,13 +666,37 @@ func serve(cfgPath string) error { return fmt.Errorf("database: %w", err) } app := &App{cfg: cfg, store: store, sealer: sealer, hub: newRackHub()} + if app.ca, err = loadOrCreateCA(cfg.CADir, "rollcall CA"); err != nil { + return fmt.Errorf("certificate authority: %w", err) + } ctx2, cancel2 := context.WithTimeout(context.Background(), 30*time.Second) err = app.startRacks(ctx2) cancel2() if err != nil { return fmt.Errorf("rack controllers: %w", err) } - srv := &http.Server{Addr: cfg.Listen, Handler: app.routes(), ReadHeaderTimeout: 10 * time.Second} + handler := app.routes() + if cfg.TLSListen != "" { + names := append([]string{}, cfg.TLSNames...) + if u, err := url.Parse(cfg.ServerURL); err == nil && u.Hostname() != "" { + names = append(names, u.Hostname()) + } + pair, err := regionCert(app.ca, cfg.CADir, names) + if err != nil { + return fmt.Errorf("region certificate: %w", err) + } + pool := x509.NewCertPool() + pool.AddCert(app.ca.Cert) + tsrv := &http.Server{Addr: cfg.TLSListen, Handler: handler, ReadHeaderTimeout: 10 * time.Second, + TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{pair}, ClientCAs: pool, ClientAuth: tls.VerifyClientCertIfGiven}} + go func() { + log.Printf("rollcall HTTPS on %s (CA %s)", cfg.TLSListen, app.ca.Fingerprint()) + if err := tsrv.ListenAndServeTLS("", ""); err != nil { + log.Printf("HTTPS listener: %v", err) + } + }() + } + srv := &http.Server{Addr: cfg.Listen, Handler: handler, ReadHeaderTimeout: 10 * time.Second} log.Printf("rollcall listening on %s (image=%s tftp=%s agent_token=%v)", cfg.Listen, cfg.ImageDir, cfg.TFTPRoot, cfg.AgentToken != "") return srv.ListenAndServe() } diff --git a/cmd/rollcall/stateless.go b/cmd/rollcall/stateless.go index 2c2988f..ebb89f8 100644 --- a/cmd/rollcall/stateless.go +++ b/cmd/rollcall/stateless.go @@ -297,6 +297,7 @@ type overlayCtx struct { Nodes []NodeCtx // all machines with a profile, sorted by name Profile Profile Overlays []string // profile overlays plus those of the node's tags + CA string // the installation's CA certificate (PEM) Vars map[string]any // cluster_vars overlaid with profile vars Server string Now string @@ -466,7 +467,11 @@ func (a *App) buildCtx(ctx context.Context, m *Machine) (*overlayCtx, error) { } } } - return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, Vars: vars, Server: a.urlFor(m), + caPEM := "" + if a.ca != nil { + caPEM = string(a.ca.PEM) + } + return &overlayCtx{Node: me, Nodes: nodes, Profile: *p, Overlays: overlays, CA: caPEM, Vars: vars, Server: a.urlFor(m), Now: time.Now().Format(time.RFC3339)}, nil } @@ -786,17 +791,17 @@ func (a *App) handleAssign(w http.ResponseWriter, r *http.Request) { // pxeBoot sets bootdev pxe and power-cycles (or powers on) through the BMC. func (a *App) pxeBoot(ctx context.Context, m *Machine) (string, bool) { - o1, e1 := a.bmc(ctx, m, "chassis", "bootdev", "pxe") + o1, e1 := a.power(ctx, m, "bootdev", "pxe") if errors.Is(e1, errNoBMC) { return e1.Error(), false } - st, _ := a.bmc(ctx, m, "chassis", "power", "status") + st, _ := a.power(ctx, m, "status", "") var o2 string var e2 error if strings.Contains(st, "off") { - o2, e2 = a.bmc(ctx, m, "chassis", "power", "on") + o2, e2 = a.power(ctx, m, "on", "") } else { - o2, e2 = a.bmc(ctx, m, "chassis", "power", "cycle") + o2, e2 = a.power(ctx, m, "cycle", "") } _ = a.store.AddEvent(ctx, m.ID, "pxe boot", strings.TrimSpace(o1+"\n"+o2)) return strings.TrimSpace(o1 + "\n" + o2), e1 == nil && e2 == nil diff --git a/cmd/rollcall/store.go b/cmd/rollcall/store.go index b43617c..af1aa91 100644 --- a/cmd/rollcall/store.go +++ b/cmd/rollcall/store.go @@ -163,6 +163,15 @@ CREATE TABLE IF NOT EXISTS rack_controllers ( last_seen timestamptz, registered timestamptz NOT NULL DEFAULT now() ); +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_serial text; +ALTER TABLE rack_controllers ADD COLUMN IF NOT EXISTS cert_expires timestamptz; +CREATE TABLE IF NOT EXISTS join_tokens ( + hash text PRIMARY KEY, + name text, + expires timestamptz NOT NULL, + used timestamptz, + created timestamptz NOT NULL DEFAULT now() +); CREATE TABLE IF NOT EXISTS pxe_files ( mac text PRIMARY KEY, machine_id text, diff --git a/cmd/rollcall/tui.go b/cmd/rollcall/tui.go index fd2b9a8..4d1c984 100644 --- a/cmd/rollcall/tui.go +++ b/cmd/rollcall/tui.go @@ -8,9 +8,12 @@ package main // terminal, a bare `rollcall` prints its help as before. import ( + "crypto/tls" + "crypto/x509" "encoding/json" "errors" "fmt" + "io" "net/http" "net/url" "os" @@ -34,6 +37,7 @@ func interactive() bool { type clientConf struct { URL string `json:"url"` Token string `json:"token"` + CA string `json:"ca,omitempty"` // PEM of the region's CA, trusted when it was first seen } func clientConfPath() string { @@ -115,8 +119,11 @@ type ( tuiTickMsg struct{ gen int } tuiResultMsg struct{ title, text string } tuiConnectMsg struct { - c *client - err error + c *client + err error + caPEM string // the server's CA when it is one this computer does not know + caFP string + caBase string } ) @@ -311,6 +318,8 @@ type tuiModel struct { cTok textField cFocus int cErr string + cCA string // a CA seen on first connect, trusted on the next Enter + cCAURL string } func newTUIModel(c *client) tuiModel { @@ -419,11 +428,17 @@ func (m tuiModel) Update(msg tea.Msg) (tea.Model, tea.Cmd) { return mm, cmd case tuiConnectMsg: m.busy = "" + if msg.caPEM != "" { + m.cCA, m.cCAURL = msg.caPEM, msg.caBase + m.cErr = "This server's certificate comes from an authority this computer does not know:\n\n " + msg.caFP + + "\n\nCompare it with `rollcall ca show` on the server. Enter trusts it for this server; Esc quits." + return m, nil + } if msg.err != nil { m.cErr = msg.err.Error() return m, nil } - if err := saveClientConf(clientConf{URL: msg.c.base, Token: msg.c.token}); err != nil { + if err := saveClientConf(clientConf{URL: msg.c.base, Token: msg.c.token, CA: msg.c.ca}); err != nil { m.cErr = "connected, but could not save " + clientConfPath() + ": " + err.Error() } m.c, m.connect = msg.c, false @@ -889,7 +904,14 @@ func (m tuiModel) actionMenu(ids []string) *tuiDialog { {label: "Power cycle", run: power("cycle", true)}, {label: "Reset", run: power("reset", true)}, {label: "Power status", run: power("status", false)}, - {label: "Boot device: PXE", sep: true, run: boot("pxe")}, + {label: "BMC settings…", sep: true, run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.bmcDialog(ids) }}, + {label: "Detect Redfish", run: func(m tuiModel) (tuiModel, tea.Cmd) { + return m.run("detect Redfish", ids, apiPost("/bmc", obj{"probe": true}, func(d obj) string { + b, _ := d["bmc"].(obj) + return "driver " + orDash(str(b["driver"])) + })) + }}, + {label: "Boot device: PXE", run: boot("pxe")}, {label: "Boot device: disk", run: boot("disk")}, {label: "Deploy OS…", sep: true, run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.deployDialog(ids) }}, {label: "Assign stateless profile…", run: func(m tuiModel) (tuiModel, tea.Cmd) { return m.assignDialog(ids) }}, @@ -1066,6 +1088,66 @@ func (m tuiModel) locationDialog(ids []string) (tuiModel, tea.Cmd) { return m, nil } +func (m tuiModel) bmcDialog(ids []string) (tuiModel, tea.Cmd) { + var cur obj + if len(ids) == 1 && m.detail != nil && m.detail.Summary.ID == ids[0] { + cur = m.detail.BMC + } + var opts, vals []string + idx := 0 + for i, d := range powerDrivers { + opts = append(opts, d.Name+" "+sDim.Render(trunc(d.Description, 44))) + vals = append(vals, d.Name) + if d.Name == str(cur["driver"]) { + idx = i + } + } + var extra []string + if ex, ok := cur["extra"].(obj); ok { + for k, v := range ex { + extra = append(extra, k+"="+str(v)) + } + } + pw := textInput("Password", "", "empty keeps the stored one") + pw.text.Mask = true + fields := []dlgField{{label: "Driver", kind: fkChoice, opts: opts, vals: vals, idx: idx}} + if len(ids) == 1 { + fields = append(fields, textInput("Address", str(cur["address"]), "host, IP or https://host:port")) + } + fields = append(fields, textInput("Port", pick(num(cur["port"]) > 0, str(cur["port"]), ""), "driver default"), + textInput("User", str(cur["user"]), ""), pw, textInput("Settings", strings.Join(extra, " "), "KEY=VALUE … (system, verify_tls, cipher, node, vmid)"), + dlgField{label: "Verify first", kind: fkToggle, on: true}) + m.dlg = &tuiDialog{title: "BMC · " + m.who(ids), note: "The password is sealed on the region and travels to the rack controller over mutual TLS.", fields: fields, okLabel: "save", + onOK: func(m tuiModel, f []dlgField) (tuiModel, tea.Cmd) { + req := obj{"driver": field(f, "Driver"), "verify": field(f, "Verify first") != ""} + for _, k := range []string{"Address", "User"} { + if v := field(f, k); v != "" { + req[strings.ToLower(k)] = v + } + } + if v := field(f, "Password"); v != "" { + req["password"] = v + } + if p, err := strconv.Atoi(field(f, "Port")); err == nil { + req["port"] = p + } + if s := field(f, "Settings"); s != "" { + ex := map[string]string{} + for _, kv := range strings.Fields(s) { + k, v, _ := strings.Cut(kv, "=") + ex[k] = v + } + req["extra"] = ex + } + return m.run("BMC settings", ids, apiPost("/bmc", req, func(d obj) string { + b, _ := d["bmc"].(obj) + return "driver " + str(b["driver"]) + " " + oneLine(str(d["output"])) + })) + }} + m.dlg.setFocus(0) + return m, nil +} + func (m tuiModel) layoutDialog(ids []string) (tuiModel, tea.Cmd) { opts, vals := []string{"(from tags, else the boot NIC on its subnet)"}, []string{""} for _, l := range m.d.layouts { @@ -1116,10 +1198,21 @@ func (m tuiModel) connectKey(k tea.KeyMsg) (tea.Model, tea.Cmd) { m.cErr = "a server URL and the admin token are needed" return m, nil } - c := &client{base: u, token: strings.TrimSpace(m.cTok.Value()), http: &http.Client{Timeout: 90 * time.Second}} + ca := "" + if m.cCA != "" && m.cCAURL == u { + ca = m.cCA // the operator compared the fingerprint and pressed Enter again + } + c := &client{base: u, token: strings.TrimSpace(m.cTok.Value()), http: newHTTPClient(ca), ca: ca} m.busy, m.cErr = "connecting", "" return m, tea.Batch(m.spin.Tick, func() tea.Msg { - return tuiConnectMsg{c: c, err: c.do("GET", "/api/v1/whoami", nil, nil)} + err := c.do("GET", "/api/v1/whoami", nil, nil) + var ua x509.UnknownAuthorityError + if err != nil && ca == "" && strings.HasPrefix(u, "https://") && (errors.As(err, &ua) || strings.Contains(err.Error(), "unknown authority")) { + if pemB, fp, e := fetchCA(u); e == nil { + return tuiConnectMsg{c: c, err: err, caPEM: pemB, caFP: fp, caBase: u} + } + } + return tuiConnectMsg{c: c, err: err} }) default: if m.cFocus == 0 { @@ -1692,6 +1785,9 @@ func (m tuiModel) machinesPanel(pw int) (string, string, string, int) { dot = sOK.Render("●") } bmc = dot + " " + str(b["address"]) + if d := str(b["driver"]); d != "" && d != "ipmi" { + bmc += sDim.Render(" " + d) + } } rows = append(rows, []string{mark, sumName(s), stateCell(s), locShort(s.Location), roleOf(s), truncateWidth(strings.Join(s.Tags, ","), 22), truncateWidth(strings.TrimSpace(deref(s.Manufacturer)+" "+deref(s.Product)), 22), hwSummary(s), bmc, agoP(s.LastSeen)}) @@ -1764,7 +1860,11 @@ func (m tuiModel) detailPanel(pw int) (string, string, string, int) { if bmc["configured"] == true { dot = sOK.Render("●") } - kv("BMC", fmt.Sprintf("%s %s %s · %s", dot, str(bmc["address"]), str(bmc["user"]), pick(d.HasBMCSecret, "credentials stored", sBad.Render("no credentials")))) + drv := str(bmc["driver"]) + if drv == "" { + drv = "ipmi" + } + kv("BMC", fmt.Sprintf("%s %s %s %s · %s", dot, drv, str(bmc["address"]), str(bmc["user"]), pick(d.HasBMCSecret, "credentials stored", sBad.Render("no credentials")))) } else { kv("BMC", sBad.Render("none")) } @@ -2142,8 +2242,12 @@ func (m tuiModel) networkBody(pw int) (string, int) { if s.Size > 0 { pct = float64(s.Used) * 100 / float64(s.Size) } + size := strconv.Itoa(s.Size) + if s.Size == 0 { + size = "∞" + } rows = append(rows, []string{s.CIDR, s.Purpose, fmt.Sprintf("%s/%d", orDash(s.Fabric), s.VID), dhcp, orDash(s.Gateway), - fmt.Sprintf("%s %d/%d", bar(pct, 8), s.Used, s.Size), strings.Join(rs, " ")}) + fmt.Sprintf("%s %d/%s", bar(pct, 8), s.Used, size), strings.Join(rs, " ")}) } b.WriteString(table([]string{"Subnet", "Purpose", "Fabric/VLAN", "DHCP", "Gateway", "Used", "Ranges"}, rows, m.row, pw-2, m.inPanel)) b.WriteString("\n " + sTitle.Render("Interface layouts") + "\n") @@ -2154,3 +2258,22 @@ func (m tuiModel) networkBody(pw int) (string, int) { b.WriteString(table([]string{"Layout", "Machines", "Interfaces", "Comment"}, rows, -1, pw-2, false)) return b.String(), head } + +// fetchCA reads the region's CA without trusting it yet, for the operator to compare. +func fetchCA(base string) (string, string, error) { + hc := &http.Client{Timeout: 15 * time.Second, Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}} //nolint:gosec // shown to the operator, not trusted + resp, err := hc.Get(base + "/ca.pem") + if err != nil { + return "", "", err + } + defer resp.Body.Close() + b, err := io.ReadAll(io.LimitReader(resp.Body, 64<<10)) + if err != nil { + return "", "", err + } + c, err := parseCertPEM(b) + if err != nil { + return "", "", err + } + return string(b), fpString(certFingerprint(c)), nil +} diff --git a/cmd/rollcall/web/app.js b/cmd/rollcall/web/app.js index 9405ff5..c6cb483 100644 --- a/cmd/rollcall/web/app.js +++ b/cmd/rollcall/web/app.js @@ -25,7 +25,15 @@ async function api(method, path, body) { if (!r.ok) throw new Error(data.error || r.statusText); return data; } -function askToken() { const d = $('#login'); if (!d.open) d.showModal(); } +function askToken() { + const d = $('#login'); + if (location.protocol === 'http:') { + $('#http-warn').hidden = false; + $('#https-link').href = 'https://' + location.hostname + location.pathname + location.hash; + fetch('/ca.pem').then(r => { const fp = r.headers.get('X-Rollcall-CA-Fingerprint'); if (fp) $('#ca-fp').textContent = fp; }).catch(() => {}); + } + if (!d.open) d.showModal(); +} $('#login form').addEventListener('submit', () => { token = $('#token').value.trim(); try { localStorage.setItem('rollcall.token', token); } catch (e) { /* ignore */ } $('#token').value = ''; refresh(true); }); $('#logout').onclick = () => { token = ''; try { localStorage.removeItem('rollcall.token'); } catch (e) { /* ignore */ } askToken(); }; @@ -205,7 +213,7 @@ function pageMachines() { ${tagPills(m)} ${esc(((m.manufacturer || '') + ' ' + (m.product || '')).trim().slice(0, 26))}
${esc((m.serial || '').slice(0, 20))} ${hwSummary(m)} - ${esc(m.bmc && m.bmc.address || '—')} + ${esc(m.bmc && m.bmc.address || '—')}${m.bmc && m.bmc.driver && m.bmc.driver !== 'ipmi' ? `
${esc(m.bmc.driver)}` : ''} ${ago(m.last_seen)}`).join('')}
${pages > 1 ? ` page ${S.page + 1} / ${pages} ` : ''} per page
`; @@ -413,7 +421,10 @@ function pageMachine(d, tab) { ${card('LLDP neighbours (switch ports)', table(['If', 'Switch', 'Descr', 'Port', 'Port descr', 'VLAN'], lldpRows), true)} `; } else if (tab === 'bmc') { - body = `
${card('BMC', kv([['Source', esc(b.source)], ['Address', `${esc(b.address)}${b.port && b.port !== 623 ? ':' + esc(b.port) : ''}`], ['MAC', `${esc(b.mac)}`], ['User', esc(b.user)], ['Password', d.has_bmc_secret ? 'stored encrypted on the server' : 'none'], ['Configured', b.configured ? 'yes' : 'no'], ['Verified out-of-band', b.verified_oob === true ? 'yes' : b.verified_oob === false ? 'no' : ''], ['Channel', esc(b.channel)], ['LAN', b.lan ? `${esc(JSON.stringify(b.lan))}` : ''], ['Firmware', esc((b.mc_info || {})['Firmware Revision'])], ['Manufacturer', esc((b.mc_info || {})['Manufacturer Name'])], ['Errors', (b.errors || []).length ? `${esc(b.errors.join('; '))}` : '']]))} + body = `
Commands run on rack controller ${esc(s.rack_controller || '')}
+
${card('BMC', kv([['Driver', `${esc(b.driver || 'ipmi')}${b.driver_set ? ' (set by operator)' : ''}${b.redfish ? ` · Redfish ${esc(b.redfish.redfish)} ${esc(b.redfish.manufacturer || '')} ${esc(b.redfish.model || '')}` : ''}`], + ['Settings', b.extra && Object.keys(b.extra).length ? `${esc(Object.entries(b.extra).map(([k, v]) => k + '=' + v).join(' '))}` : ''], + ['Source', esc(b.source)], ['Address', `${esc(b.address)}${b.port && b.port !== 623 ? ':' + esc(b.port) : ''}`], ['MAC', `${esc(b.mac)}`], ['User', esc(b.user)], ['Password', d.has_bmc_secret ? 'stored encrypted on the server' : 'none'], ['Configured', b.configured ? 'yes' : 'no'], ['Verified out-of-band', b.verified_oob === true ? 'yes' : b.verified_oob === false ? 'no' : ''], ['Channel', esc(b.channel)], ['LAN', b.lan ? `${esc(JSON.stringify(b.lan))}` : ''], ['Firmware', esc((b.mc_info || {})['Firmware Revision'])], ['Manufacturer', esc((b.mc_info || {})['Manufacturer Name'])], ['Errors', (b.errors || []).length ? `${esc(b.errors.join('; '))}` : '']]))} ${(b.users_before || []).length ? card('User table before commissioning', table(['ID', 'Name', 'Priv', 'IPMI'], b.users_before.map(u => [u.id, esc(u.name), esc(u.priv), u.ipmi_msg ? 'yes' : 'no']))) : ''} ${b.fru ? card('FRU', `
${esc(JSON.stringify(b.fru, null, 1))}
`) : ''}
`; } else if (tab === 'jobs') body = jobsTable(d.jobs || [], false); @@ -445,6 +456,8 @@ function wireMachine(d) { $$('#dmore [data-item]').forEach(b => b.onclick = () => act.more(ids, b.dataset.item)); const t = $('#d-tag'); if (t) t.onclick = () => act.tag(ids); const lo = $('#d-loc'); if (lo) lo.onclick = () => act.location(ids); + const be = $('#bmc-edit'); if (be) be.onclick = () => editBMC(id, d.bmc || {}); + const bp = $('#bmc-probe'); if (bp) bp.onclick = async () => { try { const r = await api('POST', `/api/v1/machines/${id}/bmc`, { probe: true }); toast('driver: ' + ((r.bmc || {}).driver || 'ipmi')); await loadDetail(id); } catch (e) { toast(e.message, true); } }; $$('[data-untag]').forEach(a => a.onclick = ev => { ev.preventDefault(); each(ids, 'untag', async () => (await api('POST', `/api/v1/machines/${id}/tags`, { remove: [a.dataset.untag] })).effective.join(' ')); }); } @@ -605,16 +618,17 @@ function pageNetwork() { `${ago(c.last_seen)}`, `${esc((c.interfaces || []).map(i => i.name + ' ' + (i.addrs || []).join(',')).join(' · '))}`, `${c.kind === 'remote' ? ` ` : ''}`])); const sub = table(['Subnet', 'Purpose', 'Fabric / VLAN', 'DHCP', 'Gateway', 'DNS', 'Usage', 'Ranges', ''], S.subnets.map(s => { const pct = s.size ? Math.round(s.used * 100 / s.size) : 0; + const sizeTxt = s.size ? s.size : '∞'; const ranges = (s.ranges || []).map(r => `${esc(r.kind)} ${esc(r.start)}–${esc(r.end.split('.').pop())} ×`).join(' '); return [`${esc(s.cidr)}${s.name ? `
${esc(s.name)}` : ''}`, esc(s.purpose), `${esc(s.fabric || '—')} / ${s.vid || 'untagged'}${s.mtu && s.mtu !== 1500 ? ` · mtu ${s.mtu}` : ''}`, s.dhcp ? `on · ${esc(s.primary_rack)}${s.secondary_rack ? ' + ' + esc(s.secondary_rack) : ''}` : 'off', `${esc(s.gateway || '—')}`, - `${esc((s.dns || []).join(', '))}${s.domain ? `
${esc(s.domain)}` : ''}`, `
${s.used} / ${s.size}`, + `${esc((s.dns || []).join(', '))}${s.domain ? `
${esc(s.domain)}` : ''}`, `
${s.used} / ${sizeTxt}${s.family === 6 ? ' IPv6' : ''}`, ranges + ` `, ` `]; })); const lay = table(['Layout', 'Machines', 'Interfaces', 'Comment', ''], S.layouts.map(l => [`${esc(l.name)}`, l.machines || 0, `${esc(layoutSummary(l))}`, esc(l.comment || ''), ` `])); return `

Network

fabrics, VLANs and subnets (MAAS-style); rack controllers serve DHCP/PXE on their VLANs
-

Rack controllers

${ctl}

Add one next to a segment: deploy/install-rack.sh with /etc/rollcall/rack.json (region URL, rack_secret, its own URL). It registers itself.

+

Rack controllers

${ctl}

Add one next to a segment: rollcall controller join-code -name NAME -addr REGION:443 here, then on that host ./rollcall rack join CODE -url http://ITS-ADDRESS and sh install-rack.sh. It talks to the region over mutual TLS.

Subnets

${sub}

Interface layouts

${lay}

A layout applies to a machine directly (More → Network layout…) or through a tag (Tags → layout). Without one, the boot NIC gets an address on its subnet.

`; } @@ -683,3 +697,25 @@ function wireNetwork() { $$('[data-editlayout]').forEach(b => b.onclick = () => editLayout(S.layouts.find(l => l.name === b.dataset.editlayout))); $$('[data-rmlayout]').forEach(b => b.onclick = () => { if (confirm(`Delete layout ${b.dataset.rmlayout}?`)) del('/api/v1/layouts/' + b.dataset.rmlayout, 'deleted ' + b.dataset.rmlayout); }); } + +// ---------------------------------------------------------------- BMC driver settings +const powerDriverNames = [['ipmi', 'IPMI 2.0 (ipmitool)'], ['redfish', 'Redfish, any vendor'], ['idrac', 'Dell iDRAC 8/9'], ['ilo', 'HPE iLO 4/5/6'], ['supermicro', 'Supermicro (Redfish)'], ['xcc', 'Lenovo XClarity'], ['openbmc', 'OpenBMC'], ['proxmox', 'Proxmox VE VM'], ['manual', 'manual (no power control)']]; +async function editBMC(id, b) { + const v = await dialog('BMC settings', [ + { name: 'driver', label: 'Driver', type: 'select', value: b.driver || 'ipmi', options: powerDriverNames }, + { name: 'address', label: 'Address', value: b.address || '', placeholder: '10.0.0.5, bmc.example or https://host:8443' }, + { name: 'port', label: 'Port', value: b.port ? String(b.port) : '', placeholder: "driver's default" }, + { name: 'user', label: 'User', value: b.user || '', help: 'proxmox: token id, e.g. root@pam!rollcall' }, + { name: 'password', label: 'Password', placeholder: 'leave empty to keep the stored one', help: 'proxmox: the token secret' }, + { name: 'extra', label: 'Settings', type: 'textarea', rows: 3, value: Object.entries(b.extra || {}).map(([k, x]) => `${k}=${x}`).join('\n'), placeholder: 'system=/redfish/v1/Systems/1\nverify_tls=true\ncipher=17\nnode=pve1\nvmid=211', help: 'one KEY=VALUE per line; a line KEY= removes it' }, + { name: 'verify', label: 'Check with a status call before saving', type: 'checkbox', value: true }, + ], 'The password is sealed on the region and only travels to the rack controller over mutual TLS.', 'Save'); + if (!v) return; + const extra = {}; + for (const l of v.extra.split('\n').map(x => x.trim()).filter(Boolean)) { const i = l.indexOf('='); if (i > 0) extra[l.slice(0, i).trim()] = l.slice(i + 1).trim(); } + for (const k of Object.keys(b.extra || {})) if (!(k in extra)) extra[k] = ''; + const body = { driver: v.driver, address: v.address.trim(), user: v.user.trim(), extra, verify: v.verify }; + if (v.port) body.port = Number(v.port); + if (v.password) body.password = v.password; + try { const r = await api('POST', `/api/v1/machines/${id}/bmc`, body); toast(`BMC saved${r.output ? ': ' + r.output : ''}`); await loadDetail(id); refresh(); } catch (e) { toast(e.message, true); } +} diff --git a/cmd/rollcall/web/index.html b/cmd/rollcall/web/index.html index d5f7511..7718cae 100644 --- a/cmd/rollcall/web/index.html +++ b/cmd/rollcall/web/index.html @@ -31,6 +31,7 @@

Admin token

The admin_token from the server's config.json. Stored only in this browser.

+
diff --git a/deploy/config.example.json b/deploy/config.example.json index ad4bc65..07a9f60 100644 --- a/deploy/config.example.json +++ b/deploy/config.example.json @@ -1,5 +1,5 @@ { - "listen": "10.213.77.1:80", + "listen": ":80", "image_dir": "/srv/rollcall/image", "tftp_root": "/srv/tftp", "database_url": "postgres://rollcall:@127.0.0.1:5432/rollcall?sslmode=disable", @@ -13,7 +13,9 @@ "overlays_dir": "/etc/rollcall/overlays", "dnsmasq_dir": "/var/lib/rollcall/dnsmasq", "rack_name": "pxe-server", - "rack_secret": "", + "tls_listen": ":443", + "tls_names": ["rollcall.example.com"], + "admin_http": "loopback", "os_dir": "/srv/rollcall/os", "os_sources": "/etc/rollcall/os-sources.json", "node_pools": {"mgmt": "10.213.77.50-10.213.77.99"}, diff --git a/deploy/install-rack.sh b/deploy/install-rack.sh index 72a14ea..1956ffd 100644 --- a/deploy/install-rack.sh +++ b/deploy/install-rack.sh @@ -1,25 +1,31 @@ #!/bin/sh # Install a rollcall rack controller on this host (Debian/Ubuntu). Run as root -# from this directory with the rollcall binary next to it and -# /etc/rollcall/rack.json written (see rack.example.json). The region needs a -# rack_secret in its config.json; the same value goes into rack.json. +# from this directory with the rollcall binary next to it, after enrolling: +# +# region$ rollcall controller join-code -name hall-b -addr REGION:443 +# rack$ ./rollcall rack join -url http://ADDR-ON-THE-SEGMENT +# +# which writes /etc/rollcall/rack.json and the rack's certificate; from then on +# rack and region talk over mutual TLS (rack.example.json shows the fields). # # The rack controller serves DHCP/TFTP (dnsmasq) on the VLANs the region # assigns to it, keeps boot and OS images in a local cache, runs BMC commands # and passes the node API through to the region. set -eu cd "$(dirname "$0")" -[ -r /etc/rollcall/rack.json ] || { echo "write /etc/rollcall/rack.json first (see rack.example.json)" >&2; exit 1; } +[ -r /etc/rollcall/rack.json ] || { echo "enroll first: ./rollcall rack join CODE -url http://ADDR (code from: rollcall controller join-code on the region)" >&2; exit 1; } export DEBIAN_FRONTEND=noninteractive apt-get install -y -q dnsmasq ipmitool pxelinux syslinux-common syslinux-efi jq >/dev/null getent passwd rollcall >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group rollcall [ -x ./rollcall ] && install -m 0755 rollcall /usr/local/bin/rollcall chgrp rollcall /etc/rollcall/rack.json && chmod 0640 /etc/rollcall/rack.json +# certificate from rollcall rack join: readable by the service's group, the key by nobody else +if [ -d /etc/rollcall/rack ]; then chgrp -R rollcall /etc/rollcall/rack && chmod 0750 /etc/rollcall/rack && chmod 0640 /etc/rollcall/rack/*-key.pem; fi -TFTP=$(jq -r '.tftp_root // "/srv/tftp"' /etc/rollcall/rack.json) -CACHE=$(jq -r '.cache_dir // "/var/cache/rollcall"' /etc/rollcall/rack.json) -DNS=$(jq -r '.dnsmasq_dir // "/var/lib/rollcall/dnsmasq"' /etc/rollcall/rack.json) +TFTP=$(jq -r '(.tftp_root // "") | if . == "" then "/srv/tftp" else . end' /etc/rollcall/rack.json) +CACHE=$(jq -r '(.cache_dir // "") | if . == "" then "/var/cache/rollcall" else . end' /etc/rollcall/rack.json) +DNS=$(jq -r '(.dnsmasq_dir // "") | if . == "" then "/var/lib/rollcall/dnsmasq" else . end' /etc/rollcall/rack.json) # image cache: kernels/initrds are prefetched (TFTP cannot fetch on demand), the rest fills on first use install -d -o rollcall -g rollcall "$CACHE" "$CACHE/rollcall" "$CACHE/images" "$CACHE/os" diff --git a/deploy/rack.example.json b/deploy/rack.example.json index 81182ee..e20f22d 100644 --- a/deploy/rack.example.json +++ b/deploy/rack.example.json @@ -1,12 +1,15 @@ { + "_comment": "written by: rollcall rack join -url http://10.213.78.1", "name": "hall-b", - "region": "http://10.0.0.234", - "secret": "", + "region": "https://10.0.0.234:443", "url": "http://10.213.78.1", "listen": "10.213.78.1:80", "tftp_root": "/srv/tftp", "dnsmasq_dir": "/var/lib/rollcall/dnsmasq", "cache_dir": "/var/cache/rollcall", "ipmitool": "ipmitool", - "dhcp_interfaces": [] + "dhcp_interfaces": [], + "ca_file": "/etc/rollcall/rack/ca.pem", + "cert_file": "/etc/rollcall/rack/rack.pem", + "key_file": "/etc/rollcall/rack/rack-key.pem" } diff --git a/lab/pve-vbmc/install.sh b/lab/pve-vbmc/install.sh index 54ce620..994bf6a 100644 --- a/lab/pve-vbmc/install.sh +++ b/lab/pve-vbmc/install.sh @@ -5,8 +5,9 @@ set -eu cd "$(dirname "$0")" getent passwd pve-vbmc >/dev/null || useradd --system --no-create-home --shell /usr/sbin/nologin --user-group pve-vbmc install -m 0755 pve_vbmc.py /usr/local/bin/pve_vbmc.py +install -m 0755 pve_redfish.py /usr/local/bin/pve_redfish.py install -m 0755 pve-vbmc-addrs.sh /usr/local/bin/pve-vbmc-addrs.sh -install -m 0644 pve-vbmc@.service pve-vbmc-addrs.service /etc/systemd/system/ +install -m 0644 pve-vbmc@.service pve-redfish@.service pve-vbmc-addrs.service /etc/systemd/system/ chgrp pve-vbmc /etc/pve-vbmc/config.json && chmod 0640 /etc/pve-vbmc/config.json systemctl daemon-reload systemctl enable --now pve-vbmc-addrs.service @@ -16,4 +17,9 @@ systemctl enable --now pve-vbmc-addrs.service for v in $(python3 -c 'import json;print(" ".join(str(b["vmid"]) for b in json.load(open("/etc/pve-vbmc/config.json"))["bmcs"]))'); do systemctl enable --now "pve-vbmc@$v.service" done -systemctl --no-pager --type=service | grep pve-vbmc +# BMC entries with "redfish": "idrac" | "ilo" | "generic" also get a Redfish service on port 443 +for v in $(python3 -c 'import json;print(" ".join(str(b["vmid"]) for b in json.load(open("/etc/pve-vbmc/config.json"))["bmcs"] if b.get("redfish")))'); do + systemctl enable --now "pve-redfish@$v.service" + systemctl restart "pve-redfish@$v.service" +done +systemctl --no-pager --type=service | grep -E "pve-vbmc|pve-redfish" diff --git a/lab/pve-vbmc/pve-redfish@.service b/lab/pve-vbmc/pve-redfish@.service new file mode 100644 index 0000000..432308b --- /dev/null +++ b/lab/pve-vbmc/pve-redfish@.service @@ -0,0 +1,21 @@ +[Unit] +Description=pve-redfish Redfish service for Proxmox VM %i (lab BMC driver tests) +After=network-online.target pve-vbmc-addrs.service +Wants=network-online.target +Requires=pve-vbmc-addrs.service + +[Service] +Type=simple +ExecStart=/usr/local/bin/pve_redfish.py --config /etc/pve-vbmc/config.json --vmid %i +Restart=always +RestartSec=3 +AmbientCapabilities=CAP_NET_BIND_SERVICE +User=pve-vbmc +Group=pve-vbmc +NoNewPrivileges=yes +ProtectSystem=strict +ProtectHome=yes +PrivateTmp=yes + +[Install] +WantedBy=multi-user.target diff --git a/lab/pve-vbmc/pve_redfish.py b/lab/pve-vbmc/pve_redfish.py new file mode 100644 index 0000000..5c87b05 --- /dev/null +++ b/lab/pve-vbmc/pve_redfish.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +"""pve-redfish: a small Redfish service for one Proxmox VM, for testing BMC drivers. + +It answers on the same address as the VM's pve-vbmc (port 443 instead of UDP +623) and shapes itself like a vendor so rollcall's Redfish probe and the +idrac / ilo / generic drivers see what real hardware shows: + + idrac Vendor "Dell", system /redfish/v1/Systems/System.Embedded.1, PowerCycle + ilo Vendor "HPE", system /redfish/v1/Systems/1, no PowerCycle, PushPowerButton + generic no vendor, system /redfish/v1/Systems/1, AllowableValues via @Redfish.ActionInfo + +Config: the pve-vbmc config.json; a bmcs entry may carry "redfish": "idrac"|"ilo"|"generic". +Basic auth with the same user/password as the IPMI side. Self-signed TLS. +""" +import argparse +import base64 +import ipaddress +import json +import logging +import os +import socket +import ssl +import subprocess +import sys +import tempfile +import threading +import time +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer + +import requests +import urllib3 + +log = logging.getLogger("pve-redfish") + +PROFILES = { + "idrac": {"vendor": "Dell", "manufacturer": "Dell Inc.", "model": "PowerEdge R760 (pve)", "id": "System.Embedded.1", + "resets": ["On", "ForceOff", "ForceRestart", "GracefulRestart", "GracefulShutdown", "PushPowerButton", "Nmi", "PowerCycle"], + "actioninfo": False, "etag": True}, + "ilo": {"vendor": "HPE", "manufacturer": "HPE", "model": "ProLiant DL380 Gen11 (pve)", "id": "1", + "resets": ["On", "ForceOff", "ForceRestart", "Nmi", "PushPowerButton"], "actioninfo": False, "etag": False}, + "generic": {"vendor": "", "manufacturer": "QEMU", "model": "Standard PC (pve)", "id": "1", + "resets": ["On", "ForceOff", "GracefulShutdown", "ForceRestart"], "actioninfo": True, "etag": False}, +} + + +class Pve: + def __init__(self, cfg, vmid): + self.api, self.node, self.vmid = cfg["api"].rstrip("/"), cfg["node"], int(vmid) + self.http = requests.Session() + self.http.headers["Authorization"] = "PVEAPIToken=" + cfg["token"] + self.http.verify = cfg.get("verify_tls", False) + if not self.http.verify: + urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) + + def _url(self, path): + return f"{self.api}/api2/json/nodes/{self.node}/qemu/{self.vmid}/{path}" + + def get(self, path): + r = self.http.get(self._url(path), timeout=10) + r.raise_for_status() + return r.json()["data"] + + def post(self, path, **data): + r = self.http.post(self._url(path), data=data, timeout=30) + r.raise_for_status() + return r.json()["data"] + + def put(self, path, **data): + r = self.http.put(self._url(path), data=data, timeout=30) + r.raise_for_status() + + def running(self): + return self.get("status/current")["status"] == "running" + + def boot(self, target): + cfg = self.get("config") + disks = [k for k in ("scsi0", "virtio0", "sata0", "ide0") if k in cfg] + cds = sorted(k for k, v in cfg.items() if "media=cdrom" in str(v)) + nets = sorted(k for k in cfg if k.startswith("net") and k[3:].isdigit()) + order = {"Pxe": nets + disks + cds, "Cd": cds + disks + nets}.get(target, disks + cds + nets) + self.put("config", boot="order=" + ";".join(order)) + return order + + +def make_handler(pve, prof, user, password): + sysid = prof["id"] + syspath = f"/redfish/v1/Systems/{sysid}" + state = {"override": "None", "enabled": "Disabled", "etag": 1} + want = "Basic " + base64.b64encode(f"{user}:{password}".encode()).decode() + + class H(BaseHTTPRequestHandler): + server_version = "pve-redfish" + + def log_message(self, fmt, *args): + log.info("%s %s", self.address_string(), fmt % args) + + def send(self, code, body=None, headers=None): + data = json.dumps(body).encode() if body is not None else b"" + self.send_response(code) + self.send_header("Content-Type", "application/json") + self.send_header("OData-Version", "4.0") + for k, v in (headers or {}).items(): + self.send_header(k, v) + self.send_header("Content-Length", str(len(data))) + self.end_headers() + self.wfile.write(data) + + def error(self, code, msg): + self.send(code, {"error": {"code": "Base.1.0.GeneralError", "message": msg, + "@Message.ExtendedInfo": [{"Message": msg}]}}) + + def authed(self): + if self.path.rstrip("/") == "/redfish/v1" or self.path.rstrip("/") == "/redfish": + return True + if self.headers.get("Authorization") != want: + self.error(401, "authentication required") + return False + return True + + def body(self): + n = int(self.headers.get("Content-Length") or 0) + return json.loads(self.rfile.read(n) or b"{}") + + def system(self): + reset = {"target": syspath + "/Actions/ComputerSystem.Reset"} + if prof["actioninfo"]: + reset["@Redfish.ActionInfo"] = syspath + "/ResetActionInfo" + else: + reset["ResetType@Redfish.AllowableValues"] = prof["resets"] + return {"@odata.id": syspath, "Id": sysid, "Name": "System", "Manufacturer": prof["manufacturer"], "Model": prof["model"], + "PowerState": "On" if pve.running() else "Off", + "Boot": {"BootSourceOverrideTarget": state["override"], "BootSourceOverrideEnabled": state["enabled"], + "BootSourceOverrideTarget@Redfish.AllowableValues": ["None", "Pxe", "Hdd", "Cd", "BiosSetup"]}, + "Actions": {"#ComputerSystem.Reset": reset}} + + def do_GET(self): + if not self.authed(): + return + p = self.path.split("?")[0].rstrip("/") + if p in ("/redfish/v1", "/redfish"): + root = {"@odata.id": "/redfish/v1/", "RedfishVersion": "1.15.0", "Systems": {"@odata.id": "/redfish/v1/Systems"}} + if prof["vendor"]: + root["Vendor"] = prof["vendor"] + root["Oem"] = {prof["vendor"]: {}} + return self.send(200, root) + if p == "/redfish/v1/Systems": + return self.send(200, {"Members": [{"@odata.id": syspath}], "Members@odata.count": 1}) + if p == syspath: + hdr = {"ETag": f'W/"{state["etag"]}"'} if prof["etag"] else None + return self.send(200, self.system(), hdr) + if p == syspath + "/ResetActionInfo": + return self.send(200, {"Parameters": [{"Name": "ResetType", "Required": True, "AllowableValues": prof["resets"]}]}) + self.error(404, "not found") + + def do_PATCH(self): + if not self.authed(): + return + if self.path.rstrip("/") != syspath: + return self.error(404, "not found") + if prof["etag"] and self.headers.get("If-Match") not in (f'W/"{state["etag"]}"', "*"): + return self.error(412, "If-Match does not match the current ETag") + boot = self.body().get("Boot", {}) + if "BootSourceOverrideEnabled" in boot: + state["enabled"] = boot["BootSourceOverrideEnabled"] + if "BootSourceOverrideTarget" in boot: + state["override"] = boot["BootSourceOverrideTarget"] + if state["enabled"] != "Disabled": + order = pve.boot(state["override"]) + log.info("boot override %s -> order %s", state["override"], order) + state["etag"] += 1 + self.send(204) + + def do_POST(self): + if not self.authed(): + return + if self.path.rstrip("/") != syspath + "/Actions/ComputerSystem.Reset": + return self.error(404, "not found") + t = self.body().get("ResetType", "") + if t not in prof["resets"]: + return self.error(400, f"ResetType {t} is not supported") + on = pve.running() + log.info("reset %s (was %s)", t, "on" if on else "off") + if t == "On" or (t == "PushPowerButton" and not on): + if not on: + pve.post("status/start") + elif t == "ForceOff": + if on: + pve.post("status/stop") + elif t in ("GracefulShutdown",) or (t == "PushPowerButton" and on): + pve.post("status/shutdown", forceStop=1, timeout=60) + elif t in ("ForceRestart", "GracefulRestart", "PowerCycle"): + pve.post("status/reset" if on else "status/start") + self.send(204) + + return H + + +def self_signed(addr): + d = tempfile.mkdtemp() + crt, key = os.path.join(d, "c.pem"), os.path.join(d, "k.pem") + subprocess.run(["openssl", "req", "-x509", "-newkey", "ec", "-pkeyopt", "ec_paramgen_curve:prime256v1", "-nodes", "-days", "3650", + "-subj", "/CN=pve-redfish", "-addext", f"subjectAltName=IP:{addr}", "-keyout", key, "-out", crt], + check=True, capture_output=True) + return crt, key + + +class Server6(ThreadingHTTPServer): + address_family = socket.AF_INET6 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--config", default="/etc/pve-vbmc/config.json") + ap.add_argument("--vmid", type=int, required=True) + args = ap.parse_args() + logging.basicConfig(level=logging.INFO, format="%(asctime)s %(name)s %(levelname)s %(message)s") + cfg = json.load(open(args.config)) + entry = next((b for b in cfg["bmcs"] if int(b["vmid"]) == args.vmid), None) + if entry is None: + log.error("vmid %s not in config", args.vmid) + sys.exit(2) + prof = PROFILES[entry.get("redfish", "generic")] + addr = entry["address"] + user = entry.get("user", cfg.get("user", "admin")) + password = entry.get("password", cfg.get("password", "password")) + handler = make_handler(Pve(cfg, args.vmid), prof, user, password) + ip = ipaddress.ip_address(addr) + cls = Server6 if ip.version == 6 else ThreadingHTTPServer + srv = cls((addr, int(entry.get("redfish_port", 443))), handler) + crt, key = self_signed(addr) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(crt, key) + srv.socket = ctx.wrap_socket(srv.socket, server_side=True) + log.info("Redfish (%s profile) for VM %s on https://%s:%s", entry.get("redfish", "generic"), args.vmid, addr, entry.get("redfish_port", 443)) + srv.serve_forever() + + +if __name__ == "__main__": + main()