Files
ilkerandClaude Opus 5.5 772579648c hpcod: one binary installs, checks and tests the control plane
An installation used to be this one: the deploy scripts and the Ansible
defaults carried one site's domain, hypervisor and login, the Proxmox side
and the database were set up by hand, and deploying meant copying scripts
and a tree of roles beside the binary and keeping them in step with it.

The binary now carries the roles, the Terraform module, the schema and the
catalogue (embed.go), and installs itself:

- hpcod setup: a full-screen installer for administrators. It asks where
  clusters are built, signs in to Proxmox before asking anything else
  (root's password once, or tokens), offers nodes and datastores from what
  Proxmox has, checks every field as it is typed, runs every step with its
  output live, checks the result and offers a test cluster. -yes runs the
  same steps from a site file; make deploy uses that.
- hpcod verify: each fact checked by using it.
- hpcod selftest: the smallest real cluster, by the portal's own path, a
  job run on it, then taken down.
- Closed networks: hpcod serves HTTPS itself from a CA name-constrained to
  the site's domains, issuing and renewing its own certificate, and
  answers DNS for the portal and every cluster name.
- A simulated Proxmox: Proxmox VE in QEMU on the control plane host, on a
  private bridge with NAT, for trying it without one.

Every step reads back what it changed and is safe to run again. The
site's values live in one file; group_vars/all/site.yml is generated from
it, and the playbooks refuse to run without it. ci_user and -ssh-user no
longer default to a personal account. scripts/B10, B20 and C20 are gone:
their work is the setup steps.

Proven end to end on a simulated Proxmox (Proxmox VE 9 in a VM on a real
host): the install, the Proxmox step by the password path, the head
template, verify READY, and a test cluster that ran srun on its compute
node and was destroyed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 18:20:04 +01:00

203 lines
6.7 KiB
Go

package setup
import (
"context"
"fmt"
"os"
"strings"
"time"
"golang.org/x/crypto/ssh"
)
const templateRevisionFile = VarDir + "/head-template-revision"
// TemplateState says whether the head template exists and whether it was
// built from the roles this host now has.
type TemplateState struct {
Exists bool
// Baked and Deployed are the "revision:" lines of the template's record
// and of the roles on disk.
Baked, Deployed string
}
// Current reports whether the template exists and has the deployed roles.
func (t TemplateState) Current() bool { return t.Exists && t.Baked != "" && t.Baked == t.Deployed }
// Template reads the template's state from Proxmox and the two records.
func (e *Env) Template(ctx context.Context) (TemplateState, error) {
var t TemplateState
t.Baked = revisionLine(e.P(templateRevisionFile))
t.Deployed = revisionLine(e.P(VarDir + "/ansible/REVISION"))
c, err := e.PVESSH(ctx)
if err != nil {
return t, err
}
defer c.Close()
out, _ := sshRun(c, "qm config "+e.Site["HEAD_TEMPLATE_VMID"]+" 2>/dev/null")
t.Exists = strings.Contains(out, "\ntemplate: 1") || strings.HasPrefix(out, "template: 1")
return t, nil
}
func revisionLine(p string) string {
raw, err := os.ReadFile(p)
if err != nil {
return ""
}
for _, l := range strings.Split(string(raw), "\n") {
if v, ok := strings.CutPrefix(l, "revision:"); ok {
return strings.TrimSpace(v)
}
}
return ""
}
// stepTemplate builds the head node template when there is none, or when
// asked to. A stale one is reported rather than rebuilt unasked: the build
// takes a quarter of an hour, and replaces the template every new cluster
// is cloned from.
func stepTemplate(ctx context.Context, e *Env) error {
t, err := e.Template(ctx)
if err != nil {
return fmt.Errorf("cannot reach Proxmox to look for the template: %w", err)
}
switch {
case t.Current() && !e.RebuildTemplate:
e.Logf("template %s is current (%s)", e.Site["HEAD_TEMPLATE_VMID"], t.Baked)
return nil
case t.Exists && !e.RebuildTemplate:
e.Logf("!! template %s was built from %q, the roles here are %q.", e.Site["HEAD_TEMPLATE_VMID"], t.Baked, t.Deployed)
e.Logf("!! New clusters get the old roles until it is rebuilt: hpcod setup --rebuild-template")
return nil
}
return e.buildTemplate(ctx)
}
// buildTemplate boots a VM from the cloud image, installs the head node
// stack into it with the deployed roles, and converts it to the template.
// The template carries no MUNGE key, SSH host keys or machine id: a shared
// MUNGE key would let any tenant authenticate to any other tenant's Slurm,
// and the first-boot playbook regenerates all three.
func (e *Env) buildTemplate(ctx context.Context) error {
s := e.Site
vmid, ip, user := s["HEAD_TEMPLATE_VMID"], s["BUILD_IP"], s["CI_USER"]
ansibleDir := VarDir + "/ansible"
// The roles carry the site's values into every head built from this.
// Baking without them is a template whose clusters fail at first boot;
// golden.yml refuses too, but only after a VM exists.
if got, err := os.ReadFile(e.P(ansibleDir + "/group_vars/all/site.yml")); err != nil || string(got) != SiteYAML(s) {
return fmt.Errorf("the deployed roles do not carry this site's values; run the tree step first")
}
pub, err := e.PublicKey()
if err != nil {
return err
}
pve, err := e.PVESSH(ctx)
if err != nil {
return err
}
defer pve.Close()
q := func(cmd string) error {
_, err := sshRun(pve, cmd)
return err
}
e.Logf("removing any previous VM %s", vmid)
_, _ = sshRun(pve, fmt.Sprintf("qm stop %[1]s >/dev/null 2>&1; qm destroy %[1]s --purge >/dev/null 2>&1; true", vmid))
if out, _ := sshRun(pve, "qm config "+vmid+" 2>/dev/null"); strings.TrimSpace(out) != "" {
return fmt.Errorf("VM %s still exists and could not be removed (is it a template with linked clones?)", vmid)
}
e.Logf("creating build VM %s at %s", vmid, ip)
err = q(fmt.Sprintf(`set -e
printf '%%s\n' '%[8]s' > /tmp/hpcod-golden.pub
qm create %[1]s --name hpcod-head-build --memory 4096 --cores 2 --cpu host \
--net0 virtio,bridge=%[2]s --scsihw virtio-scsi-single --ostype l26 --agent 1 --serial0 socket
qm set %[1]s --scsi0 %[3]s:0,import-from=%[4]s,discard=on,ssd=1 >/dev/null
qm resize %[1]s scsi0 40G >/dev/null
qm set %[1]s --ide2 %[3]s:cloudinit --boot order=scsi0 >/dev/null
qm set %[1]s --ciuser %[5]s --sshkeys /tmp/hpcod-golden.pub \
--ipconfig0 ip=%[6]s/24,gw=%[7]s --nameserver '%[9]s' --ciupgrade 0 >/dev/null
rm -f /tmp/hpcod-golden.pub
qm start %[1]s`, vmid, s["MGMT_BRIDGE"], s["PVE_DATASTORE"], s["PVE_IMAGE"], user, ip, s["MGMT_GATEWAY"], pub, s["NAMESERVERS"]))
if err != nil {
return err
}
e.Logf("waiting for SSH on %s", ip)
signer, err := e.Signer()
if err != nil {
return err
}
deadline := time.Now().Add(6 * time.Minute)
for {
c, err := dialSSH(ctx, ip, user, ssh.PublicKeys(signer))
if err == nil {
c.Close()
break
}
if time.Now().After(deadline) {
return fmt.Errorf("the build VM never answered SSH at %s as %s: %w", ip, user, err)
}
select {
case <-ctx.Done():
return ctx.Err()
case <-time.After(5 * time.Second):
}
}
e.Logf("installing the head node stack (about ten minutes)")
inv, err := os.CreateTemp(e.P(VarDir), ".golden-inv-*.yml")
if err != nil {
return err
}
defer os.Remove(inv.Name())
fmt.Fprintf(inv, `all:
children:
golden:
hosts:
hpcod-head-build:
ansible_host: %s
ansible_user: %s
ansible_become: true
ansible_ssh_private_key_file: %s
`, ip, user, KeyPath)
inv.Close()
// ANSIBLE_CONFIG explicitly: Ansible finds ansible.cfg relative to the
// working directory, and silently uses stock defaults when it does not.
err = e.stream(ctx, []string{"ANSIBLE_CONFIG=" + ansibleDir + "/ansible.cfg", "ANSIBLE_FORCE_COLOR=0"},
ansibleDir, "ansible-playbook", "-i", inv.Name(), "golden.yml", "--tags", "install")
if err != nil {
return fmt.Errorf("the head node playbook failed; VM %s is left running at %s to look at: %w", vmid, ip, err)
}
e.Logf("converting VM %s to a template", vmid)
if err := q(fmt.Sprintf(`set -e
qm shutdown %[1]s --timeout 120 || qm stop %[1]s
sleep 3
qm set %[1]s --name hpcod-head >/dev/null
qm template %[1]s`, vmid)); err != nil {
return err
}
// Recorded, so a role change that reaches the host but not the
// template is noticed instead of shipping clusters with old roles.
rev, err := os.ReadFile(e.P(ansibleDir + "/REVISION"))
if err != nil {
return err
}
if err := e.writeFile(templateRevisionFile, rev, 0o644, owner); err != nil {
return err
}
t, err := e.Template(ctx)
if err != nil {
return err
}
if !t.Current() {
return fmt.Errorf("built, but Proxmox does not list VM %s as a template", vmid)
}
e.Logf("template %s ready (%s)", vmid, t.Baked)
return nil
}