An installation used to be this one: the deploy scripts and the Ansible defaults carried one site's domain, hypervisor and login, the Proxmox side and the database were set up by hand, and deploying meant copying scripts and a tree of roles beside the binary and keeping them in step with it. The binary now carries the roles, the Terraform module, the schema and the catalogue (embed.go), and installs itself: - hpcod setup: a full-screen installer for administrators. It asks where clusters are built, signs in to Proxmox before asking anything else (root's password once, or tokens), offers nodes and datastores from what Proxmox has, checks every field as it is typed, runs every step with its output live, checks the result and offers a test cluster. -yes runs the same steps from a site file; make deploy uses that. - hpcod verify: each fact checked by using it. - hpcod selftest: the smallest real cluster, by the portal's own path, a job run on it, then taken down. - Closed networks: hpcod serves HTTPS itself from a CA name-constrained to the site's domains, issuing and renewing its own certificate, and answers DNS for the portal and every cluster name. - A simulated Proxmox: Proxmox VE in QEMU on the control plane host, on a private bridge with NAT, for trying it without one. Every step reads back what it changed and is safe to run again. The site's values live in one file; group_vars/all/site.yml is generated from it, and the playbooks refuse to run without it. ci_user and -ssh-user no longer default to a personal account. scripts/B10, B20 and C20 are gone: their work is the setup steps. Proven end to end on a simulated Proxmox (Proxmox VE 9 in a VM on a real host): the install, the Proxmox step by the password path, the head template, verify READY, and a test cluster that ran srun on its compute node and was destroyed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
203 lines
6.7 KiB
Go
203 lines
6.7 KiB
Go
package setup
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"golang.org/x/crypto/ssh"
|
|
)
|
|
|
|
const templateRevisionFile = VarDir + "/head-template-revision"
|
|
|
|
// TemplateState says whether the head template exists and whether it was
|
|
// built from the roles this host now has.
|
|
type TemplateState struct {
|
|
Exists bool
|
|
// Baked and Deployed are the "revision:" lines of the template's record
|
|
// and of the roles on disk.
|
|
Baked, Deployed string
|
|
}
|
|
|
|
// Current reports whether the template exists and has the deployed roles.
|
|
func (t TemplateState) Current() bool { return t.Exists && t.Baked != "" && t.Baked == t.Deployed }
|
|
|
|
// Template reads the template's state from Proxmox and the two records.
|
|
func (e *Env) Template(ctx context.Context) (TemplateState, error) {
|
|
var t TemplateState
|
|
t.Baked = revisionLine(e.P(templateRevisionFile))
|
|
t.Deployed = revisionLine(e.P(VarDir + "/ansible/REVISION"))
|
|
c, err := e.PVESSH(ctx)
|
|
if err != nil {
|
|
return t, err
|
|
}
|
|
defer c.Close()
|
|
out, _ := sshRun(c, "qm config "+e.Site["HEAD_TEMPLATE_VMID"]+" 2>/dev/null")
|
|
t.Exists = strings.Contains(out, "\ntemplate: 1") || strings.HasPrefix(out, "template: 1")
|
|
return t, nil
|
|
}
|
|
|
|
func revisionLine(p string) string {
|
|
raw, err := os.ReadFile(p)
|
|
if err != nil {
|
|
return ""
|
|
}
|
|
for _, l := range strings.Split(string(raw), "\n") {
|
|
if v, ok := strings.CutPrefix(l, "revision:"); ok {
|
|
return strings.TrimSpace(v)
|
|
}
|
|
}
|
|
return ""
|
|
}
|
|
|
|
// stepTemplate builds the head node template when there is none, or when
|
|
// asked to. A stale one is reported rather than rebuilt unasked: the build
|
|
// takes a quarter of an hour, and replaces the template every new cluster
|
|
// is cloned from.
|
|
func stepTemplate(ctx context.Context, e *Env) error {
|
|
t, err := e.Template(ctx)
|
|
if err != nil {
|
|
return fmt.Errorf("cannot reach Proxmox to look for the template: %w", err)
|
|
}
|
|
switch {
|
|
case t.Current() && !e.RebuildTemplate:
|
|
e.Logf("template %s is current (%s)", e.Site["HEAD_TEMPLATE_VMID"], t.Baked)
|
|
return nil
|
|
case t.Exists && !e.RebuildTemplate:
|
|
e.Logf("!! template %s was built from %q, the roles here are %q.", e.Site["HEAD_TEMPLATE_VMID"], t.Baked, t.Deployed)
|
|
e.Logf("!! New clusters get the old roles until it is rebuilt: hpcod setup --rebuild-template")
|
|
return nil
|
|
}
|
|
return e.buildTemplate(ctx)
|
|
}
|
|
|
|
// buildTemplate boots a VM from the cloud image, installs the head node
|
|
// stack into it with the deployed roles, and converts it to the template.
|
|
// The template carries no MUNGE key, SSH host keys or machine id: a shared
|
|
// MUNGE key would let any tenant authenticate to any other tenant's Slurm,
|
|
// and the first-boot playbook regenerates all three.
|
|
func (e *Env) buildTemplate(ctx context.Context) error {
|
|
s := e.Site
|
|
vmid, ip, user := s["HEAD_TEMPLATE_VMID"], s["BUILD_IP"], s["CI_USER"]
|
|
ansibleDir := VarDir + "/ansible"
|
|
|
|
// The roles carry the site's values into every head built from this.
|
|
// Baking without them is a template whose clusters fail at first boot;
|
|
// golden.yml refuses too, but only after a VM exists.
|
|
if got, err := os.ReadFile(e.P(ansibleDir + "/group_vars/all/site.yml")); err != nil || string(got) != SiteYAML(s) {
|
|
return fmt.Errorf("the deployed roles do not carry this site's values; run the tree step first")
|
|
}
|
|
pub, err := e.PublicKey()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
pve, err := e.PVESSH(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer pve.Close()
|
|
q := func(cmd string) error {
|
|
_, err := sshRun(pve, cmd)
|
|
return err
|
|
}
|
|
|
|
e.Logf("removing any previous VM %s", vmid)
|
|
_, _ = sshRun(pve, fmt.Sprintf("qm stop %[1]s >/dev/null 2>&1; qm destroy %[1]s --purge >/dev/null 2>&1; true", vmid))
|
|
if out, _ := sshRun(pve, "qm config "+vmid+" 2>/dev/null"); strings.TrimSpace(out) != "" {
|
|
return fmt.Errorf("VM %s still exists and could not be removed (is it a template with linked clones?)", vmid)
|
|
}
|
|
|
|
e.Logf("creating build VM %s at %s", vmid, ip)
|
|
err = q(fmt.Sprintf(`set -e
|
|
printf '%%s\n' '%[8]s' > /tmp/hpcod-golden.pub
|
|
qm create %[1]s --name hpcod-head-build --memory 4096 --cores 2 --cpu host \
|
|
--net0 virtio,bridge=%[2]s --scsihw virtio-scsi-single --ostype l26 --agent 1 --serial0 socket
|
|
qm set %[1]s --scsi0 %[3]s:0,import-from=%[4]s,discard=on,ssd=1 >/dev/null
|
|
qm resize %[1]s scsi0 40G >/dev/null
|
|
qm set %[1]s --ide2 %[3]s:cloudinit --boot order=scsi0 >/dev/null
|
|
qm set %[1]s --ciuser %[5]s --sshkeys /tmp/hpcod-golden.pub \
|
|
--ipconfig0 ip=%[6]s/24,gw=%[7]s --nameserver '%[9]s' --ciupgrade 0 >/dev/null
|
|
rm -f /tmp/hpcod-golden.pub
|
|
qm start %[1]s`, vmid, s["MGMT_BRIDGE"], s["PVE_DATASTORE"], s["PVE_IMAGE"], user, ip, s["MGMT_GATEWAY"], pub, s["NAMESERVERS"]))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
e.Logf("waiting for SSH on %s", ip)
|
|
signer, err := e.Signer()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
deadline := time.Now().Add(6 * time.Minute)
|
|
for {
|
|
c, err := dialSSH(ctx, ip, user, ssh.PublicKeys(signer))
|
|
if err == nil {
|
|
c.Close()
|
|
break
|
|
}
|
|
if time.Now().After(deadline) {
|
|
return fmt.Errorf("the build VM never answered SSH at %s as %s: %w", ip, user, err)
|
|
}
|
|
select {
|
|
case <-ctx.Done():
|
|
return ctx.Err()
|
|
case <-time.After(5 * time.Second):
|
|
}
|
|
}
|
|
|
|
e.Logf("installing the head node stack (about ten minutes)")
|
|
inv, err := os.CreateTemp(e.P(VarDir), ".golden-inv-*.yml")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer os.Remove(inv.Name())
|
|
fmt.Fprintf(inv, `all:
|
|
children:
|
|
golden:
|
|
hosts:
|
|
hpcod-head-build:
|
|
ansible_host: %s
|
|
ansible_user: %s
|
|
ansible_become: true
|
|
ansible_ssh_private_key_file: %s
|
|
`, ip, user, KeyPath)
|
|
inv.Close()
|
|
// ANSIBLE_CONFIG explicitly: Ansible finds ansible.cfg relative to the
|
|
// working directory, and silently uses stock defaults when it does not.
|
|
err = e.stream(ctx, []string{"ANSIBLE_CONFIG=" + ansibleDir + "/ansible.cfg", "ANSIBLE_FORCE_COLOR=0"},
|
|
ansibleDir, "ansible-playbook", "-i", inv.Name(), "golden.yml", "--tags", "install")
|
|
if err != nil {
|
|
return fmt.Errorf("the head node playbook failed; VM %s is left running at %s to look at: %w", vmid, ip, err)
|
|
}
|
|
|
|
e.Logf("converting VM %s to a template", vmid)
|
|
if err := q(fmt.Sprintf(`set -e
|
|
qm shutdown %[1]s --timeout 120 || qm stop %[1]s
|
|
sleep 3
|
|
qm set %[1]s --name hpcod-head >/dev/null
|
|
qm template %[1]s`, vmid)); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Recorded, so a role change that reaches the host but not the
|
|
// template is noticed instead of shipping clusters with old roles.
|
|
rev, err := os.ReadFile(e.P(ansibleDir + "/REVISION"))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(templateRevisionFile, rev, 0o644, owner); err != nil {
|
|
return err
|
|
}
|
|
t, err := e.Template(ctx)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !t.Current() {
|
|
return fmt.Errorf("built, but Proxmox does not list VM %s as a template", vmid)
|
|
}
|
|
e.Logf("template %s ready (%s)", vmid, t.Baked)
|
|
return nil
|
|
}
|