An installation used to be this one: the deploy scripts and the Ansible defaults carried one site's domain, hypervisor and login, the Proxmox side and the database were set up by hand, and deploying meant copying scripts and a tree of roles beside the binary and keeping them in step with it. The binary now carries the roles, the Terraform module, the schema and the catalogue (embed.go), and installs itself: - hpcod setup: a full-screen installer for administrators. It asks where clusters are built, signs in to Proxmox before asking anything else (root's password once, or tokens), offers nodes and datastores from what Proxmox has, checks every field as it is typed, runs every step with its output live, checks the result and offers a test cluster. -yes runs the same steps from a site file; make deploy uses that. - hpcod verify: each fact checked by using it. - hpcod selftest: the smallest real cluster, by the portal's own path, a job run on it, then taken down. - Closed networks: hpcod serves HTTPS itself from a CA name-constrained to the site's domains, issuing and renewing its own certificate, and answers DNS for the portal and every cluster name. - A simulated Proxmox: Proxmox VE in QEMU on the control plane host, on a private bridge with NAT, for trying it without one. Every step reads back what it changed and is safe to run again. The site's values live in one file; group_vars/all/site.yml is generated from it, and the playbooks refuse to run without it. ci_user and -ssh-user no longer default to a personal account. scripts/B10, B20 and C20 are gone: their work is the setup steps. Proven end to end on a simulated Proxmox (Proxmox VE 9 in a VM on a real host): the install, the Proxmox step by the password path, the head template, verify READY, and a test cluster that ran srun on its compute node and was destroyed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
225 lines
5.9 KiB
Go
225 lines
5.9 KiB
Go
package setup
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"os/user"
|
|
"path/filepath"
|
|
"strconv"
|
|
"strings"
|
|
)
|
|
|
|
// Where an installation lives. Fixed: the unit, ansible.cfg and the
|
|
// Terraform template all name these paths, and a second layout would be a
|
|
// second thing to keep in step.
|
|
const (
|
|
EtcDir = "/etc/hpcod"
|
|
VarDir = "/var/lib/hpcod"
|
|
BinPath = "/usr/local/bin/hpcod"
|
|
UnitPath = "/etc/systemd/system/hpcod-portal.service"
|
|
ServiceID = "hpcod"
|
|
Unit = "hpcod-portal"
|
|
|
|
TemplateDir = VarDir + "/terraform/template"
|
|
SiteFile = EtcDir + "/site.env"
|
|
KeyPath = VarDir + "/.ssh/id_ed25519"
|
|
CACert = EtcDir + "/ca.crt"
|
|
CAKey = EtcDir + "/ca.key"
|
|
)
|
|
|
|
// Credentials is how setup reaches Proxmox while installing. Either root's
|
|
// password, used once and never written down, or the two tokens an operator
|
|
// made by hand. Neither is needed after the first install: by then the
|
|
// tokens are in the files that use them.
|
|
type Credentials struct {
|
|
Password string // root@pam's
|
|
// Pasted instead of a password; "user@realm!id=secret".
|
|
DiscoveryToken string
|
|
TerraformToken string
|
|
}
|
|
|
|
// Has reports whether any credential was given.
|
|
func (c Credentials) Has() bool {
|
|
return c.Password != "" || c.DiscoveryToken != "" || c.TerraformToken != ""
|
|
}
|
|
|
|
// Env is what every step works with.
|
|
type Env struct {
|
|
Site Site
|
|
Creds Credentials
|
|
// Out receives a step's progress, one line at a time. The TUI shows
|
|
// it under the step; the plain runner prints it.
|
|
Out io.Writer
|
|
// Root prefixes every path, so the steps that only write files can be
|
|
// tested in a temporary directory. Empty on a real host.
|
|
Root string
|
|
// Exe is this binary, installed as BinPath.
|
|
Exe string
|
|
// Revision is this binary's, stamped on the tree it writes.
|
|
Revision string
|
|
// RebuildTemplate forces the head template to be rebuilt even when it
|
|
// is current.
|
|
RebuildTemplate bool
|
|
}
|
|
|
|
// Logf reports progress.
|
|
func (e *Env) Logf(format string, args ...any) {
|
|
if e.Out != nil {
|
|
fmt.Fprintf(e.Out, format+"\n", args...)
|
|
}
|
|
}
|
|
|
|
// P maps an absolute path under Root.
|
|
func (e *Env) P(path string) string {
|
|
if e.Root == "" {
|
|
return path
|
|
}
|
|
return filepath.Join(e.Root, path)
|
|
}
|
|
|
|
// run executes a command and returns its combined output. On failure the
|
|
// output is part of the error: "exit status 100" from apt is not a reason.
|
|
func (e *Env) run(ctx context.Context, stdin string, name string, args ...string) (string, error) {
|
|
cmd := exec.CommandContext(ctx, name, args...)
|
|
cmd.Dir = "/" // runuser'd commands complain about a cwd they cannot read
|
|
if stdin != "" {
|
|
cmd.Stdin = strings.NewReader(stdin)
|
|
}
|
|
var buf bytes.Buffer
|
|
cmd.Stdout, cmd.Stderr = &buf, &buf
|
|
err := cmd.Run()
|
|
out := strings.TrimSpace(buf.String())
|
|
if err != nil {
|
|
tail := out
|
|
if lines := strings.Split(out, "\n"); len(lines) > 15 {
|
|
tail = strings.Join(lines[len(lines)-15:], "\n")
|
|
}
|
|
return out, fmt.Errorf("%s %s: %w\n%s", name, strings.Join(args, " "), err, tail)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// stream executes a command and copies its output to Out as it arrives,
|
|
// for the long ones - a package install, a playbook - where silence for
|
|
// five minutes reads as a hang.
|
|
func (e *Env) stream(ctx context.Context, env []string, dir, name string, args ...string) error {
|
|
cmd := exec.CommandContext(ctx, name, args...)
|
|
cmd.Dir = dir
|
|
if cmd.Dir == "" {
|
|
cmd.Dir = "/"
|
|
}
|
|
if env != nil {
|
|
cmd.Env = append(os.Environ(), env...)
|
|
}
|
|
w := e.Out
|
|
if w == nil {
|
|
w = io.Discard
|
|
}
|
|
tail := &tailBuffer{n: 20}
|
|
cmd.Stdout = io.MultiWriter(w, tail)
|
|
cmd.Stderr = cmd.Stdout
|
|
if err := cmd.Run(); err != nil {
|
|
return fmt.Errorf("%s: %w\n%s", name, err, tail.String())
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// tailBuffer keeps the last n lines written to it.
|
|
type tailBuffer struct {
|
|
n int
|
|
lines []string
|
|
part string
|
|
}
|
|
|
|
func (t *tailBuffer) Write(p []byte) (int, error) {
|
|
s := t.part + string(p)
|
|
parts := strings.Split(s, "\n")
|
|
t.part = parts[len(parts)-1]
|
|
t.lines = append(t.lines, parts[:len(parts)-1]...)
|
|
if len(t.lines) > t.n {
|
|
t.lines = t.lines[len(t.lines)-t.n:]
|
|
}
|
|
return len(p), nil
|
|
}
|
|
|
|
func (t *tailBuffer) String() string { return strings.Join(append(t.lines, t.part), "\n") }
|
|
|
|
// writeFile writes atomically with mode and owner, then reads the bytes
|
|
// back. The read-back is the point: a write that "succeeded" into a
|
|
// read-only or full filesystem, or onto a path a symlink redirected, is
|
|
// caught here rather than by whatever reads the file next.
|
|
func (e *Env) writeFile(path string, data []byte, mode os.FileMode, owner string) error {
|
|
p := e.P(path)
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
return err
|
|
}
|
|
tmp := p + ".new"
|
|
if err := os.WriteFile(tmp, data, mode); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Chmod(tmp, mode); err != nil { // umask
|
|
return err
|
|
}
|
|
if err := e.chown(tmp, owner); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Rename(tmp, p); err != nil {
|
|
return err
|
|
}
|
|
got, err := os.ReadFile(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !bytes.Equal(got, data) {
|
|
return fmt.Errorf("%s: what was read back is not what was written", path)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// chown sets "user:group" on an already-mapped path. A no-op under a test
|
|
// root, where the accounts do not exist.
|
|
func (e *Env) chown(p, owner string) error {
|
|
if owner == "" || e.Root != "" {
|
|
return nil
|
|
}
|
|
u, g, _ := strings.Cut(owner, ":")
|
|
uid, gid := -1, -1
|
|
if u != "" {
|
|
uu, err := user.Lookup(u)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
uid, _ = strconv.Atoi(uu.Uid)
|
|
}
|
|
if g != "" {
|
|
gg, err := user.LookupGroup(g)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
gid, _ = strconv.Atoi(gg.Gid)
|
|
}
|
|
return os.Lchown(p, uid, gid)
|
|
}
|
|
|
|
// mkdir creates a directory with mode and owner, and sets both on one
|
|
// that already exists.
|
|
func (e *Env) mkdir(path string, mode os.FileMode, owner string) error {
|
|
p := e.P(path)
|
|
if err := os.MkdirAll(p, mode); err != nil {
|
|
return err
|
|
}
|
|
if err := os.Chmod(p, mode); err != nil {
|
|
return err
|
|
}
|
|
return e.chown(p, owner)
|
|
}
|
|
|
|
func exists(p string) bool {
|
|
_, err := os.Stat(p)
|
|
return err == nil
|
|
}
|