An installation used to be this one: the deploy scripts and the Ansible defaults carried one site's domain, hypervisor and login, the Proxmox side and the database were set up by hand, and deploying meant copying scripts and a tree of roles beside the binary and keeping them in step with it. The binary now carries the roles, the Terraform module, the schema and the catalogue (embed.go), and installs itself: - hpcod setup: a full-screen installer for administrators. It asks where clusters are built, signs in to Proxmox before asking anything else (root's password once, or tokens), offers nodes and datastores from what Proxmox has, checks every field as it is typed, runs every step with its output live, checks the result and offers a test cluster. -yes runs the same steps from a site file; make deploy uses that. - hpcod verify: each fact checked by using it. - hpcod selftest: the smallest real cluster, by the portal's own path, a job run on it, then taken down. - Closed networks: hpcod serves HTTPS itself from a CA name-constrained to the site's domains, issuing and renewing its own certificate, and answers DNS for the portal and every cluster name. - A simulated Proxmox: Proxmox VE in QEMU on the control plane host, on a private bridge with NAT, for trying it without one. Every step reads back what it changed and is safe to run again. The site's values live in one file; group_vars/all/site.yml is generated from it, and the playbooks refuse to run without it. ci_user and -ssh-user no longer default to a personal account. scripts/B10, B20 and C20 are gone: their work is the setup steps. Proven end to end on a simulated Proxmox (Proxmox VE 9 in a VM on a real host): the install, the Proxmox step by the password path, the head template, verify READY, and a test cluster that ran srun on its compute node and was destroyed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
258 lines
7.6 KiB
Go
258 lines
7.6 KiB
Go
package setup
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/fs"
|
|
"os"
|
|
"path"
|
|
"path/filepath"
|
|
"sort"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/ilkermanap/hpcod"
|
|
)
|
|
|
|
// What the binary carries, and where each part goes on the host.
|
|
var treeDirs = []struct{ from, to string }{
|
|
{"ansible", VarDir + "/ansible"},
|
|
{"terraform/modules/tenant", VarDir + "/terraform/modules/tenant"},
|
|
{"db", VarDir + "/db"},
|
|
}
|
|
|
|
// The environment template's files. Copied into TemplateDir rather than
|
|
// replacing it: the site's tfvars live there too.
|
|
var templateFiles = []string{"terraform/envs/lab/main.tf", "terraform/envs/lab/variables.tf"}
|
|
|
|
// fileMode is the mode a tree file is written with. go:embed keeps no
|
|
// permission bits, so a script is recognised by its first two bytes - less
|
|
// the Jinja templates of scripts, which are not run as they are. That
|
|
// matches the executable bits in the repository exactly, as a test checks.
|
|
func fileMode(name string, data []byte) os.FileMode {
|
|
if bytes.HasPrefix(data, []byte("#!")) && !strings.HasSuffix(name, ".j2") {
|
|
return 0o755
|
|
}
|
|
return 0o644
|
|
}
|
|
|
|
// SiteYAML is ansible/group_vars/all/site.yml for a site: the values a head
|
|
// node needs that are this installation's rather than the platform's. It
|
|
// travels inside the head template the template step bakes. JSON strings are valid YAML, and
|
|
// quote what YAML would otherwise misread - every image URI has a colon.
|
|
func SiteYAML(s Site) string {
|
|
var b strings.Builder
|
|
b.WriteString("# Generated by `hpcod setup` from " + SiteFile + ".\n")
|
|
for _, kv := range [][2]string{
|
|
{"ondemand_public_domain", "CLUSTER_DOMAIN"},
|
|
{"node_image_uri", "NODE_IMAGE_URI"},
|
|
{"registry_host", "REGISTRY_HOST"},
|
|
{"ondemand_desktop_image", "DESKTOP_IMAGE"},
|
|
} {
|
|
v, _ := json.Marshal(s[kv[1]])
|
|
fmt.Fprintf(&b, "%s: %s\n", kv[0], v)
|
|
}
|
|
return b.String()
|
|
}
|
|
|
|
// stepTree writes the embedded tree to the host. Each directory is built
|
|
// beside the live one and swapped in, so a failure halfway leaves the old
|
|
// tree whole; the previous one is kept, three deep.
|
|
func stepTree(_ context.Context, e *Env) error {
|
|
stamp := time.Now().Format("20060102-150405")
|
|
revision := fmt.Sprintf("revision: %s\nbuilt: %s\n", e.Revision, time.Now().UTC().Format(time.RFC3339))
|
|
extra := map[string]map[string][]byte{
|
|
"ansible": {
|
|
"group_vars/all/site.yml": []byte(SiteYAML(e.Site)),
|
|
// The head node reads this to say which revision configured it,
|
|
// and the template is compared against it.
|
|
"REVISION": []byte(revision),
|
|
},
|
|
}
|
|
|
|
n := 0
|
|
for _, d := range treeDirs {
|
|
staging := e.P(d.to) + ".new"
|
|
if err := os.RemoveAll(staging); err != nil {
|
|
return err
|
|
}
|
|
count, err := extract(d.from, staging)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
for rel, data := range extra[path.Base(d.from)] {
|
|
p := filepath.Join(staging, rel)
|
|
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
|
return err
|
|
}
|
|
if err := os.WriteFile(p, data, 0o644); err != nil {
|
|
return err
|
|
}
|
|
count++
|
|
}
|
|
if err := swap(e, staging, e.P(d.to), stamp); err != nil {
|
|
return err
|
|
}
|
|
n += count
|
|
}
|
|
|
|
for _, f := range templateFiles {
|
|
data, err := hpcod.Tree.ReadFile(f)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(TemplateDir+"/"+path.Base(f), data, 0o644, owner); err != nil {
|
|
return err
|
|
}
|
|
n++
|
|
}
|
|
if err := e.writeFile(TemplateDir+"/REVISION", []byte(revision), 0o644, owner); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Read back: every embedded file is on disk with the same bytes.
|
|
if err := verifyTree(e); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("%d files from this binary (%s), read back and identical", n, e.Revision)
|
|
return nil
|
|
}
|
|
|
|
// extract writes one embedded directory to dst.
|
|
func extract(from, dst string) (int, error) {
|
|
n := 0
|
|
err := fs.WalkDir(hpcod.Tree, from, func(p string, d fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
out := filepath.Join(dst, strings.TrimPrefix(strings.TrimPrefix(p, from), "/"))
|
|
if d.IsDir() {
|
|
return os.MkdirAll(out, 0o755)
|
|
}
|
|
data, err := hpcod.Tree.ReadFile(p)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
n++
|
|
return os.WriteFile(out, data, fileMode(p, data))
|
|
})
|
|
return n, err
|
|
}
|
|
|
|
// swap puts staging in place of live, keeping live as a backup.
|
|
func swap(e *Env, staging, live, stamp string) error {
|
|
if err := filepath.WalkDir(staging, func(p string, _ fs.DirEntry, err error) error {
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return e.chown(p, owner)
|
|
}); err != nil {
|
|
return err
|
|
}
|
|
if err := os.MkdirAll(filepath.Dir(live), 0o755); err != nil {
|
|
return err
|
|
}
|
|
if exists(live) {
|
|
if err := os.Rename(live, live+".bak-"+stamp); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
if err := os.Rename(staging, live); err != nil {
|
|
return err
|
|
}
|
|
// Three backups: enough to get back to this morning. The tree is in
|
|
// the binary anyway.
|
|
baks, _ := filepath.Glob(live + ".bak-*")
|
|
sort.Strings(baks) // the stamp sorts by time
|
|
for len(baks) > 3 {
|
|
if err := os.RemoveAll(baks[0]); err != nil {
|
|
return err
|
|
}
|
|
baks = baks[1:]
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func verifyTree(e *Env) error {
|
|
for _, d := range treeDirs {
|
|
err := fs.WalkDir(hpcod.Tree, d.from, func(p string, de fs.DirEntry, err error) error {
|
|
if err != nil || de.IsDir() {
|
|
return err
|
|
}
|
|
want, _ := hpcod.Tree.ReadFile(p)
|
|
got, rerr := os.ReadFile(filepath.Join(e.P(d.to), strings.TrimPrefix(strings.TrimPrefix(p, d.from), "/")))
|
|
if rerr != nil || !bytes.Equal(got, want) {
|
|
return fmt.Errorf("%s did not land in %s", p, d.to)
|
|
}
|
|
return nil
|
|
})
|
|
if err != nil {
|
|
return err
|
|
}
|
|
}
|
|
got, err := os.ReadFile(e.P(VarDir + "/ansible/group_vars/all/site.yml"))
|
|
if err != nil || string(got) != SiteYAML(e.Site) {
|
|
return fmt.Errorf("group_vars/all/site.yml did not land")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// stepCatalogue installs the application catalogue this binary was built
|
|
// with, unless the site keeps its own. The portal re-reads it when it
|
|
// changes, so no restart is needed.
|
|
func stepCatalogue(_ context.Context, e *Env) error {
|
|
dst := EtcDir + "/apps.yaml"
|
|
if exists(e.P(EtcDir + "/apps.local")) {
|
|
e.Logf("%s exists; this site manages its own catalogue", EtcDir+"/apps.local")
|
|
return nil
|
|
}
|
|
data, err := hpcod.Tree.ReadFile("config/apps.yaml")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if err := e.writeFile(dst, data, 0o640, rootGroup); err != nil {
|
|
return err
|
|
}
|
|
e.Logf("%d applications in %s", bytes.Count(data, []byte("\n- name:")), dst)
|
|
return nil
|
|
}
|
|
|
|
// stepSchema applies every migration this binary carries, in order, as
|
|
// the database owner, then the grants once more. Every file is idempotent
|
|
// (db/README.md), so nothing records which have run.
|
|
func stepSchema(ctx context.Context, e *Env) error {
|
|
entries, err := fs.ReadDir(hpcod.Tree, "db")
|
|
if err != nil {
|
|
return err
|
|
}
|
|
var files []string
|
|
for _, en := range entries {
|
|
if strings.HasSuffix(en.Name(), ".sql") {
|
|
files = append(files, "db/"+en.Name())
|
|
}
|
|
}
|
|
sort.Strings(files)
|
|
files = append(files, "db/003-grants.sql")
|
|
for _, f := range files {
|
|
sql, _ := hpcod.Tree.ReadFile(f)
|
|
if _, err := e.psql(ctx, "hpcod", string(sql)); err != nil {
|
|
return fmt.Errorf("%s: %w", path.Base(f), err)
|
|
}
|
|
}
|
|
bad, err := e.psql(ctx, "hpcod", "", "-At", "-c", `
|
|
select c.relname from pg_class c join pg_namespace n on n.oid = c.relnamespace
|
|
where n.nspname = 'public' and c.relkind in ('r','v')
|
|
and not has_table_privilege('hpcod', c.oid, 'SELECT')`)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if strings.TrimSpace(bad) != "" {
|
|
return fmt.Errorf("the hpcod role cannot read: %s", strings.Join(strings.Fields(bad), ", "))
|
|
}
|
|
n, _ := e.psql(ctx, "hpcod", "", "-At", "-c", "select count(*) from pg_tables where schemaname='public'")
|
|
e.Logf("%d migrations applied; %s tables, all readable by hpcod", len(files)-1, strings.TrimSpace(n))
|
|
return nil
|
|
}
|