Files
ilkerandClaude Opus 5.5 772579648c hpcod: one binary installs, checks and tests the control plane
An installation used to be this one: the deploy scripts and the Ansible
defaults carried one site's domain, hypervisor and login, the Proxmox side
and the database were set up by hand, and deploying meant copying scripts
and a tree of roles beside the binary and keeping them in step with it.

The binary now carries the roles, the Terraform module, the schema and the
catalogue (embed.go), and installs itself:

- hpcod setup: a full-screen installer for administrators. It asks where
  clusters are built, signs in to Proxmox before asking anything else
  (root's password once, or tokens), offers nodes and datastores from what
  Proxmox has, checks every field as it is typed, runs every step with its
  output live, checks the result and offers a test cluster. -yes runs the
  same steps from a site file; make deploy uses that.
- hpcod verify: each fact checked by using it.
- hpcod selftest: the smallest real cluster, by the portal's own path, a
  job run on it, then taken down.
- Closed networks: hpcod serves HTTPS itself from a CA name-constrained to
  the site's domains, issuing and renewing its own certificate, and
  answers DNS for the portal and every cluster name.
- A simulated Proxmox: Proxmox VE in QEMU on the control plane host, on a
  private bridge with NAT, for trying it without one.

Every step reads back what it changed and is safe to run again. The
site's values live in one file; group_vars/all/site.yml is generated from
it, and the playbooks refuse to run without it. ci_user and -ssh-user no
longer default to a personal account. scripts/B10, B20 and C20 are gone:
their work is the setup steps.

Proven end to end on a simulated Proxmox (Proxmox VE 9 in a VM on a real
host): the install, the Proxmox step by the password path, the head
template, verify READY, and a test cluster that ran srun on its compute
node and was destroyed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-16 18:20:04 +01:00

610 lines
20 KiB
Go

package httpapi
import (
"context"
"encoding/json"
"errors"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"github.com/ilkermanap/hpcod/internal/catalog"
"github.com/ilkermanap/hpcod/internal/model"
"github.com/ilkermanap/hpcod/internal/placement"
"github.com/ilkermanap/hpcod/internal/store"
)
// Without a database the portal can still show and plan, but it must refuse
// to allocate rather than pretend. An allocation nobody records is one
// nothing will ever come back to release.
func TestClusterRoutesRefuseWithoutADatabase(t *testing.T) {
s := &Server{cache: &cache{}, catalog: catalog.Static(nil)}
for _, c := range []struct{ method, path string }{
{http.MethodGet, "/v1/clusters"},
{http.MethodPost, "/v1/clusters"},
{http.MethodDelete, "/v1/clusters/1"},
} {
w := httptest.NewRecorder()
r := httptest.NewRequest(c.method, c.path, strings.NewReader("{}"))
if strings.HasSuffix(c.path, "/1") {
s.jsonCluster(w, r)
} else {
s.jsonClusters(w, r)
}
if w.Code != http.StatusServiceUnavailable {
t.Errorf("%s %s = %d, want 503", c.method, c.path, w.Code)
}
if !strings.Contains(w.Body.String(), "no database configured") {
t.Errorf("%s %s body does not say why: %s", c.method, c.path, w.Body)
}
}
}
func TestClusterIDMustBeANumber(t *testing.T) {
s := &Server{cache: &cache{}, catalog: catalog.Static(nil), store: &fakeStore{}}
for _, path := range []string{"/v1/clusters/abc", "/v1/clusters/", "/v1/clusters/-1", "/v1/clusters/0"} {
w := httptest.NewRecorder()
s.jsonCluster(w, httptest.NewRequest(http.MethodDelete, path, nil))
if w.Code != http.StatusBadRequest {
t.Errorf("DELETE %s = %d, want 400", path, w.Code)
}
}
}
// The inventory cache is shared with every other reader and refreshed on a
// timer. Marking reservations onto it directly would let one request's view
// of who holds what leak into the sites page and into every later plan.
func TestReservationsDoNotMutateTheSharedCache(t *testing.T) {
sites := []model.Site{{
ID: "hv01",
Nodes: []model.Node{{
Name: "hv01", Online: true,
Devices: []model.Device{{NativeID: "0000:01:00.0", Capability: model.CapGPU}},
}},
}}
dup := copySites(sites)
model.ApplyReservations(dup, []model.Reservation{
{Site: "hv01", HostNode: "hv01", NativeID: "0000:01:00.0", HeldBy: "c001-1"},
})
if got := sites[0].Nodes[0].Devices[0].AllocatedTo; got != "" {
t.Errorf("the original was marked too: %q", got)
}
if got := dup[0].Nodes[0].Devices[0].AllocatedTo; got == "" {
t.Error("the copy was not marked, so the copy is not being used")
}
}
// The placement is the only description of the cluster that survives the
// request, so nothing in it may be dropped in translation.
func TestPlacementSurvivesTranslationToTheStore(t *testing.T) {
c := placement.Candidate{
SiteID: "hv01",
Placement: []placement.VMPlacement{
{Role: "head", Index: 0, HostNode: "hv01", Cores: 2, MemoryGB: 4},
{Role: "compute", Index: 1, HostNode: "pve2", Cores: 8, MemoryGB: 34, Devices: []placement.PlacedDevice{
{NativeID: "0000:01:00.0", Capability: model.CapGPU, Model: "Quadro K620"},
}},
},
}
got := store.PlacementFrom(c)
if got.Site != "hv01" || len(got.VMs) != 2 {
t.Fatalf("got %+v", got)
}
if got.VMs[1].HostNode != "pve2" {
t.Errorf("compute went to %q, want pve2 - a per-VM host must not collapse to the site", got.VMs[1].HostNode)
}
if len(got.VMs[1].Devices) != 1 ||
got.VMs[1].Devices[0].NativeID != "0000:01:00.0" ||
got.VMs[1].Devices[0].Capability != "gpu" {
t.Errorf("devices = %+v", got.VMs[1].Devices)
}
if len(got.VMs[0].Devices) != 0 {
t.Errorf("the head picked up devices it was not given: %+v", got.VMs[0].Devices)
}
}
// An infeasible request is well formed and may succeed tomorrow, so it is a
// conflict, not a bad request - and the answer has to say what is in the way.
func TestInfeasibleRequestIsAConflictThatExplainsItself(t *testing.T) {
s := &Server{cache: &cache{}, catalog: catalog.Static(nil), store: &fakeStore{}}
s.cache.set([]model.Site{{ID: "hv01", Reachable: true, Healthy: true}})
body := `
name: nope
tenant: acme
owner: ops@acme.example
head: { cores: 2, memory_gb: 4 }
compute: { count: 1, cores: 4, memory_gb: 8, require: [{capability: gpu}] }
`
w := httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(body)))
if w.Code != http.StatusConflict {
t.Fatalf("status = %d, want 409; body %s", w.Code, w.Body)
}
var out struct {
Error string `json:"error"`
Decision struct {
Candidates []struct {
Reasons []string `json:"reasons"`
} `json:"candidates"`
} `json:"decision"`
}
if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
if len(out.Decision.Candidates) == 0 || len(out.Decision.Candidates[0].Reasons) == 0 {
t.Fatalf("no reason given: %s", w.Body)
}
}
func TestCreateRefusesWithoutATenant(t *testing.T) {
s := &Server{cache: &cache{}, catalog: catalog.Static(nil), store: &fakeStore{}}
s.cache.set([]model.Site{{ID: "hv01", Reachable: true, Healthy: true}})
body := "name: x\nowner: me\nhead: { cores: 1, memory_gb: 1 }\ncompute: { count: 1, cores: 1, memory_gb: 1 }\n"
w := httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(body)))
if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "tenant is required") {
t.Errorf("status %d body %s", w.Code, w.Body)
}
}
// Nobody signed in and no owner named: the record would have no one to
// attribute, so the request is refused rather than attributed to "".
func TestCreateRefusesWhenNobodyOwnsIt(t *testing.T) {
s := &Server{cache: &cache{}, catalog: catalog.Static(nil), store: &fakeStore{}}
s.cache.set([]model.Site{{ID: "hv01", Reachable: true, Healthy: true}})
body := "name: x\ntenant: acme\nhead: { cores: 1, memory_gb: 1 }\ncompute: { count: 1, cores: 1, memory_gb: 1 }\n"
w := httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(body)))
if w.Code != http.StatusBadRequest || !strings.Contains(w.Body.String(), "owner is required") {
t.Errorf("status %d body %s", w.Code, w.Body)
}
}
// fakeStore stands in for the database. It records nothing and allocates
// nothing; a test that needs a real allocation says so by setting allocated.
type fakeStore struct {
mu sync.Mutex
states []string
reservations []model.Reservation
allocated *store.Cluster
allocErr error
gotPlacement store.Placement
gotTenant string
gotOwner string
gotLease time.Duration
gotApps []store.ClusterApp
released []int64
leaseSet []int64
leaseTo *time.Time
accounts map[string]*store.Account
hashFor map[string]string
hashes []string
// createOverride lets a test simulate a database that has stopped
// defaulting new accounts to pending, which is the one thing the
// signup guard exists for.
createOverride func(*store.Account) *store.Account
deleted []string
deleteErr error
projects []store.Project
projectErr error
usage []store.Usage
policies map[string]model.Overcommit
policyErr error
}
func (f *fakeStore) Reservations(context.Context) ([]model.Reservation, error) {
return f.reservations, nil
}
func (f *fakeStore) EnsureTenant(_ context.Context, name, owner string) (int64, error) {
f.gotTenant, f.gotOwner = name, owner
return 1, nil
}
func (f *fakeStore) AllocateCluster(_ context.Context, _ int64, p store.Placement, lease time.Duration, apps []store.ClusterApp, projectID *int64) (*store.Cluster, error) {
f.gotPlacement, f.gotLease, f.gotApps = p, lease, apps
if f.allocErr != nil {
return nil, f.allocErr
}
if f.allocated == nil {
return nil, errors.New("fakeStore: no allocation configured for this test")
}
return f.allocated, nil
}
func (f *fakeStore) Live(context.Context) ([]store.Cluster, error) {
if f.allocated == nil {
return nil, nil
}
return []store.Cluster{*f.allocated}, nil
}
func (f *fakeStore) VMs(context.Context, int64) ([]store.VM, error) {
return f.gotPlacement.VMs, nil
}
// Apps reads back what AllocateCluster was given, so a page test that drives
// a real provision.Runner exercises the whole carry: the catalogue resolves
// an application, allocation records it, and the Runner renders it into the
// variables. Returning a fixture instead would let the middle of that chain
// break without a test noticing - which is how it came to be broken.
func (f *fakeStore) Apps(context.Context, int64) ([]store.ClusterApp, error) {
return f.gotApps, nil
}
func (f *fakeStore) Release(_ context.Context, id int64) error {
f.released = append(f.released, id)
return nil
}
// SetState and seenStates let the page tests drive a real provision.Runner
// without a hypervisor: the Runner is what moves a cluster through
// provisioning, and the pages are what report it.
func (f *fakeStore) SetState(_ context.Context, _ int64, state string) error {
f.mu.Lock()
defer f.mu.Unlock()
f.states = append(f.states, state)
return nil
}
func (f *fakeStore) seenStates() []string {
f.mu.Lock()
defer f.mu.Unlock()
return append([]string(nil), f.states...)
}
// Projects and usage. The fake keeps them in memory so the pages can be
// driven without a PostgreSQL; the store's own rules - a project has to
// be the tenant's and open - are tested against the real thing.
func (f *fakeStore) CreateProject(_ context.Context, tenant, _, name, description, by string) (*store.Project, error) {
f.mu.Lock()
defer f.mu.Unlock()
if f.projectErr != nil {
return nil, f.projectErr
}
for _, p := range f.projects {
if p.TenantName == tenant && p.Name == name {
return nil, errors.New(tenant + " already has a project called " + name)
}
}
p := store.Project{
ID: int64(len(f.projects) + 1), TenantName: tenant, Name: name,
Description: description, CreatedBy: by, CreatedAt: time.Now(),
}
f.projects = append(f.projects, p)
return &p, nil
}
func (f *fakeStore) Projects(_ context.Context, tenant string) ([]store.Project, error) {
f.mu.Lock()
defer f.mu.Unlock()
var out []store.Project
for _, p := range f.projects {
if tenant == "" || p.TenantName == tenant {
out = append(out, p)
}
}
return out, nil
}
func (f *fakeStore) CloseProject(_ context.Context, id int64, tenant string) error {
f.mu.Lock()
defer f.mu.Unlock()
for i := range f.projects {
if f.projects[i].ID == id && (tenant == "" || f.projects[i].TenantName == tenant) {
now := time.Now()
f.projects[i].ClosedAt = &now
return nil
}
}
return errors.New("no such project")
}
func (f *fakeStore) Usage(_ context.Context, tenant string, _ int) ([]store.Usage, error) {
var out []store.Usage
for _, u := range f.usage {
if tenant == "" || u.TenantName == tenant {
out = append(out, u)
}
}
return out, nil
}
func (f *fakeStore) UsageFor(_ context.Context, serialID int64) (*store.Usage, error) {
for i := range f.usage {
if f.usage[i].SerialID == serialID {
return &f.usage[i], nil
}
}
return nil, errors.New("no usage recorded")
}
func (f *fakeStore) Close() {}
func gpuSite() []model.Site {
return []model.Site{{
ID: "hv01", Reachable: true, Healthy: true,
Nodes: []model.Node{{
Name: "hv01", Online: true, Cores: 64, MemBytes: 256 << 30,
Devices: []model.Device{{
NativeID: "0000:01:00.0", Capability: model.CapGPU,
Confidence: model.ConfClass, Passthrough: true, Model: "Quadro K620",
}},
}},
}}
}
const gpuRequest = `
name: sim
tenant: acme
owner: ops@acme.example
lease: 7d
head: { cores: 2, memory_gb: 4 }
compute:
count: 1
cores: 4
memory_gb: 8
require:
- capability: gpu
`
func TestCreateAllocatesExactlyWhatWasPlanned(t *testing.T) {
f := &fakeStore{allocated: &store.Cluster{
SerialID: 7, ClusterID: 3, Site: "hv01", AccountingName: "c003-7", State: "requested",
}}
s := &Server{cache: &cache{}, catalog: catalog.Static(nil), store: f}
s.cache.set(gpuSite())
w := httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(gpuRequest)))
if w.Code != http.StatusCreated {
t.Fatalf("status = %d, want 201; body %s", w.Code, w.Body)
}
if f.gotTenant != "acme" || f.gotOwner != "ops@acme.example" {
t.Errorf("tenant/owner recorded as %q/%q", f.gotTenant, f.gotOwner)
}
if f.gotLease != 7*24*time.Hour {
t.Errorf("lease = %v, want 168h", f.gotLease)
}
if f.gotPlacement.Site != "hv01" || len(f.gotPlacement.VMs) != 2 {
t.Fatalf("placement = %+v", f.gotPlacement)
}
// The card the planner chose is the card that gets written down. If this
// drifts, the cluster is built with hardware nobody reserved.
var devices []store.Device
for _, vm := range f.gotPlacement.VMs {
devices = append(devices, vm.Devices...)
}
if len(devices) != 1 || devices[0].NativeID != "0000:01:00.0" || devices[0].Capability != "gpu" {
t.Errorf("devices written = %+v", devices)
}
var out map[string]any
if err := json.Unmarshal(w.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
if out["accounting_name"] != "c003-7" || out["state"] != "requested" {
t.Errorf("response = %v", out)
}
if !strings.Contains(out["note"].(string), "not built") {
t.Errorf("the response does not say the cluster has not been built: %v", out["note"])
}
}
// The loop that had to be closed: a card promised to a cluster that does
// not exist yet must stop being offered to the next request. Nothing on the
// hypervisor shows it as taken, so only the reservation can say so.
func TestADeviceReservedForAnUnbuiltClusterIsNotOfferedAgain(t *testing.T) {
f := &fakeStore{reservations: []model.Reservation{
{Site: "hv01", HostNode: "hv01", NativeID: "0000:01:00.0", HeldBy: "c003-7"},
}}
s := &Server{cache: &cache{}, catalog: catalog.Static(nil), store: f}
s.cache.set(gpuSite())
w := httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(gpuRequest)))
if w.Code != http.StatusConflict {
t.Fatalf("status = %d, want 409 - the only GPU is already promised; body %s", w.Code, w.Body)
}
if !strings.Contains(w.Body.String(), "gpu") {
t.Errorf("the refusal does not mention the GPU: %s", w.Body)
}
// Without the reservation the very same request must succeed, otherwise
// this test proves only that the request was impossible anyway.
f.reservations = nil
f.allocated = &store.Cluster{SerialID: 8, ClusterID: 4, Site: "hv01", AccountingName: "c004-8", State: "requested"}
w = httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(gpuRequest)))
if w.Code != http.StatusCreated {
t.Fatalf("without the reservation: status = %d, want 201; body %s", w.Code, w.Body)
}
}
// Losing the race between planning and writing is a conflict to re-plan
// against, not a server error.
func TestAllocationLosingTheRaceIsAConflict(t *testing.T) {
f := &fakeStore{allocErr: store.ErrDeviceHeld}
s := &Server{cache: &cache{}, catalog: catalog.Static(nil), store: f}
s.cache.set(gpuSite())
w := httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(gpuRequest)))
if w.Code != http.StatusConflict {
t.Errorf("status = %d, want 409; body %s", w.Code, w.Body)
}
}
// A database the caller cannot see is not their mistake. Reported as a 400
// it would send them rewriting a request that was never wrong.
func TestStoreFailureIsFiveHundredNotFourHundred(t *testing.T) {
s := &Server{cache: &cache{}, catalog: catalog.Static(nil), store: &brokenStore{}}
s.cache.set(gpuSite())
w := httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(gpuRequest)))
if w.Code != http.StatusInternalServerError {
t.Errorf("status = %d, want 500; body %s", w.Code, w.Body)
}
}
type brokenStore struct{ fakeStore }
func (b *brokenStore) Reservations(context.Context) ([]model.Reservation, error) {
return nil, errors.New("permission denied for table cluster_device")
}
func (f *fakeStore) SetLease(_ context.Context, id int64, expires *time.Time) error {
f.leaseSet = append(f.leaseSet, id)
f.leaseTo = expires
return nil
}
// ---------- accounts ----------
func (f *fakeStore) CreateAccount(_ context.Context, username, hash, email, reason string) (*store.Account, error) {
if f.accounts == nil {
f.accounts = map[string]*store.Account{}
}
if _, taken := f.accounts[username]; taken {
return nil, store.ErrAccountExists
}
// Mirrors the column defaults: pending, never an administrator.
a := &store.Account{Username: username, Email: email, Reason: reason, State: "pending"}
if f.createOverride != nil {
a = f.createOverride(a)
}
f.accounts[username] = a
f.hashes = append(f.hashes, hash)
return a, nil
}
func (f *fakeStore) AccountWithHash(_ context.Context, username string) (*store.Account, string, error) {
a, ok := f.accounts[username]
if !ok {
return nil, "", store.ErrNoAccount
}
return a, f.hashFor[username], nil
}
func (f *fakeStore) Accounts(context.Context) ([]store.Account, error) {
out := make([]store.Account, 0, len(f.accounts))
for _, a := range f.accounts {
out = append(out, *a)
}
return out, nil
}
func (f *fakeStore) DecideAccount(_ context.Context, username, state, by string) (*store.Account, error) {
a, ok := f.accounts[username]
if !ok {
return nil, store.ErrNoAccount
}
a.State, a.DecidedBy = state, by
return a, nil
}
func (f *fakeStore) DeleteAccount(_ context.Context, username string) error {
if f.accounts == nil || f.accounts[username] == nil {
return errors.New("no account " + username)
}
if f.deleteErr != nil {
return f.deleteErr
}
delete(f.accounts, username)
f.deleted = append(f.deleted, username)
return nil
}
func (f *fakeStore) SetAdmin(_ context.Context, username string, admin bool) (*store.Account, error) {
a, ok := f.accounts[username]
if !ok {
return nil, store.ErrNoAccount
}
a.IsAdmin = admin
return a, nil
}
// ---------- site policy ----------
func (f *fakeStore) Policies(context.Context) (map[string]model.Overcommit, error) {
return f.policies, nil
}
func (f *fakeStore) SetPolicy(_ context.Context, o model.Overcommit, by string) (*model.Overcommit, error) {
if f.policyErr != nil {
return nil, f.policyErr
}
if f.policies == nil {
f.policies = map[string]model.Overcommit{}
}
if !o.Enabled {
o.CPU, o.Mem = 1, 1
}
o.UpdatedBy = by
f.policies[o.Site] = o
return &o, nil
}
// The catalogue's answer has to reach the record, because the record is the
// only thing the provisioner reads. Before this the resolved list was
// computed, shown in the response, and dropped on the floor - so the reply
// said "apps: [gromacs]" and the cluster that came back had none.
func TestResolvedApplicationsReachTheAllocation(t *testing.T) {
f := &fakeStore{allocated: &store.Cluster{SerialID: 1, ClusterID: 1, AccountingName: "c001-1"}}
s := &Server{cache: &cache{}, store: f, catalog: catalog.Static(&catalog.Catalog{Apps: []catalog.App{
{Name: "gromacs", Version: "2024", Image: "oras://reg.example/hpcapps/gromacs:2024"},
}})}
s.cache.set([]model.Site{{
ID: "hv01", Reachable: true, Healthy: true,
Nodes: []model.Node{{Name: "hv01", Online: true, Cores: 64, MemBytes: 256 << 30}},
}})
body := `
name: sim
tenant: acme
owner: ops@acme.example
head: { cores: 2, memory_gb: 4 }
compute: { count: 1, cores: 4, memory_gb: 8 }
stack: { apps: [gromacs] }
`
w := httptest.NewRecorder()
s.createCluster(w, httptest.NewRequest(http.MethodPost, "/v1/clusters", strings.NewReader(body)))
if w.Code != http.StatusCreated {
t.Fatalf("status = %d, want 201; body %s", w.Code, w.Body)
}
// Not the reply - the reply was always right. What the store was given.
if len(f.gotApps) != 1 {
t.Fatalf("the store was given %d applications, want 1: %+v", len(f.gotApps), f.gotApps)
}
if f.gotApps[0].Name != "gromacs" {
t.Errorf("name recorded as %q", f.gotApps[0].Name)
}
if f.gotApps[0].Image != "oras://reg.example/hpcapps/gromacs:2024" {
t.Errorf("image recorded as %q; the head pulls this, so a reference it cannot resolve is a cluster with no software",
f.gotApps[0].Image)
}
}