"How much did the protein run use" is a question about a piece of work, not about c003-15. A tenant asks for clusters over months; individually each one is an address, some hardware and a lease, and nothing joined them up. A project groups them, and is optional throughout. Somebody who wants one cluster for an afternoon should not have to invent a project to get it, and requiring one would mean every tenant carrying a "default" that means nothing. The form does not even ask when there are none: a field whose only option is "none" is a question with one answer. The cost side needed no new columns. The cluster record already said when it was created and destroyed, and cluster_vm said how many cores and how much memory it held; what was missing was anything that read them. cluster_usage is that, and it is a view rather than a table because a derived number stored is a number that can disagree with what it was derived from. It works retroactively - every cluster this estate has ever built now has a line. Core-hours are reserved, not executed, and the view says so. That is the honest unit for an on-demand cluster: the hardware was yours and nobody else could have it, whether or not a job ran. What was actually executed is a Slurm accounting question and there is no slurmdbd. The project is attached inside the allocating transaction, for the same reason the placement and the applications are, and it is checked there to be the tenant's own and still open - a cluster filed against somebody else's project leaks what they are working on. Closing keeps history and stops it being offered. There is no delete: the reference from cluster is RESTRICT, because a project removed from under its clusters leaves their cost attributed to nothing, and afterwards is when the question gets asked. A release now writes manifest.json beside home.tgz: the nodes, the wall time, the core-hours. The database knows all of it and outlives the cluster, but the tarball can be moved off this host and the description travels with it. Not fatal if it fails - the data is already safe by then, and holding hardware because a note could not be written is the wrong trade. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
856 lines
26 KiB
Go
856 lines
26 KiB
Go
package provision
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"io"
|
|
"os"
|
|
"os/exec"
|
|
"path/filepath"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/ilkermanap/hpcod/internal/store"
|
|
)
|
|
|
|
type fakeStore struct {
|
|
mu sync.Mutex
|
|
live []store.Cluster
|
|
usage *store.Usage
|
|
vms []store.VM
|
|
apps []store.ClusterApp
|
|
states []string
|
|
released int
|
|
err error
|
|
relErr error
|
|
}
|
|
|
|
func (f *fakeStore) VMs(context.Context, int64) ([]store.VM, error) { return f.vms, f.err }
|
|
|
|
func (f *fakeStore) Apps(context.Context, int64) ([]store.ClusterApp, error) {
|
|
return f.apps, f.err
|
|
}
|
|
|
|
func (f *fakeStore) SetState(_ context.Context, _ int64, state string) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
f.states = append(f.states, state)
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeStore) Live(context.Context) ([]store.Cluster, error) {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return append([]store.Cluster(nil), f.live...), nil
|
|
}
|
|
|
|
func (f *fakeStore) UsageFor(context.Context, int64) (*store.Usage, error) {
|
|
if f.usage == nil {
|
|
return &store.Usage{AccountingName: "c003-7", TenantName: "acme", Site: "hv01",
|
|
Cores: 6, MemoryGB: 48, Wall: 2 * time.Hour, CoreHours: 12, GBHours: 96}, nil
|
|
}
|
|
return f.usage, nil
|
|
}
|
|
|
|
func (f *fakeStore) Release(context.Context, int64) error {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
if f.relErr != nil {
|
|
return f.relErr
|
|
}
|
|
f.released++
|
|
return nil
|
|
}
|
|
|
|
func (f *fakeStore) releases() int {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return f.released
|
|
}
|
|
|
|
func (f *fakeStore) seen() []string {
|
|
f.mu.Lock()
|
|
defer f.mu.Unlock()
|
|
return append([]string(nil), f.states...)
|
|
}
|
|
|
|
// A fake "tofu" that records how it was called and can be told to fail.
|
|
func fakeBinary(t *testing.T, dir string, exitCode int) string {
|
|
t.Helper()
|
|
path := filepath.Join(dir, "fake-tofu")
|
|
script := "#!/bin/sh\necho \"called: $*\"\necho \"cwd: $(pwd)\"\n"
|
|
if exitCode != 0 {
|
|
script += "echo 'something went wrong' >&2\n"
|
|
}
|
|
script += "exit " + string(rune('0'+exitCode)) + "\n"
|
|
if err := os.WriteFile(path, []byte(script), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
return path
|
|
}
|
|
|
|
func testRunner(t *testing.T, f *fakeStore, exitCode int) *Runner {
|
|
t.Helper()
|
|
root := t.TempDir()
|
|
tmpl := filepath.Join(root, "template")
|
|
if err := os.MkdirAll(tmpl, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for name, body := range map[string]string{
|
|
"main.tf": "# module wiring\n",
|
|
"site.auto.tfvars": "node_name = \"hv01\"\n",
|
|
"terraform.tfstate": "SHOULD NOT BE COPIED\n",
|
|
} {
|
|
if err := os.WriteFile(filepath.Join(tmpl, name), []byte(body), 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
return NewRunner(f, Options{
|
|
EnvRoot: filepath.Join(root, "envs"), Template: tmpl,
|
|
Binary: fakeBinary(t, root, exitCode), MgmtFirst: "10.0.0.160", MgmtCount: 40,
|
|
Timeout: 30 * time.Second,
|
|
// There is no head node here, and dialling 10.0.0.160 from a unit
|
|
// test would reach whatever happens to be at that address.
|
|
headReady: func(context.Context, string, io.Writer) error { return nil },
|
|
})
|
|
}
|
|
|
|
func aCluster() store.Cluster {
|
|
return store.Cluster{SerialID: 7, ClusterID: 3, TenantName: "acme", AccountingName: "c003-7", Site: "hv01"}
|
|
}
|
|
|
|
func goodVMs() []store.VM {
|
|
return []store.VM{
|
|
{Role: "head", NodeIndex: 0, HostNode: "hv01", Cores: 2, MemoryGB: 4},
|
|
{Role: "compute", NodeIndex: 1, HostNode: "hv01", Cores: 4, MemoryGB: 10},
|
|
}
|
|
}
|
|
|
|
func waitForState(t *testing.T, f *fakeStore, want string) []string {
|
|
t.Helper()
|
|
deadline := time.Now().Add(10 * time.Second)
|
|
for time.Now().Before(deadline) {
|
|
got := f.seen()
|
|
if len(got) > 0 && got[len(got)-1] == want {
|
|
return got
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
t.Fatalf("never reached %q; states were %v", want, f.seen())
|
|
return nil
|
|
}
|
|
|
|
// The states are the only thing anyone reads afterwards, so they have to
|
|
// tell the whole story.
|
|
func TestASuccessfulBuildMovesRequestedToReady(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := waitForState(t, f, "ready")
|
|
if len(got) != 2 || got[0] != "provisioning" {
|
|
t.Errorf("states = %v, want [provisioning ready]", got)
|
|
}
|
|
|
|
log, err := os.ReadFile(r.LogPath(7))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{"init", "apply -auto-approve", "done"} {
|
|
if !strings.Contains(string(log), want) {
|
|
t.Errorf("the log does not mention %q:\n%s", want, log)
|
|
}
|
|
}
|
|
}
|
|
|
|
// A failure has to be recorded as one. A cluster stuck at "provisioning"
|
|
// with nothing to read is worse than one that plainly failed.
|
|
func TestAFailedBuildSaysSoAndKeepsTheOutput(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 1)
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
got := waitForState(t, f, "failed")
|
|
if len(got) != 2 || got[0] != "provisioning" {
|
|
t.Errorf("states = %v, want [provisioning failed]", got)
|
|
}
|
|
|
|
log, err := os.ReadFile(r.LogPath(7))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(log), "something went wrong") {
|
|
t.Errorf("the tool's own output was lost:\n%s", log)
|
|
}
|
|
if !strings.Contains(string(log), "failed") {
|
|
t.Errorf("the log does not say it failed:\n%s", log)
|
|
}
|
|
}
|
|
|
|
// Two Terraform processes in one directory would fight over the state file.
|
|
func TestTheSameClusterCannotBeBuiltTwiceAtOnce(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Racy by nature, so only assert when the first is demonstrably still
|
|
// in flight.
|
|
if r.Running(7) {
|
|
if err := r.Start(aCluster()); err == nil {
|
|
t.Error("a second build was allowed to start")
|
|
}
|
|
}
|
|
waitForState(t, f, "ready")
|
|
|
|
// And once it is done, building again is fine.
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Errorf("rebuilding after it finished: %v", err)
|
|
}
|
|
waitForState(t, f, "ready")
|
|
}
|
|
|
|
// The working directory is seeded from the template, but Terraform's own
|
|
// state must never be overwritten by it: that is the record of what was
|
|
// actually created.
|
|
func TestTheTemplateNeverOverwritesTheState(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
dir := r.Dir(7)
|
|
if err := os.MkdirAll(dir, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "terraform.tfstate"), []byte("REAL STATE"), 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForState(t, f, "ready")
|
|
|
|
got, err := os.ReadFile(filepath.Join(dir, "terraform.tfstate"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if string(got) != "REAL STATE" {
|
|
t.Errorf("the state was overwritten with %q", got)
|
|
}
|
|
if _, err := os.Stat(filepath.Join(dir, "main.tf")); err != nil {
|
|
t.Errorf("the template was not copied: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestTheVariablesAreWrittenFromTheRecord(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForState(t, f, "ready")
|
|
|
|
got, err := os.ReadFile(filepath.Join(r.Dir(7), "cluster.auto.tfvars"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
for _, want := range []string{`cluster_id = 3`, `tenant_name = "acme"`, `mgmt_ip_cidr = "10.0.0.162/24"`} {
|
|
if !strings.Contains(string(got), want) {
|
|
t.Errorf("missing %q in:\n%s", want, got)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Addresses come from the cluster id, so two live clusters cannot be
|
|
// handed the same one - within the window, and never outside it.
|
|
func TestManagementAddressesStayInsideTheWindow(t *testing.T) {
|
|
r := NewRunner(&fakeStore{}, Options{MgmtFirst: "10.0.0.160", MgmtCount: 40, MgmtPrefix: 24})
|
|
|
|
seen := map[string]int16{}
|
|
for id := int16(1); id <= 40; id++ {
|
|
got, err := r.MgmtCIDR(id)
|
|
if err != nil {
|
|
t.Fatalf("cluster %d: %v", id, err)
|
|
}
|
|
if prev, clash := seen[got]; clash {
|
|
t.Fatalf("clusters %d and %d both get %s", prev, id, got)
|
|
}
|
|
seen[got] = id
|
|
}
|
|
if got, _ := r.MgmtCIDR(1); got != "10.0.0.160/24" {
|
|
t.Errorf("the first cluster gets %q, want the first address of the window", got)
|
|
}
|
|
if got, _ := r.MgmtCIDR(3); got != "10.0.0.162/24" {
|
|
t.Errorf("MgmtCIDR(3) = %q", got)
|
|
}
|
|
|
|
// The failure this replaced: deriving base + id gave cluster 10 the
|
|
// hypervisor's own address. Past the window it must refuse, not wrap
|
|
// or wander into somebody else's machine.
|
|
if _, err := r.MgmtCIDR(41); err == nil {
|
|
t.Error("an id past the window was given an address anyway")
|
|
}
|
|
if _, err := r.MgmtCIDR(200); err == nil {
|
|
t.Error("an id far past the window was given an address anyway")
|
|
}
|
|
}
|
|
|
|
// A window that runs off the end of the network is refused rather than
|
|
// silently producing .255 or beyond.
|
|
func TestAWindowThatOverrunsTheNetworkIsRefused(t *testing.T) {
|
|
r := NewRunner(&fakeStore{}, Options{MgmtFirst: "10.0.0.250", MgmtCount: 40, MgmtPrefix: 24})
|
|
if _, err := r.MgmtCIDR(10); err == nil {
|
|
t.Error("an address past .254 was produced")
|
|
}
|
|
}
|
|
|
|
// With no window stated, building has to stop rather than guess.
|
|
func TestNoWindowIsRefused(t *testing.T) {
|
|
r := NewRunner(&fakeStore{}, Options{})
|
|
if _, err := r.MgmtCIDR(1); err == nil {
|
|
t.Error("an address was invented with no window configured")
|
|
}
|
|
}
|
|
|
|
// Without a template there is nothing to build from, and saying so beats
|
|
// running the tool in an empty directory and failing obscurely.
|
|
func TestNoTemplateIsRefusedClearly(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := NewRunner(f, Options{EnvRoot: t.TempDir(), Binary: "true"})
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForState(t, f, "failed")
|
|
}
|
|
|
|
// The cluster is told to trust the certificate this portal presents, and
|
|
// it is told by the portal rather than by a value an operator maintains
|
|
// separately. When they differ nothing looks wrong: the cluster builds,
|
|
// its portal answers, and every request through the proxy is refused at
|
|
// the handshake.
|
|
func TestTheClusterIsToldWhichCertificateToTrust(t *testing.T) {
|
|
pem := "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n"
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
r.opt.ControlPlaneCert = pem
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForState(t, f, "ready")
|
|
|
|
got, err := os.ReadFile(filepath.Join(r.Dir(7), "cluster.auto.tfvars"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Quoted, so the newlines survive into HCL as escapes rather than
|
|
// ending the string at the first one.
|
|
want := `control_plane_cert = "-----BEGIN CERTIFICATE-----\nMIIB\n-----END CERTIFICATE-----\n"`
|
|
if !strings.Contains(string(got), want) {
|
|
t.Errorf("the certificate was not carried into the variables:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// A portal with no certificate of its own says nothing about one, rather
|
|
// than writing an empty value that would leave the cluster requiring a
|
|
// client certificate it can never verify.
|
|
func TestNoPortalCertificateMeansNoVariable(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForState(t, f, "ready")
|
|
|
|
got, err := os.ReadFile(filepath.Join(r.Dir(7), "cluster.auto.tfvars"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(got), "control_plane_cert") {
|
|
t.Errorf("an empty certificate was written anyway:\n%s", got)
|
|
}
|
|
}
|
|
|
|
// stateful gives a cluster's directory a non-empty state file, which is how
|
|
// Destroy tells "built" from "never built".
|
|
func stateful(t *testing.T, r *Runner, id int64) {
|
|
t.Helper()
|
|
dir := r.Dir(id)
|
|
if err := os.MkdirAll(dir, 0o750); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(dir, "terraform.tfstate"),
|
|
[]byte(`{"resources":[{}]}`), 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func waitFor(t *testing.T, cond func() bool, what string) {
|
|
t.Helper()
|
|
for i := 0; i < 200; i++ {
|
|
if cond() {
|
|
return
|
|
}
|
|
time.Sleep(20 * time.Millisecond)
|
|
}
|
|
t.Fatalf("timed out waiting for %s", what)
|
|
}
|
|
|
|
// Releasing used to be a database write on its own: the record said the
|
|
// hardware was free while the machines were still running on it.
|
|
func TestReleasingDestroysTheMachinesFirst(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
stateful(t, r, 7)
|
|
|
|
if err := r.Destroy(aCluster(), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFor(t, func() bool { return f.releases() == 1 }, "the record to be released")
|
|
|
|
log, err := os.ReadFile(r.LogPath(7))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(log), "called: destroy") {
|
|
t.Errorf("the machines were not destroyed:\n%s", log)
|
|
}
|
|
if got := f.seen(); len(got) == 0 || got[0] != "deleting" {
|
|
t.Errorf("states = %v, want it to start at deleting", got)
|
|
}
|
|
}
|
|
|
|
// The one state nothing can recover from is machines that still exist and a
|
|
// record saying nobody owns them. A failed destroy keeps the cluster held.
|
|
func TestAFailedDestroyDoesNotGiveTheHardwareBack(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 1) // the fake tofu exits non-zero
|
|
stateful(t, r, 7)
|
|
|
|
if err := r.Destroy(aCluster(), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFor(t, func() bool {
|
|
for _, s := range f.seen() {
|
|
if s == "failed" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}, "the destroy to fail")
|
|
|
|
if n := f.releases(); n != 0 {
|
|
t.Errorf("the record was released %d time(s) after a failed destroy", n)
|
|
}
|
|
}
|
|
|
|
// A cluster that was never built has nothing to tear down, and waiting a
|
|
// minute for Terraform to say so turns releasing a mistyped request into a
|
|
// wait.
|
|
func TestReleasingSomethingNeverBuiltIsImmediate(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 1) // would fail if it ran at all
|
|
|
|
if err := r.Destroy(aCluster(), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n := f.releases(); n != 1 {
|
|
t.Errorf("releases = %d, want 1 straight away", n)
|
|
}
|
|
if got := f.seen(); len(got) != 0 {
|
|
t.Errorf("states = %v, want none: nothing was being deleted", got)
|
|
}
|
|
if _, err := os.Stat(r.LogPath(7)); err == nil {
|
|
t.Error("Terraform was run for a cluster that had never been built")
|
|
}
|
|
}
|
|
|
|
// Two releases at once, or a release while a build is in flight, would have
|
|
// two Terraform runs in one directory.
|
|
func TestOnlyOneRunPerClusterAtATime(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
stateful(t, r, 7)
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := r.Destroy(aCluster(), false); err != ErrAlreadyRunning {
|
|
t.Errorf("a release during a build returned %v, want ErrAlreadyRunning", err)
|
|
}
|
|
// Let the build finish before the temporary directory goes away, or
|
|
// the cleanup races a process still writing into it.
|
|
waitFor(t, func() bool { return !r.Running(7) }, "the build to finish")
|
|
}
|
|
|
|
// A build lives in a goroutine. Restart the portal and the goroutine is
|
|
// gone while the record still says "provisioning" - and nothing will ever
|
|
// move it again. That is not a lie anyone notices; the state just stops
|
|
// changing, and the page goes on showing a build that is not happening.
|
|
func TestWorkInFlightAtStartupIsMarkedFailed(t *testing.T) {
|
|
f := &fakeStore{live: []store.Cluster{
|
|
{SerialID: 7, AccountingName: "c003-7", State: "provisioning"},
|
|
{SerialID: 8, AccountingName: "c004-8", State: "deleting"},
|
|
{SerialID: 9, AccountingName: "c005-9", State: "ready"},
|
|
{SerialID: 10, AccountingName: "c006-10", State: "requested"},
|
|
}}
|
|
r := testRunner(t, f, 0)
|
|
|
|
n, err := r.Recover(context.Background())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n != 2 {
|
|
t.Errorf("recovered %d, want the two that were moving", n)
|
|
}
|
|
if got := f.seen(); len(got) != 2 || got[0] != "failed" || got[1] != "failed" {
|
|
t.Errorf("states written = %v, want two failures", got)
|
|
}
|
|
}
|
|
|
|
// And it says so in the log, because "failed" with nothing to read is the
|
|
// thing this exists to avoid.
|
|
func TestRecoveryLeavesANoteInTheLog(t *testing.T) {
|
|
f := &fakeStore{live: []store.Cluster{
|
|
{SerialID: 7, AccountingName: "c003-7", State: "provisioning"},
|
|
}}
|
|
r := testRunner(t, f, 0)
|
|
stateful(t, r, 7) // gives the directory the log can be written into
|
|
|
|
if _, err := r.Recover(context.Background()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, err := os.ReadFile(r.LogPath(7))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(body), "the portal restarted") {
|
|
t.Errorf("the log does not say what happened:\n%s", body)
|
|
}
|
|
}
|
|
|
|
// Nothing in flight, nothing touched: a restart on a quiet estate must not
|
|
// rewrite records that were fine.
|
|
func TestRecoveryLeavesSettledClustersAlone(t *testing.T) {
|
|
f := &fakeStore{live: []store.Cluster{
|
|
{SerialID: 9, State: "ready"},
|
|
{SerialID: 10, State: "failed"},
|
|
{SerialID: 11, State: "requested"},
|
|
}}
|
|
r := testRunner(t, f, 0)
|
|
|
|
n, err := r.Recover(context.Background())
|
|
if err != nil || n != 0 {
|
|
t.Fatalf("recovered %d (err %v), want none", n, err)
|
|
}
|
|
if got := f.seen(); len(got) != 0 {
|
|
t.Errorf("states written = %v, want none", got)
|
|
}
|
|
}
|
|
|
|
// The compute nodes are created switched off, because they PXE boot from
|
|
// the head and there is nothing to boot from until Warewulf on it is
|
|
// configured. Nothing used to switch them on afterwards: the cluster
|
|
// reported ready, published, served a desktop request, and queued it for
|
|
// ever behind four nodes that were off.
|
|
func TestTheComputeNodesArePoweredOnAfterTheHeadIsUp(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForState(t, f, "ready")
|
|
|
|
log, err := os.ReadFile(r.LogPath(7))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body := string(log)
|
|
if !strings.Contains(body, "called: apply -auto-approve -input=false -no-color -var started=true") {
|
|
t.Errorf("the compute nodes were never switched on:\n%s", body)
|
|
}
|
|
// And in this order: switching them on before the head can serve them
|
|
// a boot image is a node that PXE boots into nothing.
|
|
wait := strings.Index(body, "waiting for the portal")
|
|
power := strings.Index(body, "started=true")
|
|
if wait < 0 || power < 0 || wait > power {
|
|
t.Errorf("powered on at %d, waited for the head at %d - the order is wrong", power, wait)
|
|
}
|
|
}
|
|
|
|
// Waiting for the head is not part of publishing. It used to be, so a
|
|
// portal with no registry configured neither waited nor powered anything
|
|
// on - it built the machines and called it ready.
|
|
func TestTheHeadIsWaitedForEvenWithNoRegistry(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
waited := false
|
|
r.opt.headReady = func(context.Context, string, io.Writer) error { waited = true; return nil }
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForState(t, f, "ready")
|
|
|
|
if !waited {
|
|
t.Error("the build never waited for the head node")
|
|
}
|
|
}
|
|
|
|
// The tenant's work lives on the head node's own disk. A release that
|
|
// destroys the head before taking a copy is irreversible in the one
|
|
// direction that matters, so a failed archive stops the release.
|
|
func TestAFailedArchiveStopsTheRelease(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
stateful(t, r, 7)
|
|
r.opt.SSHKey = "" // nothing to read the head with
|
|
|
|
if err := r.Destroy(aCluster(), true); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFor(t, func() bool {
|
|
for _, s := range f.seen() {
|
|
if s == "failed" {
|
|
return true
|
|
}
|
|
}
|
|
return false
|
|
}, "the release to fail")
|
|
|
|
if n := f.releases(); n != 0 {
|
|
t.Errorf("the record was released %d time(s) despite losing the data", n)
|
|
}
|
|
log, err := os.ReadFile(r.LogPath(7))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if strings.Contains(string(log), "called: destroy") {
|
|
t.Error("the machines were destroyed even though the archive failed")
|
|
}
|
|
if !strings.Contains(string(log), "the data is still only on the head") {
|
|
t.Errorf("the log does not say why it stopped:\n%s", log)
|
|
}
|
|
}
|
|
|
|
// And "discard" is the way past it, for data nobody wants or a head too
|
|
// broken to read - otherwise a failed archive leaves a cluster nobody can
|
|
// get rid of.
|
|
func TestDiscardReleasesWithoutArchiving(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs()}
|
|
r := testRunner(t, f, 0)
|
|
stateful(t, r, 7)
|
|
r.opt.SSHKey = ""
|
|
|
|
if err := r.Destroy(aCluster(), false); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitFor(t, func() bool { return f.releases() == 1 }, "the record to be released")
|
|
|
|
log, err := os.ReadFile(r.LogPath(7))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !strings.Contains(string(log), "called: destroy") {
|
|
t.Errorf("the machines were not destroyed:\n%s", log)
|
|
}
|
|
if !strings.Contains(string(log), "without keeping /home") {
|
|
t.Errorf("the log does not record the choice:\n%s", log)
|
|
}
|
|
}
|
|
|
|
// A file that exists is not an archive. tar over ssh reports success from
|
|
// the local end of a pipe, so a truncated stream leaves something that
|
|
// looks like a result and is not one.
|
|
func TestATruncatedArchiveIsNotAccepted(t *testing.T) {
|
|
dir := t.TempDir()
|
|
good := filepath.Join(dir, "good.tgz")
|
|
if err := exec.Command("sh", "-c",
|
|
"printf hello > "+filepath.Join(dir, "f")+" && tar czf "+good+" -C "+dir+" f").Run(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
// Not an exact count: macOS tar adds an AppleDouble twin for a file
|
|
// with extended attributes, and what matters here is that a real
|
|
// archive reads and a broken one does not.
|
|
if _, n, err := archiveHomes(good); err != nil || n < 1 {
|
|
t.Fatalf("a real archive read as %d entries, %v", n, err)
|
|
}
|
|
|
|
body, err := os.ReadFile(good)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cut := filepath.Join(dir, "cut.tgz")
|
|
if err := os.WriteFile(cut, body[:len(body)/2], 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := archiveHomes(cut); err == nil {
|
|
t.Error("half an archive was accepted as one")
|
|
}
|
|
|
|
empty := filepath.Join(dir, "empty.tgz")
|
|
if err := os.WriteFile(empty, nil, 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := archiveHomes(empty); err == nil {
|
|
t.Error("an empty file was accepted as an archive")
|
|
}
|
|
}
|
|
|
|
// A build that cannot even prepare its directory still has to say why.
|
|
// This one used to leave nothing at all: no log, because the log was
|
|
// opened after prepare, and no journal line, because the goroutine
|
|
// dropped the error - so a cluster went to "failed" with a Retry button
|
|
// and no reason, and finding the cause meant reproducing it by hand.
|
|
func TestAPrepareFailureIsStillWrittenDown(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs(), err: errors.New("relation \"cluster_app\" does not exist")}
|
|
r := testRunner(t, f, 0)
|
|
|
|
if err := r.Start(aCluster()); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
waitForState(t, f, "failed")
|
|
|
|
body, err := os.ReadFile(r.LogPath(7))
|
|
if err != nil {
|
|
t.Fatalf("a failed build left no log at all: %v", err)
|
|
}
|
|
if !strings.Contains(string(body), "could not prepare") ||
|
|
!strings.Contains(string(body), "cluster_app") {
|
|
t.Errorf("the log does not say what went wrong:\n%s", body)
|
|
}
|
|
}
|
|
|
|
// The release that could not happen. A desktop was open, XFCE held
|
|
// recently-used.xbel, tar exited non-zero, and a complete archive of
|
|
// everybody's work was thrown away unread - so a cluster nobody wanted
|
|
// could not be destroyed.
|
|
//
|
|
// What decides now is whose work is in the archive, not what tar's exit
|
|
// code was.
|
|
func TestAnArchiveIsJudgedByWhatIsInItNotByTarsExitCode(t *testing.T) {
|
|
dir := t.TempDir()
|
|
home := filepath.Join(dir, "home")
|
|
for _, u := range []string{"alice", "bob"} {
|
|
if err := os.MkdirAll(filepath.Join(home, u), 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(home, u, "thesis.tex"), []byte("work"), 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
out := filepath.Join(dir, "home.tgz")
|
|
if err := exec.Command("tar", "czf", out, "-C", home, ".").Run(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
|
|
homes, n, err := archiveHomes(out)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if n < 2 {
|
|
t.Fatalf("only %d entries", n)
|
|
}
|
|
for _, u := range []string{"alice", "bob"} {
|
|
if !homes[u] {
|
|
t.Errorf("%s's home is not recognised in the archive: %v", u, homes)
|
|
}
|
|
}
|
|
// Somebody who was never there must not appear, or the completeness
|
|
// check would pass for an archive that is missing a whole home.
|
|
if homes["carol"] {
|
|
t.Error("a home that does not exist was found in the archive")
|
|
}
|
|
}
|
|
|
|
// A truncated stream must still be refused, and for the same reason as
|
|
// before: that is a tar which died in the middle, which is the one failure
|
|
// that must never be mistaken for a complete copy.
|
|
func TestATruncatedArchiveStillFailsEvenWhenHomesAppear(t *testing.T) {
|
|
dir := t.TempDir()
|
|
home := filepath.Join(dir, "home", "alice")
|
|
if err := os.MkdirAll(home, 0o755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
big := make([]byte, 400*1024)
|
|
for i := range big {
|
|
big[i] = byte(i)
|
|
}
|
|
if err := os.WriteFile(filepath.Join(home, "data"), big, 0o644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
out := filepath.Join(dir, "home.tgz")
|
|
if err := exec.Command("tar", "czf", out, "-C", filepath.Join(dir, "home"), ".").Run(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, err := os.ReadFile(out)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
cut := filepath.Join(dir, "cut.tgz")
|
|
if err := os.WriteFile(cut, body[:len(body)/2], 0o640); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, _, err := archiveHomes(cut); err == nil {
|
|
t.Error("a truncated archive was accepted because a home directory appeared in it")
|
|
}
|
|
}
|
|
|
|
// An archive that cannot say how many machines it came from or how long
|
|
// they ran is a directory of files somebody will be guessing about in six
|
|
// months. The manifest travels with the tarball, which the database
|
|
// cannot: the archive can be moved off this host.
|
|
func TestTheArchiveIsDescribedBesideIt(t *testing.T) {
|
|
f := &fakeStore{vms: goodVMs(), usage: &store.Usage{
|
|
AccountingName: "c003-7", TenantName: "acme", ProjectName: "protein-folding",
|
|
Site: "hv01", Cores: 6, MemoryGB: 48,
|
|
Wall: 2*time.Hour + 9*time.Minute, CoreHours: 12.97, GBHours: 103.74,
|
|
}}
|
|
r := testRunner(t, f, 0)
|
|
stateful(t, r, 7)
|
|
r.opt.SSHKey = t.TempDir() + "/key" // archiveHome will fail; see below
|
|
|
|
// The manifest is written only on the path where the data was kept,
|
|
// so drive the real one with a working archive step.
|
|
if err := r.writeManifest(context.Background(), aCluster(), r.Dir(7), io.Discard); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
body, err := os.ReadFile(filepath.Join(r.Dir(7), "manifest.json"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
var got struct {
|
|
Cluster string `json:"cluster"`
|
|
Project string `json:"project"`
|
|
Wall string `json:"wall"`
|
|
CoreHours float64 `json:"core_hours"`
|
|
Nodes []struct {
|
|
Role string `json:"role"`
|
|
Cores int `json:"cores"`
|
|
} `json:"nodes"`
|
|
}
|
|
if err := json.Unmarshal(body, &got); err != nil {
|
|
t.Fatalf("the manifest is not JSON: %v\n%s", err, body)
|
|
}
|
|
if got.Cluster != "c003-7" || got.Project != "protein-folding" {
|
|
t.Errorf("cluster=%q project=%q", got.Cluster, got.Project)
|
|
}
|
|
if got.CoreHours != 12.97 || got.Wall == "" {
|
|
t.Errorf("cost not carried: %.2f core-hours, wall %q", got.CoreHours, got.Wall)
|
|
}
|
|
if len(got.Nodes) != len(goodVMs()) {
|
|
t.Errorf("%d nodes described, want %d", len(got.Nodes), len(goodVMs()))
|
|
}
|
|
}
|