An installation used to be this one: the deploy scripts and the Ansible defaults carried one site's domain, hypervisor and login, the Proxmox side and the database were set up by hand, and deploying meant copying scripts and a tree of roles beside the binary and keeping them in step with it. The binary now carries the roles, the Terraform module, the schema and the catalogue (embed.go), and installs itself: - hpcod setup: a full-screen installer for administrators. It asks where clusters are built, signs in to Proxmox before asking anything else (root's password once, or tokens), offers nodes and datastores from what Proxmox has, checks every field as it is typed, runs every step with its output live, checks the result and offers a test cluster. -yes runs the same steps from a site file; make deploy uses that. - hpcod verify: each fact checked by using it. - hpcod selftest: the smallest real cluster, by the portal's own path, a job run on it, then taken down. - Closed networks: hpcod serves HTTPS itself from a CA name-constrained to the site's domains, issuing and renewing its own certificate, and answers DNS for the portal and every cluster name. - A simulated Proxmox: Proxmox VE in QEMU on the control plane host, on a private bridge with NAT, for trying it without one. Every step reads back what it changed and is safe to run again. The site's values live in one file; group_vars/all/site.yml is generated from it, and the playbooks refuse to run without it. ci_user and -ssh-user no longer default to a personal account. scripts/B10, B20 and C20 are gone: their work is the setup steps. Proven end to end on a simulated Proxmox (Proxmox VE 9 in a VM on a real host): the install, the Proxmox step by the password path, the head template, verify READY, and a test cluster that ran srun on its compute node and was destroyed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
365 lines
13 KiB
Go
365 lines
13 KiB
Go
package httpapi
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"io"
|
|
"log"
|
|
"net/http"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/ilkermanap/hpcod/internal/catalog"
|
|
"github.com/ilkermanap/hpcod/internal/model"
|
|
"github.com/ilkermanap/hpcod/internal/placement"
|
|
"github.com/ilkermanap/hpcod/internal/spec"
|
|
"github.com/ilkermanap/hpcod/internal/store"
|
|
)
|
|
|
|
// Creating a cluster is the first route here with a side effect. Everything
|
|
// it does before writing is the same work /v1/dry-run does, on purpose: if
|
|
// the two disagreed, a dry run would stop being worth running.
|
|
//
|
|
// What it does not do is provision. It settles what the cluster is - its
|
|
// address, its accounting name, which hypervisors and which cards - and
|
|
// records that, leaving the cluster in state "requested". Building it is a
|
|
// separate step that reads this record, because building takes minutes and
|
|
// an HTTP request should not be holding the decision in memory while it
|
|
// happens.
|
|
|
|
// errInternal marks a failure that is ours, not the caller's, so a shared
|
|
// helper's error can still be given the right status code.
|
|
var errInternal = errors.New("internal error")
|
|
|
|
func statusFor(err error) int {
|
|
if errors.Is(err, errInternal) {
|
|
return http.StatusInternalServerError
|
|
}
|
|
return http.StatusBadRequest
|
|
}
|
|
|
|
// clusterStore is the slice of the store these routes use. It is an
|
|
// interface so the handlers can be tested without a PostgreSQL: what is
|
|
// worth checking here is status codes and what gets dropped in translation,
|
|
// and neither of those is a database question.
|
|
type clusterStore interface {
|
|
Reservations(ctx context.Context) ([]model.Reservation, error)
|
|
EnsureTenant(ctx context.Context, name, owner string) (int64, error)
|
|
AllocateCluster(ctx context.Context, tenantID int64, p store.Placement, lease time.Duration, apps []store.ClusterApp, projectID *int64) (*store.Cluster, error)
|
|
Live(ctx context.Context) ([]store.Cluster, error)
|
|
VMs(ctx context.Context, serialID int64) ([]store.VM, error)
|
|
Release(ctx context.Context, serialID int64) error
|
|
SetLease(ctx context.Context, serialID int64, expires *time.Time) error
|
|
|
|
CreateAccount(ctx context.Context, username, passwordHash, email, reason string) (*store.Account, error)
|
|
AccountWithHash(ctx context.Context, username string) (*store.Account, string, error)
|
|
Accounts(ctx context.Context) ([]store.Account, error)
|
|
DecideAccount(ctx context.Context, username, state, by string) (*store.Account, error)
|
|
SetAdmin(ctx context.Context, username string, admin bool) (*store.Account, error)
|
|
DeleteAccount(ctx context.Context, username string) error
|
|
|
|
// Projects group a tenant's clusters over time, and the usage view
|
|
// is what says what each one cost. Both outlive the machines.
|
|
CreateProject(ctx context.Context, tenant, owner, name, description, by string) (*store.Project, error)
|
|
Projects(ctx context.Context, tenant string) ([]store.Project, error)
|
|
CloseProject(ctx context.Context, id int64, tenant string) error
|
|
Usage(ctx context.Context, tenant string, limit int) ([]store.Usage, error)
|
|
UsageFor(ctx context.Context, serialID int64) (*store.Usage, error)
|
|
|
|
Policies(ctx context.Context) (map[string]model.Overcommit, error)
|
|
SetPolicy(ctx context.Context, o model.Overcommit, by string) (*model.Overcommit, error)
|
|
|
|
Close()
|
|
}
|
|
|
|
// decide runs the shared path: fold in the applications, subtract what is
|
|
// already promised, and plan.
|
|
func (s *Server) decide(ctx context.Context, req spec.Request) (spec.Request, []catalog.App, placement.Decision, error) {
|
|
req, apps, err := s.catalog.Current().Expand(req)
|
|
if err != nil {
|
|
return req, nil, placement.Decision{}, err
|
|
}
|
|
|
|
sites, _ := s.cache.get()
|
|
if s.store != nil {
|
|
res, err := s.store.Reservations(ctx)
|
|
if err != nil {
|
|
// Planning against an inventory that ignores reservations would
|
|
// hand out hardware twice, so this is fatal to the request
|
|
// rather than a degraded answer. Marked as ours, because a
|
|
// database the caller cannot see is not their mistake to fix -
|
|
// reported as a 400 it would send them rewriting a request that
|
|
// was never wrong.
|
|
return req, nil, placement.Decision{}, fmt.Errorf("%w: reading reservations: %w", errInternal, err)
|
|
}
|
|
// The cache is shared with every other reader, so mark a copy.
|
|
sites = copySites(sites)
|
|
model.ApplyReservations(sites, res)
|
|
}
|
|
return req, apps, placement.Plan(sites, req, placement.DefaultWeights()), nil
|
|
}
|
|
|
|
// copySites deep-copies as far as the device slices, which is the only part
|
|
// ApplyReservations writes to.
|
|
func copySites(in []model.Site) []model.Site {
|
|
out := make([]model.Site, len(in))
|
|
copy(out, in)
|
|
for i := range out {
|
|
nodes := make([]model.Node, len(out[i].Nodes))
|
|
copy(nodes, out[i].Nodes)
|
|
for j := range nodes {
|
|
devs := make([]model.Device, len(nodes[j].Devices))
|
|
copy(devs, nodes[j].Devices)
|
|
nodes[j].Devices = devs
|
|
}
|
|
out[i].Nodes = nodes
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (s *Server) jsonClusters(w http.ResponseWriter, r *http.Request) {
|
|
if s.store == nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]string{
|
|
"error": "no database configured; clusters cannot be recorded (set -db or $HPCOD_DB_DSN)"})
|
|
return
|
|
}
|
|
switch r.Method {
|
|
case http.MethodGet:
|
|
s.listClusters(w, r)
|
|
case http.MethodPost:
|
|
s.createCluster(w, r)
|
|
default:
|
|
w.Header().Set("Allow", "GET, POST")
|
|
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "GET to list, POST to create"})
|
|
}
|
|
}
|
|
|
|
func (s *Server) listClusters(w http.ResponseWriter, r *http.Request) {
|
|
clusters, err := s.store.Live(r.Context())
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
out := make([]map[string]any, 0, len(clusters))
|
|
for _, c := range clusters {
|
|
vms, err := s.store.VMs(r.Context(), c.SerialID)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
out = append(out, clusterJSON(c, vms))
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"clusters": out})
|
|
}
|
|
|
|
func (s *Server) createCluster(w http.ResponseWriter, r *http.Request) {
|
|
body, err := io.ReadAll(io.LimitReader(r.Body, 256<<10))
|
|
if err != nil {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
req, err := spec.Parse(body)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
if strings.TrimSpace(req.Tenant) == "" {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{
|
|
"error": "tenant is required to create a cluster (it is not, to plan one)"})
|
|
return
|
|
}
|
|
owner := strings.TrimSpace(req.Owner)
|
|
if owner == "" {
|
|
owner = userFromContext(r.Context())
|
|
}
|
|
if owner == "" {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{
|
|
"error": "owner is required when nobody is signed in"})
|
|
return
|
|
}
|
|
lease, err := req.LeaseDuration()
|
|
if err != nil {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
|
|
req, apps, decision, err := s.decide(r.Context(), req)
|
|
if err != nil {
|
|
writeJSON(w, statusFor(err), map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
if !decision.Feasible {
|
|
// 409, not 400: the request is well formed and may well succeed
|
|
// tomorrow. The candidates say what is in the way.
|
|
writeJSON(w, http.StatusConflict, map[string]any{
|
|
"error": decision.Summary, "decision": decision})
|
|
return
|
|
}
|
|
|
|
tenantID, err := s.store.EnsureTenant(r.Context(), req.Tenant, owner)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
|
|
// No project from the JSON API: the request document has no field for
|
|
// one, and attributing work to a guess is worse than leaving it
|
|
// unattributed.
|
|
c, err := s.store.AllocateCluster(r.Context(), tenantID, store.PlacementFrom(*decision.Chosen), lease, toClusterApps(apps), nil)
|
|
switch {
|
|
case errors.Is(err, store.ErrDeviceHeld), errors.Is(err, store.ErrNoFreeID):
|
|
// Both mean the estate moved between planning and writing. The
|
|
// caller should plan again rather than be handed a half-truth.
|
|
writeJSON(w, http.StatusConflict, map[string]string{"error": err.Error()})
|
|
return
|
|
case err != nil:
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
|
|
vms, err := s.store.VMs(r.Context(), c.SerialID)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
log.Printf("cluster %s allocated to tenant %s on %s by %s",
|
|
c.AccountingName, req.Tenant, c.Site, owner)
|
|
|
|
out := clusterJSON(*c, vms)
|
|
out["tenant"] = req.Tenant
|
|
out["apps"] = appRefs(apps)
|
|
out["note"] = "allocated, not built: state is \"requested\" until a provisioner acts on this record"
|
|
writeJSON(w, http.StatusCreated, out)
|
|
}
|
|
|
|
// jsonCluster handles one cluster by its serial id: /v1/clusters/{id}.
|
|
func (s *Server) jsonCluster(w http.ResponseWriter, r *http.Request) {
|
|
if s.store == nil {
|
|
writeJSON(w, http.StatusServiceUnavailable, map[string]string{
|
|
"error": "no database configured; clusters cannot be recorded (set -db or $HPCOD_DB_DSN)"})
|
|
return
|
|
}
|
|
idStr := strings.TrimPrefix(r.URL.Path, "/v1/clusters/")
|
|
id, err := strconv.ParseInt(idStr, 10, 64)
|
|
if err != nil || id <= 0 {
|
|
writeJSON(w, http.StatusBadRequest, map[string]string{
|
|
"error": "expected /v1/clusters/{serial_id}, a positive integer"})
|
|
return
|
|
}
|
|
|
|
switch r.Method {
|
|
case http.MethodDelete:
|
|
if err := s.store.Release(r.Context(), id); err != nil {
|
|
writeJSON(w, http.StatusNotFound, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
log.Printf("cluster %d released by %s", id, orAnon(userFromContext(r.Context())))
|
|
writeJSON(w, http.StatusOK, map[string]any{
|
|
"released": id,
|
|
"note": "the record and its accounting name remain; the address and any devices are free again",
|
|
})
|
|
case http.MethodGet:
|
|
vms, err := s.store.VMs(r.Context(), id)
|
|
if err != nil {
|
|
writeJSON(w, http.StatusInternalServerError, map[string]string{"error": err.Error()})
|
|
return
|
|
}
|
|
if len(vms) == 0 {
|
|
writeJSON(w, http.StatusNotFound, map[string]string{"error": "no such cluster"})
|
|
return
|
|
}
|
|
writeJSON(w, http.StatusOK, map[string]any{"serial_id": id, "vms": vms})
|
|
default:
|
|
w.Header().Set("Allow", "GET, DELETE")
|
|
writeJSON(w, http.StatusMethodNotAllowed, map[string]string{"error": "GET or DELETE"})
|
|
}
|
|
}
|
|
|
|
func clusterJSON(c store.Cluster, vms []store.VM) map[string]any {
|
|
return map[string]any{
|
|
"serial_id": c.SerialID,
|
|
"cluster_id": c.ClusterID,
|
|
"accounting_name": c.AccountingName,
|
|
"site": c.Site,
|
|
"state": c.State,
|
|
"lease_expires_at": c.LeaseExpiresAt,
|
|
"created_at": c.CreatedAt,
|
|
"vms": vms,
|
|
}
|
|
}
|
|
|
|
func appRefs(apps []catalog.App) []string {
|
|
out := make([]string, 0, len(apps))
|
|
for _, a := range apps {
|
|
out = append(out, a.Ref())
|
|
}
|
|
return out
|
|
}
|
|
|
|
// toClusterApps narrows what the catalogue resolved to what a cluster needs
|
|
// recorded: a name and a URI. The capability an application implies has
|
|
// already done its work by this point - placement read it - and the size and
|
|
// signature describe the image rather than this cluster's claim on it.
|
|
func toClusterApps(apps []catalog.App) []store.ClusterApp {
|
|
out := make([]store.ClusterApp, 0, len(apps))
|
|
for _, a := range apps {
|
|
out = append(out, store.ClusterApp{
|
|
Name: a.Name, Image: a.Image,
|
|
Digest: a.Digest, SIFSHA256: a.SIFSHA256,
|
|
Version: a.Version, Module: a.Module, Binaries: a.Binaries,
|
|
})
|
|
}
|
|
return out
|
|
}
|
|
|
|
func orAnon(u string) string {
|
|
if u == "" {
|
|
return "an unauthenticated caller"
|
|
}
|
|
return u
|
|
}
|
|
|
|
// createFromPlan turns a decision the person has just been shown into a
|
|
// cluster. It is the page's half of POST /v1/clusters and shares its
|
|
// rules, so the two cannot drift into allocating on different terms.
|
|
//
|
|
// The tenant defaults to whoever is signed in. Someone asking for their
|
|
// own cluster should not have to name themselves, and letting the form
|
|
// name somebody else would be a way to spend another tenant's quota.
|
|
func (s *Server) createFromPlan(r *http.Request, req spec.Request, apps []catalog.App, dec placement.Decision, projectID *int64) (*store.Cluster, error) {
|
|
who := userFromContext(r.Context())
|
|
if who == "" {
|
|
return nil, errors.New("sign in before creating a cluster")
|
|
}
|
|
if s.store == nil {
|
|
return nil, errors.New("no database is configured, so nothing can be recorded")
|
|
}
|
|
if !dec.Feasible {
|
|
return nil, errors.New(dec.Summary)
|
|
}
|
|
lease, err := req.LeaseDuration()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
tenant := strings.TrimSpace(req.Tenant)
|
|
if tenant == "" {
|
|
tenant = who
|
|
} else if tenant != who && !isAdmin(r.Context()) {
|
|
// Only an administrator may create on someone else's behalf.
|
|
return nil, fmt.Errorf("you cannot create a cluster for %q", tenant)
|
|
}
|
|
owner := strings.TrimSpace(req.Owner)
|
|
if owner == "" {
|
|
owner = who
|
|
}
|
|
|
|
tenantID, err := s.store.EnsureTenant(r.Context(), tenant, owner)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
return s.store.AllocateCluster(r.Context(), tenantID, store.PlacementFrom(*dec.Chosen), lease, toClusterApps(apps), projectID)
|
|
}
|