Files
ilkerandClaude Opus 5 a760db8926 hpcod: keep the tenant's work when the cluster that held it goes
/home lives on the head node's own disk, so releasing a cluster threw
it away. Release now takes a copy first: compressed on the head and
streamed here, because the head's CPU is about to be discarded and
the bytes crossing the management network should be the small ones.
Caches are excluded by name - a Spack or conda build tree under a
home directory is larger than everything worth keeping in it.

A failed archive stops the release. Destroying anyway is
irreversible in the one direction that matters, and "the cluster is
still here, and here is why" is recoverable. Posting discard is the
way past it: for data nobody wants, or a head too broken to read,
which would otherwise leave a cluster nobody can get rid of.

The archive is read back as an archive before the machines go. tar
over ssh reports success from the local end of a pipe, so a truncated
stream, a connection refused after the first block or a sudo that
asked for a password all leave a file that exists and is not one.

Two timing faults in waitForPortal came out of testing this. The dial
was bounded only by the whole build's context, and a dial to an
address that routes nowhere hangs for about seventy-five seconds
rather than failing - a few of those swallow a ten-minute budget
without the loop noticing it is being starved. The pause between
attempts ignored the budget too: a 200ms wait with a 10s interval
slept the full ten seconds before rechecking a deadline that had
passed in the first fifth of a second. Both are now bounded by what
is left.

This is a safety net at teardown, not a data strategy: it does
nothing for a head that dies, and nothing for someone who wants their
files while the cluster runs. Storage is deliberately unbounded for
now - the estate is a proof of concept - so there is no quota and no
retention yet.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-22 08:52:33 +02:00

519 lines
16 KiB
Go

package httpapi
import (
"fmt"
"log"
"net/http"
"os"
"sort"
"strconv"
"strings"
"time"
"github.com/ilkermanap/hpcod/internal/model"
"github.com/ilkermanap/hpcod/internal/store"
)
// The screens for what has actually been handed out.
//
// Everything the portal showed until now was about the estate: what
// hardware exists, and where a request would go. None of it said what is
// running, who has it, or when it expires - the answers an operator
// actually opens a portal for. The API learned those first; this is the
// same information without a terminal.
// clusterView is one row, with the parts a page needs precomputed. Doing
// this in Go rather than in the template keeps the arithmetic somewhere it
// can be tested.
type clusterView struct {
store.Cluster
// PortalURL is where the cluster's own Open OnDemand lives. Empty
// until the cluster is published, because a link that leads to "no
// cluster is published here" is worse than no link.
PortalURL string
MayAct bool
Building bool
HasLog bool
VMs []store.VM
Hosts string // distinct hypervisors, for the common one-host case
Devices []string
LeaseIn string // "6 days", "expired", or "" when there is no lease
Expired bool
Expiring bool // inside a day
}
func makeClusterView(c store.Cluster, vms []store.VM) clusterView {
v := clusterView{Cluster: c, VMs: vms}
hosts := map[string]bool{}
for _, vm := range vms {
hosts[vm.HostNode] = true
for _, d := range vm.Devices {
v.Devices = append(v.Devices, fmt.Sprintf("%s %s", d.Capability, d.NativeID))
}
}
names := make([]string, 0, len(hosts))
for h := range hosts {
names = append(names, h)
}
sort.Strings(names)
v.Hosts = strings.Join(names, ", ")
if left, ok := c.LeaseLeft(); ok {
switch {
case left <= 0:
v.LeaseIn, v.Expired = "expired", true
case left < 24*time.Hour:
v.LeaseIn, v.Expiring = humanDuration(left), true
default:
v.LeaseIn = humanDuration(left)
}
}
return v
}
// humanDuration is deliberately coarse. A lease measured to the second
// invites reading it as a promise, and nothing here reclaims anything on a
// schedule that fine.
func humanDuration(d time.Duration) string {
switch {
case d >= 48*time.Hour:
return fmt.Sprintf("%d days", int(d.Hours()/24))
case d >= 2*time.Hour:
return fmt.Sprintf("%d hours", int(d.Hours()))
case d >= time.Minute:
return fmt.Sprintf("%d min", int(d.Minutes()))
default:
return "under a minute"
}
}
func (s *Server) clusterViews(r *http.Request) ([]clusterView, error) {
if s.store == nil {
return nil, nil
}
clusters, err := s.store.Live(r.Context())
if err != nil {
return nil, err
}
// Who is allowed to see a cluster at all.
//
// The list used to be the whole estate for everyone signed in, which
// is right for an operator and wrong for a tenant: the row names the
// tenant holding it, the hypervisor it sits on and the PCI devices it
// is keeping. None of that is another tenant's business, and a portal
// that shows it has handed out an inventory of who runs what.
//
// With authentication disabled the portal is loopback-only and there
// is nobody to be signed in as, so the filter would hide everything
// from the only person who can reach it.
all := s.auth == nil || isAdmin(r.Context())
who := userFromContext(r.Context())
out := make([]clusterView, 0, len(clusters))
for _, c := range clusters {
if !all && c.TenantName != who {
continue
}
vms, err := s.store.VMs(r.Context(), c.SerialID)
if err != nil {
return nil, err
}
v := makeClusterView(c, vms)
v.MayAct = mayAct(r, c)
if v.MayAct {
// Same rule as mayUseCluster, which is what the proxy
// enforces when the link is followed. Showing it to
// anyone else names a cluster that is not theirs,
// publishes the hostname it answers on, and offers a
// click the proxy will refuse - a page promising
// something the system will not honour.
v.PortalURL = s.portalURL(c)
}
if s.builder != nil {
v.Building = s.builder.Running(c.SerialID)
if _, err := os.Stat(s.builder.LogPath(c.SerialID)); err == nil {
v.HasLog = true
}
}
out = append(out, v)
}
return out, nil
}
func (s *Server) pageClusters(w http.ResponseWriter, r *http.Request) {
if r.Method == http.MethodPost {
s.clusterAction(w, r)
return
}
d := s.base("clusters", r)
if s.store == nil {
d.Err = "No database is configured, so nothing can be recorded or shown here. Set -db or $HPCOD_DB_DSN."
s.render(w, "clusters.html", d)
return
}
views, err := s.clusterViews(r)
if err != nil {
d.Err = err.Error()
}
d.Clusters = views
d.CanBuild = s.builder != nil
s.render(w, "clusters.html", d)
}
// clusterAction handles the two things an operator does from this page.
// Both change the world, so both are POSTs; the session cookie is
// SameSite=Strict, which is what keeps another site from driving them.
func (s *Server) clusterAction(w http.ResponseWriter, r *http.Request) {
if s.store == nil {
http.Error(w, "no database configured", http.StatusServiceUnavailable)
return
}
if err := r.ParseForm(); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
id, err := strconv.ParseInt(r.PostFormValue("cluster"), 10, 64)
if err != nil || id <= 0 {
http.Error(w, "which cluster?", http.StatusBadRequest)
return
}
who := orAnon(userFromContext(r.Context()))
// Find the cluster and check it is theirs, once, before anything
// dispatches on the verb.
//
// It used to be checked inside "build" alone. The template hides the
// buttons for a cluster that is not yours, and that was doing all the
// work: "release" and "extend" arrived here with an id and acted on
// it, so any signed-in account could destroy or extend somebody
// else's cluster by posting the form themselves. A hidden button is
// not an authorization check.
c, ok := s.clusterForAction(w, r, id)
if !ok {
return
}
switch r.PostFormValue("do") {
case "build":
s.startBuild(w, r, id)
return
case "release":
// Unpublish first. A released cluster that is still in the
// registry keeps answering at its own name, and the proxy would
// go on trying to reach a machine nobody is maintaining.
s.unpublish(r, id)
// Then take the machines down, and only then give the record
// back. This used to be the database write alone: the record
// said the hardware was free while the virtual machines were
// still running on it, holding the cores, the memory and any
// passed-through device, and the next tenant to ask for that
// GPU would be told it was theirs.
//
// Without a provisioner there is nothing that could destroy
// them, and refusing would leave no way to correct a record at
// all - so the write still happens, and says what it did not do.
if s.builder == nil {
if err := s.store.Release(r.Context(), id); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
log.Printf("cluster %d released from the portal by %s (record only: this portal cannot destroy machines)", id, who)
break
}
// The tenant's work is on the head node's own disk and goes with
// it, so a copy is taken first and a failed copy stops the
// release. "discard" is the way to say the data is genuinely not
// wanted - or that the head is too broken to read, which would
// otherwise leave a cluster nobody can get rid of.
keep := r.PostFormValue("discard") == ""
if err := s.builder.Destroy(*c, keep); err != nil {
http.Error(w, err.Error(), statusFor(err))
return
}
log.Printf("cluster %d: release started by %s (keeping /home: %v)", id, who, keep)
case "extend":
days, err := strconv.Atoi(r.PostFormValue("days"))
if err != nil || days < 1 || days > 365 {
http.Error(w, "extend by 1 to 365 days", http.StatusBadRequest)
return
}
t := time.Now().Add(time.Duration(days) * 24 * time.Hour)
if err := s.store.SetLease(r.Context(), id, &t); err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
log.Printf("cluster %d lease set to %s by %s", id, t.Format(time.RFC3339), who)
default:
http.Error(w, "unknown action", http.StatusBadRequest)
return
}
// Redirect after POST so a reload does not repeat a destructive action.
http.Redirect(w, r, "/clusters", http.StatusSeeOther)
}
// ---------- landing ----------
// overview is the first screen. It answers, in order: is anything wrong, what
// is running, and is there room for more - rather than opening on a dump of
// every PCI device in the estate, which is what it used to do.
type overview struct {
Clusters []clusterView
Expiring []clusterView
SitesUp int
SitesTotal int
Unhealthy []string
FreeCores int
FreeMemGB int64
Schedulable map[model.Capability]int
HeldDevices int
}
// pageFront is the one public route. A visitor gets the front door; a
// signed-in operator gets their overview at the same address, because
// showing someone who is already inside a page inviting them in is silly.
//
// When authentication is disabled entirely - loopback only - there is
// nobody to be signed in, so the overview is what "/" shows. The public
// page would otherwise be the only thing reachable on a portal that has no
// login at all.
func (s *Server) pageFront(w http.ResponseWriter, r *http.Request) {
if s.auth == nil || userFromContext(r.Context()) != "" {
s.pageOverview(w, r)
return
}
s.render(w, "welcome.html", pageData{Page: "welcome"})
}
func (s *Server) pageOverview(w http.ResponseWriter, r *http.Request) {
d := s.base("overview", r)
o := overview{Schedulable: map[model.Capability]int{}}
for _, site := range d.Sites {
o.SitesTotal++
if site.Reachable && site.Healthy {
o.SitesUp++
} else {
why := "unreachable"
if site.Reachable {
why = "not healthy"
}
o.Unhealthy = append(o.Unhealthy, site.ID+": "+why)
}
o.FreeCores += site.TotalFreeCores()
o.FreeMemGB += site.TotalFreeMemBytes() / (1 << 30)
for c, n := range site.Capabilities() {
o.Schedulable[c] += n
}
for _, n := range site.Nodes {
for _, dev := range n.Devices {
if dev.AllocatedTo != "" {
o.HeldDevices++
}
}
}
}
views, err := s.clusterViews(r)
if err != nil {
d.Err = err.Error()
}
o.Clusters = views
for _, v := range views {
if v.Expired || v.Expiring {
o.Expiring = append(o.Expiring, v)
}
}
d.Overview = &o
s.render(w, "overview.html", d)
}
// ---------- building ----------
// mayAct reports whether this request can build or release a cluster.
// Its owner or an administrator: a tenant should not need to ask someone
// else to rebuild their own cluster, and should not be able to touch
// anybody else's.
func mayAct(r *http.Request, c store.Cluster) bool {
if isAdmin(r.Context()) {
return true
}
who := userFromContext(r.Context())
return who != "" && who == c.TenantName
}
// startBuild turns a recorded cluster into machines. It returns as soon as
// the work has started; what happened is read from the cluster's state,
// not from this response.
// clusterForAction resolves the id in a posted form to a live cluster the
// caller may act on, and answers the request itself when it cannot.
//
// A cluster that does not exist and one that belongs to somebody else are
// both 404: which of the two it is would tell a stranger whether an id is
// in use, and by process of elimination how many clusters the estate is
// running.
func (s *Server) clusterForAction(w http.ResponseWriter, r *http.Request, id int64) (*store.Cluster, bool) {
live, err := s.store.Live(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return nil, false
}
for i := range live {
if live[i].SerialID == id {
if !mayAct(r, live[i]) {
http.NotFound(w, r)
return nil, false
}
return &live[i], true
}
}
http.Error(w, "no such live cluster", http.StatusNotFound)
return nil, false
}
func (s *Server) startBuild(w http.ResponseWriter, r *http.Request, id int64) {
// Order matters. Whether this portal can build at all is checked last,
// so a stranger asking about somebody else's cluster learns nothing
// but 404, and the owner of a running cluster is told that it is
// running rather than that the portal is misconfigured.
live, err := s.store.Live(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
var c *store.Cluster
for i := range live {
if live[i].SerialID == id {
c = &live[i]
break
}
}
if c == nil {
http.Error(w, "no such live cluster", http.StatusNotFound)
return
}
if !mayAct(r, *c) {
http.NotFound(w, r)
return
}
// Building a cluster that is already there would re-run Terraform
// against live machines. Refusing is the conservative half; rebuilding
// a failed one is the useful half.
switch c.State {
case "requested", "failed":
default:
http.Error(w, "a cluster is built from \"requested\" or \"failed\", not from \""+c.State+"\"",
http.StatusConflict)
return
}
if s.builder == nil {
http.Error(w, "this portal cannot build clusters: no environment template is configured",
http.StatusServiceUnavailable)
return
}
if err := s.builder.Start(*c); err != nil {
http.Error(w, err.Error(), http.StatusConflict)
return
}
log.Printf("cluster %s: build started by %s", c.AccountingName, orAnon(userFromContext(r.Context())))
http.Redirect(w, r, "/clusters", http.StatusSeeOther)
}
// mayUseCluster decides whether a request may be proxied into a cluster's
// own portal.
//
// The cluster no longer asks for anything: it accepts the control plane's
// certificate and trusts whoever arrives through it. That makes this the
// only thing standing between the internet and somebody's dashboard, so
// it is deliberately strict - a valid session, and the cluster has to be
// theirs.
func (s *Server) mayUseCluster(r *http.Request, label string) bool {
// With no login configured at all the portal is loopback-only, and
// there is nobody to be signed in as.
if s.auth == nil {
return true
}
c, err := r.Cookie(sessionCookie)
if err != nil {
return false
}
sess := s.auth.lookup(c.Value)
if sess == nil {
return false
}
admin, ok := s.auth.stillAllowed(r.Context(), sess.user)
if !ok {
return false
}
if admin {
return true
}
if s.store == nil {
// Nothing to check ownership against. Refusing is the safe half:
// a cluster reachable by every signed-in account is not what the
// owner was promised.
return false
}
// label is "c001"; the cluster it names is the live one with that id.
var id int
if _, err := fmt.Sscanf(label, "c%03d", &id); err != nil {
return false
}
live, err := s.store.Live(r.Context())
if err != nil {
return false
}
for _, cl := range live {
if int(cl.ClusterID) == id {
return cl.TenantName == sess.user
}
}
return false
}
// portalURL is the public address of a cluster's own portal, or empty if
// there is not one to link to yet.
//
// Published, not merely ready: the proxy answers a name it does not know
// with "no cluster is published here", and sending someone there from
// their own cluster's row would read as the cluster being broken.
func (s *Server) portalURL(c store.Cluster) string {
if s.opt.ClusterDomain == "" || s.registry == nil {
return ""
}
label := fmt.Sprintf("c%03d", c.ClusterID)
if _, ok := s.registry.lookup(label + "." + s.opt.ClusterDomain); !ok {
return ""
}
return "https://" + label + "." + s.opt.ClusterDomain + "/"
}
// unpublish takes a cluster out of the proxy's routing. Best effort and
// logged rather than fatal: failing to release a cluster because its name
// could not be withdrawn would leave the address and the hardware held by
// something that is being thrown away anyway.
func (s *Server) unpublish(r *http.Request, serialID int64) {
if s.opt.Provision.Registry == nil || s.store == nil {
return
}
live, err := s.store.Live(r.Context())
if err != nil {
log.Printf("unpublish %d: %v", serialID, err)
return
}
for _, c := range live {
if c.SerialID == serialID {
if err := s.opt.Provision.Registry.Unpublish(c.ClusterID); err != nil {
log.Printf("unpublish c%03d: %v", c.ClusterID, err)
}
return
}
}
}