Slurm's shared files and users: one path everywhere, accounts as Linux users

slurm.shared mounts the site's NFS export, has an llmgw node export the path
over NFS to the Slurm machines alone (2049 in the firewall), or looks for a
file system the site mounts. slurm.users "llmgw" makes each active account a
Linux user on every Slurm machine: login from its e-mail, a UID from 200000
kept in slurm_users and never reused, its home on the shared path, its SSH
keys from the portal's Profile (or ctl slurm users keys) read by sshd from
/etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first
machine and llmgw's users; --slurm-shared and --slurm-users change that.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/control/adminapi/api.go

@@ -98,6 +98,8 @@ type API struct {
 	usage     Usage
 	quotas    Quotas
 	ctl       *control.Controller
+	// slurmUsers are the Linux users made from the accounts; nil without a store.
+	slurmUsers SlurmUsers
 }
 
 // New builds the API over a controller.

internal/control/adminapi/routes.go

@@ -85,6 +85,7 @@ var PathParams = map[string]string{
 	"projects/{id}":     "The project's ID.",
 	"keys/{id}":         "The key's ID, as listed (not the key itself).",
 	"users/{id}":        "The person's ID or email.",
+	"users/{user}":      "The person's ID or email.",
 	"artifacts/{kind}":  "What the job made: peft, a gzipped tar of the PEFT adapter, or gguf, the adapter for llama.cpp.",
 }
 
@@ -398,6 +399,14 @@ var Routes = []Route{
 			"(drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs " +
 			"still running) or unknown (its agent has not said).",
 		Response: slurmsvc.Status{}},
+	{Method: "GET", Path: "/api/v1/slurm/users", Summary: "The Linux users made from llmgw's accounts for the Slurm cluster",
+		Description: "Every active account, with its login, UID and SSH keys; an account is given its login and UID the first " +
+			"time it is listed. The machines have them when slurm.users is \"llmgw\".",
+		Response: []slurmsvc.User{}},
+	{Method: "PUT", Path: "/api/v1/slurm/users/{user}/ssh-keys", Summary: "Replace an account's SSH public keys",
+		Description: "{user} is an ID or e-mail. keys is OpenSSH public keys, one a line, without options; empty takes them " +
+			"all away. The machines have them within a minute.",
+		Request: SlurmKeysRequest{}, Response: slurmsvc.User{}},
 
 	{Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node",
 		Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " +

internal/control/adminapi/slurm.go

@@ -1,15 +1,43 @@
 package adminapi
 
 import (
+	"context"
+	"encoding/json"
+	"errors"
 	"net/http"
+	"strings"
 
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
 )
 
 // SlurmService is the Slurm cluster beside llmgw (BACKLOG P), as the leader
 // keeps it.
 type SlurmService interface {
 	Status() slurmsvc.Status
+	UsersChanged()
+}
+
+// SlurmUsers are the Linux users llmgw makes from its accounts.
+type SlurmUsers interface {
+	SlurmUsers(ctx context.Context) ([]store.SlurmUser, error)
+	SlurmUserByID(ctx context.Context, userID string) (store.SlurmUser, error)
+	SetSlurmSSHKeys(ctx context.Context, userID string, keys []string) error
+}
+
+// SetSlurmUsers enables the Slurm users' endpoints.
+func (a *API) SetSlurmUsers(u SlurmUsers) { a.slurmUsers = u }
+
+// SlurmKeysRequest is a person's SSH public keys, one a line, as
+// authorized_keys has them; empty takes them all away.
+type SlurmKeysRequest struct {
+	Keys string `json:"keys"`
+}
+
+// SlurmUserView is the user from a store row.
+func SlurmUserView(u store.SlurmUser) slurmsvc.User {
+	return slurmsvc.User{Login: u.Login, UID: u.UID, Name: u.Name, Email: u.Email, SSHKeys: u.SSHKeys}
 }
 
 // SetSlurm gives the API the Slurm service.
@@ -17,6 +45,69 @@ func (a *API) SetSlurm(s SlurmService) { a.slurm = s }
 
 func (a *API) registerSlurm(mux Mux) {
 	mux.HandleFunc("GET /api/v1/slurm", a.slurmStatus)
+	mux.HandleFunc("GET /api/v1/slurm/users", a.slurmUserList)
+	mux.HandleFunc("PUT /api/v1/slurm/users/{user}/ssh-keys", a.slurmUserKeys)
+}
+
+func (a *API) slurmUserList(w http.ResponseWriter, r *http.Request) {
+	if a.slurmUsers == nil {
+		writeError(w, http.StatusNotImplemented, "the Slurm users need the store, which this control plane runs without")
+		return
+	}
+	us, err := a.slurmUsers.SlurmUsers(r.Context())
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	out := make([]slurmsvc.User, 0, len(us))
+	for _, u := range us {
+		out = append(out, SlurmUserView(u))
+	}
+	writeJSON(w, http.StatusOK, out)
+}
+
+// slurmUserKeys replaces a person's SSH keys; {user} is an ID or e-mail.
+func (a *API) slurmUserKeys(w http.ResponseWriter, r *http.Request) {
+	if a.slurmUsers == nil {
+		writeError(w, http.StatusNotImplemented, "the Slurm users need the store, which this control plane runs without")
+		return
+	}
+	if !a.tenancyReady(w) {
+		return
+	}
+	var in SlurmKeysRequest
+	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&in); err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	keys, err := slurm.ParseSSHKeys(in.Keys)
+	if err != nil {
+		writeError(w, http.StatusBadRequest, err.Error())
+		return
+	}
+	ctx := r.Context()
+	u, err := a.findUser(ctx, r.PathValue("user"))
+	if err != nil {
+		writeError(w, http.StatusNotFound, strings.TrimPrefix(err.Error(), store.ErrNotFound.Error()+": "))
+		return
+	}
+	if err := a.slurmUsers.SetSlurmSSHKeys(ctx, u.ID, keys); err != nil {
+		code := http.StatusInternalServerError
+		if errors.Is(err, store.ErrNotFound) {
+			code = http.StatusNotFound
+		}
+		writeError(w, code, err.Error())
+		return
+	}
+	if a.slurm != nil {
+		a.slurm.UsersChanged()
+	}
+	su, err := a.slurmUsers.SlurmUserByID(ctx, u.ID)
+	if err != nil {
+		writeError(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+	writeJSON(w, http.StatusOK, SlurmUserView(su))
 }
 
 func (a *API) slurmStatus(w http.ResponseWriter, _ *http.Request) {