Slurm's shared files and users: one path everywhere, accounts as Linux users
CI / test (push) Failing after 7m1s
CI / test (push) Failing after 7m1s
slurm.shared mounts the site's NFS export, has an llmgw node export the path over NFS to the Slurm machines alone (2049 in the firewall), or looks for a file system the site mounts. slurm.users "llmgw" makes each active account a Linux user on every Slurm machine: login from its e-mail, a UID from 200000 kept in slurm_users and never reused, its home on the shared path, its SSH keys from the portal's Profile (or ctl slurm users keys) read by sshd from /etc/llmgw/ssh-keys. server init --slurm chooses /shared from the first machine and llmgw's users; --slurm-shared and --slurm-users change that. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
b9d20e7bba
commit
d8ce573bdb
45 files changed
+2505
-79
No files matched your search
@@ -98,6 +98,8 @@ type API struct {
|
||||
usage Usage
|
||||
quotas Quotas
|
||||
ctl *control.Controller
|
||||
// slurmUsers are the Linux users made from the accounts; nil without a store.
|
||||
slurmUsers SlurmUsers
|
||||
}
|
||||
|
||||
// New builds the API over a controller.
|
||||
|
||||
@@ -85,6 +85,7 @@ var PathParams = map[string]string{
|
||||
"projects/{id}": "The project's ID.",
|
||||
"keys/{id}": "The key's ID, as listed (not the key itself).",
|
||||
"users/{id}": "The person's ID or email.",
|
||||
"users/{user}": "The person's ID or email.",
|
||||
"artifacts/{kind}": "What the job made: peft, a gzipped tar of the PEFT adapter, or gguf, the adapter for llama.cpp.",
|
||||
}
|
||||
|
||||
@@ -398,6 +399,14 @@ var Routes = []Route{
|
||||
"(drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs " +
|
||||
"still running) or unknown (its agent has not said).",
|
||||
Response: slurmsvc.Status{}},
|
||||
{Method: "GET", Path: "/api/v1/slurm/users", Summary: "The Linux users made from llmgw's accounts for the Slurm cluster",
|
||||
Description: "Every active account, with its login, UID and SSH keys; an account is given its login and UID the first " +
|
||||
"time it is listed. The machines have them when slurm.users is \"llmgw\".",
|
||||
Response: []slurmsvc.User{}},
|
||||
{Method: "PUT", Path: "/api/v1/slurm/users/{user}/ssh-keys", Summary: "Replace an account's SSH public keys",
|
||||
Description: "{user} is an ID or e-mail. keys is OpenSSH public keys, one a line, without options; empty takes them " +
|
||||
"all away. The machines have them within a minute.",
|
||||
Request: SlurmKeysRequest{}, Response: slurmsvc.User{}},
|
||||
|
||||
{Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node",
|
||||
Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " +
|
||||
|
||||
@@ -1,15 +1,43 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// SlurmService is the Slurm cluster beside llmgw (BACKLOG P), as the leader
|
||||
// keeps it.
|
||||
type SlurmService interface {
|
||||
Status() slurmsvc.Status
|
||||
UsersChanged()
|
||||
}
|
||||
|
||||
// SlurmUsers are the Linux users llmgw makes from its accounts.
|
||||
type SlurmUsers interface {
|
||||
SlurmUsers(ctx context.Context) ([]store.SlurmUser, error)
|
||||
SlurmUserByID(ctx context.Context, userID string) (store.SlurmUser, error)
|
||||
SetSlurmSSHKeys(ctx context.Context, userID string, keys []string) error
|
||||
}
|
||||
|
||||
// SetSlurmUsers enables the Slurm users' endpoints.
|
||||
func (a *API) SetSlurmUsers(u SlurmUsers) { a.slurmUsers = u }
|
||||
|
||||
// SlurmKeysRequest is a person's SSH public keys, one a line, as
|
||||
// authorized_keys has them; empty takes them all away.
|
||||
type SlurmKeysRequest struct {
|
||||
Keys string `json:"keys"`
|
||||
}
|
||||
|
||||
// SlurmUserView is the user from a store row.
|
||||
func SlurmUserView(u store.SlurmUser) slurmsvc.User {
|
||||
return slurmsvc.User{Login: u.Login, UID: u.UID, Name: u.Name, Email: u.Email, SSHKeys: u.SSHKeys}
|
||||
}
|
||||
|
||||
// SetSlurm gives the API the Slurm service.
|
||||
@@ -17,6 +45,69 @@ func (a *API) SetSlurm(s SlurmService) { a.slurm = s }
|
||||
|
||||
func (a *API) registerSlurm(mux Mux) {
|
||||
mux.HandleFunc("GET /api/v1/slurm", a.slurmStatus)
|
||||
mux.HandleFunc("GET /api/v1/slurm/users", a.slurmUserList)
|
||||
mux.HandleFunc("PUT /api/v1/slurm/users/{user}/ssh-keys", a.slurmUserKeys)
|
||||
}
|
||||
|
||||
func (a *API) slurmUserList(w http.ResponseWriter, r *http.Request) {
|
||||
if a.slurmUsers == nil {
|
||||
writeError(w, http.StatusNotImplemented, "the Slurm users need the store, which this control plane runs without")
|
||||
return
|
||||
}
|
||||
us, err := a.slurmUsers.SlurmUsers(r.Context())
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
out := make([]slurmsvc.User, 0, len(us))
|
||||
for _, u := range us {
|
||||
out = append(out, SlurmUserView(u))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
// slurmUserKeys replaces a person's SSH keys; {user} is an ID or e-mail.
|
||||
func (a *API) slurmUserKeys(w http.ResponseWriter, r *http.Request) {
|
||||
if a.slurmUsers == nil {
|
||||
writeError(w, http.StatusNotImplemented, "the Slurm users need the store, which this control plane runs without")
|
||||
return
|
||||
}
|
||||
if !a.tenancyReady(w) {
|
||||
return
|
||||
}
|
||||
var in SlurmKeysRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
keys, err := slurm.ParseSSHKeys(in.Keys)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
ctx := r.Context()
|
||||
u, err := a.findUser(ctx, r.PathValue("user"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusNotFound, strings.TrimPrefix(err.Error(), store.ErrNotFound.Error()+": "))
|
||||
return
|
||||
}
|
||||
if err := a.slurmUsers.SetSlurmSSHKeys(ctx, u.ID, keys); err != nil {
|
||||
code := http.StatusInternalServerError
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
code = http.StatusNotFound
|
||||
}
|
||||
writeError(w, code, err.Error())
|
||||
return
|
||||
}
|
||||
if a.slurm != nil {
|
||||
a.slurm.UsersChanged()
|
||||
}
|
||||
su, err := a.slurmUsers.SlurmUserByID(ctx, u.ID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, SlurmUserView(su))
|
||||
}
|
||||
|
||||
func (a *API) slurmStatus(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
@@ -71,17 +71,21 @@ type NodeStatus struct {
|
||||
|
||||
// Reported is the agent's SlurmStatus, for the admin API.
|
||||
type Reported struct {
|
||||
Version string `json:"version,omitempty"`
|
||||
Hardware string `json:"hardware,omitempty"`
|
||||
SlurmdRunning bool `json:"slurmd_running"`
|
||||
SlurmctldRunning bool `json:"slurmctld_running"`
|
||||
SlurmdbdRunning bool `json:"slurmdbd_running"`
|
||||
State string `json:"state,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Jobs int `json:"jobs"`
|
||||
Error string `json:"error,omitempty"`
|
||||
FilesSHA256 string `json:"files_sha256,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
Version string `json:"version,omitempty"`
|
||||
Hardware string `json:"hardware,omitempty"`
|
||||
SlurmdRunning bool `json:"slurmd_running"`
|
||||
SlurmctldRunning bool `json:"slurmctld_running"`
|
||||
SlurmdbdRunning bool `json:"slurmdbd_running"`
|
||||
// Shared is the shared path here: "mounted", "exported", "missing".
|
||||
Shared string `json:"shared,omitempty"`
|
||||
// Users is how many of llmgw's users the machine has.
|
||||
Users int `json:"users"`
|
||||
State string `json:"state,omitempty"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
Jobs int `json:"jobs"`
|
||||
Error string `json:"error,omitempty"`
|
||||
FilesSHA256 string `json:"files_sha256,omitempty"`
|
||||
At time.Time `json:"at"`
|
||||
}
|
||||
|
||||
// Status is the whole Slurm cluster, for `ctl slurm status`.
|
||||
@@ -95,6 +99,20 @@ type Status struct {
|
||||
// Waiting are nodes left out of slurm.conf until they report their
|
||||
// hardware.
|
||||
Waiting []string `json:"waiting,omitempty"`
|
||||
// Shared is where the shared files are and come from; Users who keeps
|
||||
// the users, and how many llmgw has.
|
||||
Shared config.SharedStorage `json:"shared"`
|
||||
Users string `json:"users,omitempty"`
|
||||
UserCount int `json:"user_count,omitempty"`
|
||||
}
|
||||
|
||||
// User is a Linux user made from an llmgw account.
|
||||
type User struct {
|
||||
Login string `json:"login"`
|
||||
UID int `json:"uid"`
|
||||
Name string `json:"name"`
|
||||
Email string `json:"email"`
|
||||
SSHKeys []string `json:"ssh_keys,omitempty"`
|
||||
}
|
||||
|
||||
// Service keeps the Slurm cluster.
|
||||
@@ -127,6 +145,46 @@ type Service struct {
|
||||
// started is the first pass, from which a node neither heard from nor
|
||||
// kept is waited for, for HoldFor.
|
||||
started time.Time
|
||||
// userList reads the users from the accounts; users is what it last
|
||||
// said, read at usersAt. Until it has been read once no node is told
|
||||
// anything of users: an empty list would take every user away.
|
||||
userList func(ctx context.Context) ([]User, error)
|
||||
users []User
|
||||
usersAt time.Time
|
||||
usersOK bool
|
||||
}
|
||||
|
||||
// UsersEvery is how often the users are read again without being asked.
|
||||
var UsersEvery = 30 * time.Second
|
||||
|
||||
// SetUsers has the users read with list, for slurm.users "llmgw".
|
||||
func (s *Service) SetUsers(list func(ctx context.Context) ([]User, error)) {
|
||||
s.mu.Lock()
|
||||
s.userList = list
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// UsersChanged has the users read again now: an account or its keys
|
||||
// changed.
|
||||
func (s *Service) UsersChanged() {
|
||||
s.mu.Lock()
|
||||
s.usersAt = time.Time{}
|
||||
s.mu.Unlock()
|
||||
s.Kick()
|
||||
}
|
||||
|
||||
// readUsers reads the users when they are due. Under s.mu.
|
||||
func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
|
||||
if set.Users != config.UsersLlmgw || s.userList == nil || time.Since(s.usersAt) < UsersEvery {
|
||||
return
|
||||
}
|
||||
us, err := s.userList(ctx)
|
||||
s.usersAt = time.Now()
|
||||
if err != nil {
|
||||
s.log.Warn("could not read the Slurm users from the accounts; the nodes keep the last ones", "err", err)
|
||||
return
|
||||
}
|
||||
s.users, s.usersOK = us, true
|
||||
}
|
||||
|
||||
// HoldFor is how long a new leader waits for a node it neither has heard
|
||||
@@ -198,6 +256,7 @@ func (s *Service) Kick() {
|
||||
// OnSlurmStatus records what a node's agent says.
|
||||
func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
|
||||
r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(),
|
||||
Shared: st.GetShared(), Users: int(st.GetUsers()),
|
||||
SlurmctldRunning: st.GetSlurmctldRunning(), SlurmdbdRunning: st.GetSlurmdbdRunning(), State: st.GetState(), Reason: st.GetReason(),
|
||||
Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()}
|
||||
s.mu.Lock()
|
||||
@@ -227,6 +286,7 @@ func (s *Service) Tick(ctx context.Context) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.recall(ctx)
|
||||
s.readUsers(ctx, set)
|
||||
if s.started.IsZero() {
|
||||
s.started = time.Now()
|
||||
}
|
||||
@@ -477,16 +537,103 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
|
||||
m.Controller = true
|
||||
m.Files = maps.Clone(files)
|
||||
}
|
||||
if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
|
||||
return nil
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
// sharedPart puts the shared files in m, false when where they come from
|
||||
// is not known yet: told nothing rather than no shared files, which would
|
||||
// unmount them under running jobs. Under s.mu.
|
||||
func (s *Service) sharedPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool {
|
||||
sh := set.Shared
|
||||
if sh.Path == "" {
|
||||
return true
|
||||
}
|
||||
m.Shared = &llmgwv1.SlurmShared{Path: sh.Path, Nfs: sh.NFS}
|
||||
switch {
|
||||
case sh.Server == "":
|
||||
case sh.Server == node:
|
||||
m.Shared.Serve = true
|
||||
for _, n := range s.machines(set) {
|
||||
if n == node {
|
||||
continue
|
||||
}
|
||||
if a := s.addr(set, n); a != "" {
|
||||
m.Shared.Clients = append(m.Shared.Clients, a)
|
||||
}
|
||||
}
|
||||
slices.Sort(m.Shared.Clients)
|
||||
default:
|
||||
a := s.addr(set, sh.Server)
|
||||
if a == "" {
|
||||
return false
|
||||
}
|
||||
if strings.Contains(a, ":") {
|
||||
a = "[" + a + "]"
|
||||
}
|
||||
m.Shared.Nfs = a + ":" + sh.Path
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// usersPart puts the users in m, false while they have not been read.
|
||||
// Under s.mu.
|
||||
func (s *Service) usersPart(set config.SlurmSettings, node string, m *llmgwv1.Slurm) bool {
|
||||
if set.Users != config.UsersLlmgw {
|
||||
return true
|
||||
}
|
||||
if !s.usersOK {
|
||||
return false
|
||||
}
|
||||
m.ManageUsers = true
|
||||
switch {
|
||||
case set.Shared.Path == "":
|
||||
m.MakeHomes = true // each machine's own /home
|
||||
case set.Shared.Server != "":
|
||||
m.MakeHomes = set.Shared.Server == node
|
||||
default:
|
||||
m.MakeHomes = set.Controller == node
|
||||
}
|
||||
for _, u := range s.users {
|
||||
m.Users = append(m.Users, &llmgwv1.SlurmUser{Login: u.Login, Uid: uint32(u.UID), Name: u.Name, //nolint:gosec // from FirstSlurmUID up
|
||||
Home: set.HomeDir(u.Login), SshKeys: slices.Clone(u.SSHKeys)})
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// machines are the controller and the nodes.
|
||||
func (s *Service) machines(set config.SlurmSettings) []string {
|
||||
out := slices.Clone(set.Nodes)
|
||||
if !slices.Contains(out, set.Controller) {
|
||||
out = append(out, set.Controller)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// addr is a machine's management address, from the inventory, else kept
|
||||
// from before. Under s.mu.
|
||||
func (s *Service) addr(set config.SlurmSettings, node string) string {
|
||||
if n, ok := s.inv.Get(node); ok && n.MgmtAddr != "" {
|
||||
return n.MgmtAddr
|
||||
}
|
||||
if node == set.Controller {
|
||||
return s.controllerAddr(set)
|
||||
}
|
||||
return s.known.Nodes[node].Addr
|
||||
}
|
||||
|
||||
// Status is the cluster as the service sees it now.
|
||||
func (s *Service) Status() Status {
|
||||
set := s.current()
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller,
|
||||
Partition: set.PartitionName(), Ctld: s.status[set.Controller]}
|
||||
Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
|
||||
if set.Users == config.UsersLlmgw {
|
||||
out.UserCount = len(s.users)
|
||||
}
|
||||
if !set.Enabled {
|
||||
return out
|
||||
}
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
package slurmsvc
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"slices"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
)
|
||||
|
||||
func TestSharedFilesFromTheController(t *testing.T) {
|
||||
r := newRig()
|
||||
r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"}
|
||||
r.tick()
|
||||
ctl, node := r.agents.last("node06").GetShared(), r.agents.last("cuda01").GetShared()
|
||||
if !ctl.GetServe() || !slices.Equal(ctl.GetClients(), []string{"10.0.0.147"}) || ctl.GetPath() != "/shared" {
|
||||
t.Fatalf("server told %v", ctl)
|
||||
}
|
||||
if node.GetServe() || node.GetNfs() != "10.0.0.146:/shared" {
|
||||
t.Fatalf("client told %v", node)
|
||||
}
|
||||
// The site's own export: every machine mounts it.
|
||||
r.set.Shared = config.SharedStorage{Path: "/shared", NFS: "filer:/hpc"}
|
||||
r.tick()
|
||||
if r.agents.last("node06").GetShared().GetNfs() != "filer:/hpc" || r.agents.last("cuda01").GetShared().GetServe() {
|
||||
t.Fatal("the site's export not sent")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSharedServerNotKnownTellsNothing(t *testing.T) {
|
||||
r := newRig()
|
||||
r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"}
|
||||
delete(r.inv.nodes, "node06")
|
||||
r.svc.known.ControllerAddr, r.svc.known.Controller = "10.0.0.146", "node06"
|
||||
r.tick()
|
||||
if r.agents.last("cuda01").GetShared().GetNfs() != "10.0.0.146:/shared" {
|
||||
t.Fatalf("the kept address not used: %v", r.agents.last("cuda01"))
|
||||
}
|
||||
r2 := newRig()
|
||||
r2.set.Nodes = []string{"cuda01", "site2"}
|
||||
r2.agents.conn = append(r2.agents.conn, "site2")
|
||||
r2.set.Shared = config.SharedStorage{Path: "/shared", Server: "site2"}
|
||||
r2.tick()
|
||||
if r2.agents.last("cuda01") != nil {
|
||||
t.Fatal("told with no address for the server: its mount would go")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsersSentOnceRead(t *testing.T) {
|
||||
r := newRig()
|
||||
r.set.Users = config.UsersLlmgw
|
||||
r.set.Shared = config.SharedStorage{Path: "/shared", Server: "node06"}
|
||||
fail := true
|
||||
r.svc.SetUsers(func(context.Context) ([]User, error) {
|
||||
if fail {
|
||||
return nil, errors.New("store down")
|
||||
}
|
||||
return []User{{Login: "ayse", UID: 200000, Name: "Ayşe", SSHKeys: []string{"ssh-ed25519 AAAA"}}}, nil
|
||||
})
|
||||
r.tick()
|
||||
if len(r.agents.got["cuda01"]) != 0 {
|
||||
t.Fatal("told before the users were read: every user would go")
|
||||
}
|
||||
fail = false
|
||||
r.svc.UsersChanged()
|
||||
r.tick()
|
||||
ctl, node := r.agents.last("node06"), r.agents.last("cuda01")
|
||||
if !ctl.GetManageUsers() || !ctl.GetMakeHomes() || node.GetMakeHomes() || len(node.GetUsers()) != 1 {
|
||||
t.Fatalf("controller %v\nnode %v", ctl, node)
|
||||
}
|
||||
if u := node.GetUsers()[0]; u.GetHome() != "/shared/home/ayse" || u.GetUid() != 200000 || len(u.GetSshKeys()) != 1 {
|
||||
t.Fatalf("user %v", u)
|
||||
}
|
||||
// The store down later: the last users are kept.
|
||||
fail = true
|
||||
r.svc.UsersChanged()
|
||||
r.tick()
|
||||
if len(r.agents.last("cuda01").GetUsers()) != 1 {
|
||||
t.Fatal("users lost when the store could not be read")
|
||||
}
|
||||
if st := r.svc.Status(); st.UserCount != 1 || st.Shared.Server != "node06" {
|
||||
t.Fatalf("status %+v", st)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user