Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s
CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8547528fb7
commit
fcda4d3f85
22 files changed
+1390
-76
No files matched your search
@@ -116,6 +116,8 @@ type Status struct {
|
||||
// machine has pyxis and enroot, "installing" until then, empty when
|
||||
// slurm.containers is off.
|
||||
Containers string `json:"containers,omitempty"`
|
||||
// Accounts is how many of Slurm's accounts are llmgw's projects'.
|
||||
Accounts int `json:"accounts,omitempty"`
|
||||
}
|
||||
|
||||
// User is a Linux user made from an llmgw account.
|
||||
@@ -166,6 +168,53 @@ type Service struct {
|
||||
users []User
|
||||
usersAt time.Time
|
||||
usersOK bool
|
||||
// accountList reads Slurm's accounts from the projects, with the users;
|
||||
// accountsOK is false until it has been read, and the controller is
|
||||
// told nothing of accounts until then: none would take every one away.
|
||||
accountList func(ctx context.Context) ([]slurm.Account, error)
|
||||
accounts []slurm.Account
|
||||
accountsOK bool
|
||||
// jobSink records the jobs that ended; recorded are those it took,
|
||||
// recording whether a recording is under way.
|
||||
jobSink func(ctx context.Context, jobs []JobRecord) error
|
||||
recorded map[string]time.Time
|
||||
recording bool
|
||||
}
|
||||
|
||||
// JobRecord is a job that ended, for llmgw's accounting.
|
||||
type JobRecord struct {
|
||||
ID, Login, Account, Partition, State string
|
||||
Start, End time.Time
|
||||
Elapsed time.Duration
|
||||
CPUs, GPUs int
|
||||
}
|
||||
|
||||
// SetAccounts has Slurm's accounts read with list, beside the users.
|
||||
func (s *Service) SetAccounts(list func(ctx context.Context) ([]slurm.Account, error)) {
|
||||
s.mu.Lock()
|
||||
s.accountList = list
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetJobs has the jobs that ended recorded with record; each is given to it
|
||||
// once it took it without an error.
|
||||
func (s *Service) SetJobs(record func(ctx context.Context, jobs []JobRecord) error) {
|
||||
s.mu.Lock()
|
||||
s.jobSink = record
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// AccountsOf are the accounts a login may run jobs on.
|
||||
func (s *Service) AccountsOf(login string) []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []string
|
||||
for _, a := range s.accounts {
|
||||
if slices.Contains(a.Users, login) {
|
||||
out = append(out, a.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// UsersEvery is how often the users are read again without being asked.
|
||||
@@ -199,6 +248,64 @@ func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
|
||||
return
|
||||
}
|
||||
s.users, s.usersOK = us, true
|
||||
if s.accountList == nil {
|
||||
return
|
||||
}
|
||||
as, err := s.accountList(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("could not read Slurm's accounts from the projects; the controller keeps the last ones", "err", err)
|
||||
return
|
||||
}
|
||||
s.accounts, s.accountsOK = as, true
|
||||
}
|
||||
|
||||
// takeJobs records the jobs a controller says ended that are not recorded
|
||||
// yet, away from the agent's stream.
|
||||
func (s *Service) takeJobs(ended []*llmgwv1.SlurmJob) {
|
||||
s.mu.Lock()
|
||||
if s.jobSink == nil || s.recording {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if s.recorded == nil {
|
||||
s.recorded = map[string]time.Time{}
|
||||
}
|
||||
for id, at := range s.recorded {
|
||||
if time.Since(at) > 3*24*time.Hour {
|
||||
delete(s.recorded, id)
|
||||
}
|
||||
}
|
||||
var jobs []JobRecord
|
||||
for _, j := range ended {
|
||||
if _, ok := s.recorded[j.GetId()]; ok || j.GetId() == "" {
|
||||
continue
|
||||
}
|
||||
jobs = append(jobs, JobRecord{ID: j.GetId(), Login: j.GetUser(), Account: j.GetAccount(), Partition: j.GetPartition(),
|
||||
State: j.GetState(), Start: time.Unix(j.GetStartUnix(), 0), End: time.Unix(j.GetEndUnix(), 0),
|
||||
Elapsed: time.Duration(j.GetElapsedSeconds()) * time.Second, CPUs: int(j.GetCpus()), GPUs: int(j.GetGpus())})
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
sink := s.jobSink
|
||||
s.recording = true
|
||||
s.mu.Unlock()
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
err := sink(ctx, jobs)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.recording = false
|
||||
if err != nil {
|
||||
s.log.Warn("could not record the Slurm jobs that ended; tried again with the next status", "jobs", len(jobs), "err", err)
|
||||
return
|
||||
}
|
||||
for _, j := range jobs {
|
||||
s.recorded[j.ID] = time.Now()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// HoldFor is how long a new leader waits for a node it neither has heard
|
||||
@@ -276,7 +383,11 @@ func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
|
||||
s.mu.Lock()
|
||||
prev := s.status[node]
|
||||
s.status[node] = r
|
||||
isCtl := s.settings().Controller == node
|
||||
s.mu.Unlock()
|
||||
if isCtl && len(st.GetEnded()) > 0 {
|
||||
s.takeJobs(st.GetEnded())
|
||||
}
|
||||
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
|
||||
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
|
||||
}
|
||||
@@ -517,6 +628,7 @@ func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
|
||||
return c, nil
|
||||
}
|
||||
c.ControllerAddr = s.controllerAddr(set)
|
||||
c.EnforceAccounts = set.Users == config.UsersLlmgw && s.accountList != nil
|
||||
var waiting []string
|
||||
for _, node := range set.Nodes {
|
||||
sn, ok := s.reported(node)
|
||||
@@ -582,6 +694,15 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
|
||||
if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
|
||||
return nil
|
||||
}
|
||||
if isCtl && set.Users == config.UsersLlmgw && s.accountList != nil {
|
||||
if !s.accountsOK {
|
||||
return nil
|
||||
}
|
||||
m.ManageAccounts = true
|
||||
for _, a := range s.accounts {
|
||||
m.Accounts = append(m.Accounts, &llmgwv1.SlurmAccount{Name: a.Name, Parent: a.Parent, Users: slices.Clone(a.Users)})
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -675,6 +796,11 @@ func (s *Service) Status() Status {
|
||||
Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
|
||||
if set.Users == config.UsersLlmgw {
|
||||
out.UserCount = len(s.users)
|
||||
for _, a := range s.accounts {
|
||||
if a.Project != "" {
|
||||
out.Accounts++
|
||||
}
|
||||
}
|
||||
}
|
||||
if !set.Enabled {
|
||||
return out
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
)
|
||||
|
||||
type fakeInv struct {
|
||||
@@ -323,3 +324,59 @@ func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
|
||||
t.Fatal("pyxis still loaded with containers off")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllerKeepsTheProjectsAccountsAndJobsAreRecordedOnce(t *testing.T) {
|
||||
r := newRig()
|
||||
r.set.Users = config.UsersLlmgw
|
||||
r.svc.SetUsers(func(context.Context) ([]User, error) { return []User{{Login: "ayse", UID: 200001}}, nil })
|
||||
read := 0
|
||||
r.svc.SetAccounts(func(context.Context) ([]slurm.Account, error) {
|
||||
read++
|
||||
return []slurm.Account{{Name: "test", Parent: "root"},
|
||||
{Name: "test-arge-deneme", Parent: "test", Project: "p1", Users: []string{"ayse"}}}, nil
|
||||
})
|
||||
got := make(chan []JobRecord, 4)
|
||||
r.svc.SetJobs(func(_ context.Context, jobs []JobRecord) error { got <- jobs; return nil })
|
||||
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
|
||||
r.tick()
|
||||
ctl := r.agents.last("node06")
|
||||
if !ctl.GetManageAccounts() || len(ctl.GetAccounts()) != 2 || ctl.GetAccounts()[1].GetUsers()[0] != "ayse" {
|
||||
t.Fatalf("controller told %v", ctl.GetAccounts())
|
||||
}
|
||||
if r.agents.last("cuda01").GetManageAccounts() {
|
||||
t.Error("a node was told the accounts")
|
||||
}
|
||||
if !strings.Contains(ctl.GetFiles()["slurm.conf"], "AccountingStorageEnforce=associations") {
|
||||
t.Error("jobs not held to accounts")
|
||||
}
|
||||
if r.svc.Status().Accounts != 1 || len(r.svc.AccountsOf("ayse")) != 1 {
|
||||
t.Errorf("status %d, ayse's %v", r.svc.Status().Accounts, r.svc.AccountsOf("ayse"))
|
||||
}
|
||||
ended := []*llmgwv1.SlurmJob{{Id: "41", User: "ayse", Account: "test-arge-deneme", ElapsedSeconds: 60, Gpus: 1}}
|
||||
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
|
||||
select {
|
||||
case jobs := <-got:
|
||||
if len(jobs) != 1 || jobs[0].Login != "ayse" || jobs[0].GPUs != 1 {
|
||||
t.Fatalf("jobs %+v", jobs)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("not recorded")
|
||||
}
|
||||
// The next status says it again: not recorded twice.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
r.svc.mu.Lock()
|
||||
busy := r.svc.recording
|
||||
r.svc.mu.Unlock()
|
||||
if !busy {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
|
||||
select {
|
||||
case jobs := <-got:
|
||||
t.Fatalf("recorded again: %+v", jobs)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user