Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s

With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-15 11:54:01 +01:00
1 parent 8547528fb7
commit fcda4d3f85
22 files changed
+1390 -76

No files matched your search

+126
View File
@@ -116,6 +116,8 @@ type Status struct {
// machine has pyxis and enroot, "installing" until then, empty when
// slurm.containers is off.
Containers string `json:"containers,omitempty"`
// Accounts is how many of Slurm's accounts are llmgw's projects'.
Accounts int `json:"accounts,omitempty"`
}
// User is a Linux user made from an llmgw account.
@@ -166,6 +168,53 @@ type Service struct {
users []User
usersAt time.Time
usersOK bool
// accountList reads Slurm's accounts from the projects, with the users;
// accountsOK is false until it has been read, and the controller is
// told nothing of accounts until then: none would take every one away.
accountList func(ctx context.Context) ([]slurm.Account, error)
accounts []slurm.Account
accountsOK bool
// jobSink records the jobs that ended; recorded are those it took,
// recording whether a recording is under way.
jobSink func(ctx context.Context, jobs []JobRecord) error
recorded map[string]time.Time
recording bool
}
// JobRecord is a job that ended, for llmgw's accounting.
type JobRecord struct {
ID, Login, Account, Partition, State string
Start, End time.Time
Elapsed time.Duration
CPUs, GPUs int
}
// SetAccounts has Slurm's accounts read with list, beside the users.
func (s *Service) SetAccounts(list func(ctx context.Context) ([]slurm.Account, error)) {
s.mu.Lock()
s.accountList = list
s.mu.Unlock()
}
// SetJobs has the jobs that ended recorded with record; each is given to it
// once it took it without an error.
func (s *Service) SetJobs(record func(ctx context.Context, jobs []JobRecord) error) {
s.mu.Lock()
s.jobSink = record
s.mu.Unlock()
}
// AccountsOf are the accounts a login may run jobs on.
func (s *Service) AccountsOf(login string) []string {
s.mu.Lock()
defer s.mu.Unlock()
var out []string
for _, a := range s.accounts {
if slices.Contains(a.Users, login) {
out = append(out, a.Name)
}
}
return out
}
// UsersEvery is how often the users are read again without being asked.
@@ -199,6 +248,64 @@ func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
return
}
s.users, s.usersOK = us, true
if s.accountList == nil {
return
}
as, err := s.accountList(ctx)
if err != nil {
s.log.Warn("could not read Slurm's accounts from the projects; the controller keeps the last ones", "err", err)
return
}
s.accounts, s.accountsOK = as, true
}
// takeJobs records the jobs a controller says ended that are not recorded
// yet, away from the agent's stream.
func (s *Service) takeJobs(ended []*llmgwv1.SlurmJob) {
s.mu.Lock()
if s.jobSink == nil || s.recording {
s.mu.Unlock()
return
}
if s.recorded == nil {
s.recorded = map[string]time.Time{}
}
for id, at := range s.recorded {
if time.Since(at) > 3*24*time.Hour {
delete(s.recorded, id)
}
}
var jobs []JobRecord
for _, j := range ended {
if _, ok := s.recorded[j.GetId()]; ok || j.GetId() == "" {
continue
}
jobs = append(jobs, JobRecord{ID: j.GetId(), Login: j.GetUser(), Account: j.GetAccount(), Partition: j.GetPartition(),
State: j.GetState(), Start: time.Unix(j.GetStartUnix(), 0), End: time.Unix(j.GetEndUnix(), 0),
Elapsed: time.Duration(j.GetElapsedSeconds()) * time.Second, CPUs: int(j.GetCpus()), GPUs: int(j.GetGpus())})
}
if len(jobs) == 0 {
s.mu.Unlock()
return
}
sink := s.jobSink
s.recording = true
s.mu.Unlock()
go func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
err := sink(ctx, jobs)
s.mu.Lock()
defer s.mu.Unlock()
s.recording = false
if err != nil {
s.log.Warn("could not record the Slurm jobs that ended; tried again with the next status", "jobs", len(jobs), "err", err)
return
}
for _, j := range jobs {
s.recorded[j.ID] = time.Now()
}
}()
}
// HoldFor is how long a new leader waits for a node it neither has heard
@@ -276,7 +383,11 @@ func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
s.mu.Lock()
prev := s.status[node]
s.status[node] = r
isCtl := s.settings().Controller == node
s.mu.Unlock()
if isCtl && len(st.GetEnded()) > 0 {
s.takeJobs(st.GetEnded())
}
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
}
@@ -517,6 +628,7 @@ func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
return c, nil
}
c.ControllerAddr = s.controllerAddr(set)
c.EnforceAccounts = set.Users == config.UsersLlmgw && s.accountList != nil
var waiting []string
for _, node := range set.Nodes {
sn, ok := s.reported(node)
@@ -582,6 +694,15 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
return nil
}
if isCtl && set.Users == config.UsersLlmgw && s.accountList != nil {
if !s.accountsOK {
return nil
}
m.ManageAccounts = true
for _, a := range s.accounts {
m.Accounts = append(m.Accounts, &llmgwv1.SlurmAccount{Name: a.Name, Parent: a.Parent, Users: slices.Clone(a.Users)})
}
}
return m
}
@@ -675,6 +796,11 @@ func (s *Service) Status() Status {
Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
if set.Users == config.UsersLlmgw {
out.UserCount = len(s.users)
for _, a := range s.accounts {
if a.Project != "" {
out.Accounts++
}
}
}
if !set.Enabled {
return out
+57
View File
@@ -11,6 +11,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
type fakeInv struct {
@@ -323,3 +324,59 @@ func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
t.Fatal("pyxis still loaded with containers off")
}
}
func TestTheControllerKeepsTheProjectsAccountsAndJobsAreRecordedOnce(t *testing.T) {
r := newRig()
r.set.Users = config.UsersLlmgw
r.svc.SetUsers(func(context.Context) ([]User, error) { return []User{{Login: "ayse", UID: 200001}}, nil })
read := 0
r.svc.SetAccounts(func(context.Context) ([]slurm.Account, error) {
read++
return []slurm.Account{{Name: "test", Parent: "root"},
{Name: "test-arge-deneme", Parent: "test", Project: "p1", Users: []string{"ayse"}}}, nil
})
got := make(chan []JobRecord, 4)
r.svc.SetJobs(func(_ context.Context, jobs []JobRecord) error { got <- jobs; return nil })
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
r.tick()
ctl := r.agents.last("node06")
if !ctl.GetManageAccounts() || len(ctl.GetAccounts()) != 2 || ctl.GetAccounts()[1].GetUsers()[0] != "ayse" {
t.Fatalf("controller told %v", ctl.GetAccounts())
}
if r.agents.last("cuda01").GetManageAccounts() {
t.Error("a node was told the accounts")
}
if !strings.Contains(ctl.GetFiles()["slurm.conf"], "AccountingStorageEnforce=associations") {
t.Error("jobs not held to accounts")
}
if r.svc.Status().Accounts != 1 || len(r.svc.AccountsOf("ayse")) != 1 {
t.Errorf("status %d, ayse's %v", r.svc.Status().Accounts, r.svc.AccountsOf("ayse"))
}
ended := []*llmgwv1.SlurmJob{{Id: "41", User: "ayse", Account: "test-arge-deneme", ElapsedSeconds: 60, Gpus: 1}}
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
select {
case jobs := <-got:
if len(jobs) != 1 || jobs[0].Login != "ayse" || jobs[0].GPUs != 1 {
t.Fatalf("jobs %+v", jobs)
}
case <-time.After(5 * time.Second):
t.Fatal("not recorded")
}
// The next status says it again: not recorded twice.
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
r.svc.mu.Lock()
busy := r.svc.recording
r.svc.mu.Unlock()
if !busy {
break
}
time.Sleep(10 * time.Millisecond)
}
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
select {
case jobs := <-got:
t.Fatalf("recorded again: %+v", jobs)
case <-time.After(200 * time.Millisecond):
}
}