Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s
CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8547528fb7
commit
fcda4d3f85
22 files changed
+1390
-76
No files matched your search
@@ -116,6 +116,8 @@ type Status struct {
|
||||
// machine has pyxis and enroot, "installing" until then, empty when
|
||||
// slurm.containers is off.
|
||||
Containers string `json:"containers,omitempty"`
|
||||
// Accounts is how many of Slurm's accounts are llmgw's projects'.
|
||||
Accounts int `json:"accounts,omitempty"`
|
||||
}
|
||||
|
||||
// User is a Linux user made from an llmgw account.
|
||||
@@ -166,6 +168,53 @@ type Service struct {
|
||||
users []User
|
||||
usersAt time.Time
|
||||
usersOK bool
|
||||
// accountList reads Slurm's accounts from the projects, with the users;
|
||||
// accountsOK is false until it has been read, and the controller is
|
||||
// told nothing of accounts until then: none would take every one away.
|
||||
accountList func(ctx context.Context) ([]slurm.Account, error)
|
||||
accounts []slurm.Account
|
||||
accountsOK bool
|
||||
// jobSink records the jobs that ended; recorded are those it took,
|
||||
// recording whether a recording is under way.
|
||||
jobSink func(ctx context.Context, jobs []JobRecord) error
|
||||
recorded map[string]time.Time
|
||||
recording bool
|
||||
}
|
||||
|
||||
// JobRecord is a job that ended, for llmgw's accounting.
|
||||
type JobRecord struct {
|
||||
ID, Login, Account, Partition, State string
|
||||
Start, End time.Time
|
||||
Elapsed time.Duration
|
||||
CPUs, GPUs int
|
||||
}
|
||||
|
||||
// SetAccounts has Slurm's accounts read with list, beside the users.
|
||||
func (s *Service) SetAccounts(list func(ctx context.Context) ([]slurm.Account, error)) {
|
||||
s.mu.Lock()
|
||||
s.accountList = list
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetJobs has the jobs that ended recorded with record; each is given to it
|
||||
// once it took it without an error.
|
||||
func (s *Service) SetJobs(record func(ctx context.Context, jobs []JobRecord) error) {
|
||||
s.mu.Lock()
|
||||
s.jobSink = record
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// AccountsOf are the accounts a login may run jobs on.
|
||||
func (s *Service) AccountsOf(login string) []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []string
|
||||
for _, a := range s.accounts {
|
||||
if slices.Contains(a.Users, login) {
|
||||
out = append(out, a.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// UsersEvery is how often the users are read again without being asked.
|
||||
@@ -199,6 +248,64 @@ func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
|
||||
return
|
||||
}
|
||||
s.users, s.usersOK = us, true
|
||||
if s.accountList == nil {
|
||||
return
|
||||
}
|
||||
as, err := s.accountList(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("could not read Slurm's accounts from the projects; the controller keeps the last ones", "err", err)
|
||||
return
|
||||
}
|
||||
s.accounts, s.accountsOK = as, true
|
||||
}
|
||||
|
||||
// takeJobs records the jobs a controller says ended that are not recorded
|
||||
// yet, away from the agent's stream.
|
||||
func (s *Service) takeJobs(ended []*llmgwv1.SlurmJob) {
|
||||
s.mu.Lock()
|
||||
if s.jobSink == nil || s.recording {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if s.recorded == nil {
|
||||
s.recorded = map[string]time.Time{}
|
||||
}
|
||||
for id, at := range s.recorded {
|
||||
if time.Since(at) > 3*24*time.Hour {
|
||||
delete(s.recorded, id)
|
||||
}
|
||||
}
|
||||
var jobs []JobRecord
|
||||
for _, j := range ended {
|
||||
if _, ok := s.recorded[j.GetId()]; ok || j.GetId() == "" {
|
||||
continue
|
||||
}
|
||||
jobs = append(jobs, JobRecord{ID: j.GetId(), Login: j.GetUser(), Account: j.GetAccount(), Partition: j.GetPartition(),
|
||||
State: j.GetState(), Start: time.Unix(j.GetStartUnix(), 0), End: time.Unix(j.GetEndUnix(), 0),
|
||||
Elapsed: time.Duration(j.GetElapsedSeconds()) * time.Second, CPUs: int(j.GetCpus()), GPUs: int(j.GetGpus())})
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
sink := s.jobSink
|
||||
s.recording = true
|
||||
s.mu.Unlock()
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
err := sink(ctx, jobs)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.recording = false
|
||||
if err != nil {
|
||||
s.log.Warn("could not record the Slurm jobs that ended; tried again with the next status", "jobs", len(jobs), "err", err)
|
||||
return
|
||||
}
|
||||
for _, j := range jobs {
|
||||
s.recorded[j.ID] = time.Now()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// HoldFor is how long a new leader waits for a node it neither has heard
|
||||
@@ -276,7 +383,11 @@ func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
|
||||
s.mu.Lock()
|
||||
prev := s.status[node]
|
||||
s.status[node] = r
|
||||
isCtl := s.settings().Controller == node
|
||||
s.mu.Unlock()
|
||||
if isCtl && len(st.GetEnded()) > 0 {
|
||||
s.takeJobs(st.GetEnded())
|
||||
}
|
||||
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
|
||||
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
|
||||
}
|
||||
@@ -517,6 +628,7 @@ func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
|
||||
return c, nil
|
||||
}
|
||||
c.ControllerAddr = s.controllerAddr(set)
|
||||
c.EnforceAccounts = set.Users == config.UsersLlmgw && s.accountList != nil
|
||||
var waiting []string
|
||||
for _, node := range set.Nodes {
|
||||
sn, ok := s.reported(node)
|
||||
@@ -582,6 +694,15 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
|
||||
if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
|
||||
return nil
|
||||
}
|
||||
if isCtl && set.Users == config.UsersLlmgw && s.accountList != nil {
|
||||
if !s.accountsOK {
|
||||
return nil
|
||||
}
|
||||
m.ManageAccounts = true
|
||||
for _, a := range s.accounts {
|
||||
m.Accounts = append(m.Accounts, &llmgwv1.SlurmAccount{Name: a.Name, Parent: a.Parent, Users: slices.Clone(a.Users)})
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -675,6 +796,11 @@ func (s *Service) Status() Status {
|
||||
Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
|
||||
if set.Users == config.UsersLlmgw {
|
||||
out.UserCount = len(s.users)
|
||||
for _, a := range s.accounts {
|
||||
if a.Project != "" {
|
||||
out.Accounts++
|
||||
}
|
||||
}
|
||||
}
|
||||
if !set.Enabled {
|
||||
return out
|
||||
|
||||
Reference in new issue
Block a user