CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
194 lines
5.9 KiB
Go
194 lines
5.9 KiB
Go
package slurm
|
||
|
||
import (
|
||
"crypto/sha256"
|
||
"encoding/hex"
|
||
"fmt"
|
||
"slices"
|
||
"strconv"
|
||
"strings"
|
||
"time"
|
||
)
|
||
|
||
// Slurm's accounts from llmgw's tree: an account for each organization,
|
||
// team and project, nested as they are, and on each project's the users
|
||
// that may use it, its members and its team's and organization's. A job
|
||
// is charged to the project its --account names (`sbatch --account
|
||
// acme-ml-training`), and what it used to that project's spend.
|
||
|
||
// Account is one Slurm account llmgw keeps.
|
||
type Account struct {
|
||
Name string
|
||
Parent string // "root" at the top
|
||
// Project is the llmgw project it is for; empty for an organization's
|
||
// or a team's.
|
||
Project string
|
||
// Users are the logins that may run jobs on it.
|
||
Users []string
|
||
}
|
||
|
||
// TenancyProject is a project with where it is.
|
||
type TenancyProject struct {
|
||
ID, Name, TeamID, TeamName, OrgID, OrgName string
|
||
}
|
||
|
||
// TenancyMember is a membership: a person in an organization, team or
|
||
// project ("org", "team", "project").
|
||
type TenancyMember struct {
|
||
UserID, Scope, ScopeID string
|
||
}
|
||
|
||
// Accounts are the accounts for projects, with logins by user id. Parents
|
||
// come before their children, as sacctmgr needs them.
|
||
func Accounts(projects []TenancyProject, members []TenancyMember, logins map[string]string) []Account {
|
||
// By kind and id: an organization, its team and its project may share
|
||
// an id (the first install's are all "default").
|
||
names := map[string]string{} // kind/id -> name
|
||
taken := map[string]string{} // name -> kind/id
|
||
name := func(id string, parts ...string) string {
|
||
if n, ok := names[id]; ok {
|
||
return n
|
||
}
|
||
n := AccountName(parts...)
|
||
if other, ok := taken[n]; ok && other != id {
|
||
// Two places whose names come out the same: told apart by id.
|
||
h := sha256.Sum256([]byte(id))
|
||
n = AccountName(append(parts, hex.EncodeToString(h[:3]))...)
|
||
}
|
||
names[id], taken[n] = n, id
|
||
return n
|
||
}
|
||
in := map[string][]string{} // kind/id -> logins
|
||
for _, m := range members {
|
||
if l, ok := logins[m.UserID]; ok {
|
||
in[m.Scope+"/"+m.ScopeID] = append(in[m.Scope+"/"+m.ScopeID], l)
|
||
}
|
||
}
|
||
ps := slices.Clone(projects)
|
||
slices.SortFunc(ps, func(a, b TenancyProject) int {
|
||
return strings.Compare(a.OrgName+"\x00"+a.TeamName+"\x00"+a.Name, b.OrgName+"\x00"+b.TeamName+"\x00"+b.Name)
|
||
})
|
||
var out []Account
|
||
seen := map[string]bool{}
|
||
for _, p := range ps {
|
||
org := name("org/"+p.OrgID, p.OrgName)
|
||
team := name("team/"+p.TeamID, p.OrgName, p.TeamName)
|
||
proj := name("project/"+p.ID, p.OrgName, p.TeamName, p.Name)
|
||
if !seen[org] {
|
||
out = append(out, Account{Name: org, Parent: "root"})
|
||
seen[org] = true
|
||
}
|
||
if !seen[team] {
|
||
out = append(out, Account{Name: team, Parent: org})
|
||
seen[team] = true
|
||
}
|
||
var users []string
|
||
for _, id := range []string{"project/" + p.ID, "team/" + p.TeamID, "org/" + p.OrgID} {
|
||
users = append(users, in[id]...)
|
||
}
|
||
slices.Sort(users)
|
||
out = append(out, Account{Name: proj, Parent: team, Project: p.ID, Users: slices.Compact(users)})
|
||
}
|
||
return out
|
||
}
|
||
|
||
// AccountName is a Slurm account name from names: lower case, ASCII,
|
||
// joined with "-", at most 64 characters.
|
||
func AccountName(parts ...string) string {
|
||
var segs []string
|
||
for _, p := range parts {
|
||
var b strings.Builder
|
||
for _, r := range strings.ToLower(p) {
|
||
switch {
|
||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_':
|
||
b.WriteRune(r)
|
||
case strings.ContainsRune("çğıöşüâîû", r):
|
||
b.WriteString(map[rune]string{'ç': "c", 'ğ': "g", 'ı': "i", 'ö': "o", 'ş': "s", 'ü': "u",
|
||
'â': "a", 'î': "i", 'û': "u"}[r])
|
||
default:
|
||
b.WriteByte('_')
|
||
}
|
||
}
|
||
if s := strings.Trim(b.String(), "_"); s != "" {
|
||
segs = append(segs, s)
|
||
}
|
||
}
|
||
n := strings.Join(segs, "-")
|
||
if n == "" {
|
||
n = "x"
|
||
}
|
||
if len(n) > 64 {
|
||
h := sha256.Sum256([]byte(n))
|
||
n = n[:57] + "-" + hex.EncodeToString(h[:3])
|
||
}
|
||
return n
|
||
}
|
||
|
||
// Association is one line of `sacctmgr show assoc format=account,user,parentname`.
|
||
type Association struct {
|
||
Account, User, Parent string
|
||
}
|
||
|
||
// AssocFormat is what ParseAssociations reads.
|
||
var AssocFormat = []string{"show", "assoc", "format=account,user,parentname", "-n", "-P"}
|
||
|
||
// ParseAssociations reads sacctmgr's parsable lines.
|
||
func ParseAssociations(out string) []Association {
|
||
var as []Association
|
||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||
f := strings.Split(line, "|")
|
||
if len(f) < 3 || f[0] == "" {
|
||
continue
|
||
}
|
||
as = append(as, Association{Account: f[0], User: f[1], Parent: f[2]})
|
||
}
|
||
return as
|
||
}
|
||
|
||
// Job is a finished job, as sacct has it.
|
||
type Job struct {
|
||
ID, User, Account, Partition, State string
|
||
Start, End time.Time
|
||
Elapsed time.Duration
|
||
CPUs, GPUs int
|
||
}
|
||
|
||
// SacctArgs are sacct's for the jobs that ended in the last day.
|
||
var SacctArgs = []string{"-a", "-X", "-n", "-P", "--noconvert", "-S", "now-1days", "-E", "now",
|
||
"-s", "CD,F,TO,CA,NF,OOM,PR,DL,BF",
|
||
"-o", "JobIDRaw,User,Account,Partition,Start,End,ElapsedRaw,AllocTRES,State"}
|
||
|
||
// ParseJobs reads sacct's lines; a job that has not ended is left out.
|
||
func ParseJobs(out string) ([]Job, error) {
|
||
var jobs []Job
|
||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||
if line == "" {
|
||
continue
|
||
}
|
||
f := strings.Split(line, "|")
|
||
if len(f) < 9 {
|
||
return nil, fmt.Errorf("sacct said %q, not 9 fields", line)
|
||
}
|
||
end, err := time.ParseInLocation("2006-01-02T15:04:05", f[5], time.Local)
|
||
if err != nil {
|
||
continue // Unknown: still running, or never started
|
||
}
|
||
start, _ := time.ParseInLocation("2006-01-02T15:04:05", f[4], time.Local)
|
||
secs, _ := strconv.ParseInt(f[6], 10, 64)
|
||
j := Job{ID: f[0], User: f[1], Account: f[2], Partition: f[3], Start: start, End: end,
|
||
Elapsed: time.Duration(secs) * time.Second, State: strings.Fields(f[8] + " ")[0]}
|
||
for _, t := range strings.Split(f[7], ",") {
|
||
k, v, _ := strings.Cut(t, "=")
|
||
n, _ := strconv.Atoi(v)
|
||
switch k {
|
||
case "cpu":
|
||
j.CPUs = n
|
||
case "gres/gpu":
|
||
j.GPUs = n
|
||
}
|
||
}
|
||
jobs = append(jobs, j)
|
||
}
|
||
return jobs, nil
|
||
}
|