Files
llmgw/internal/slurm/accounts.go
T
Ilker ManapandClaude Opus 5.5 fcda4d3f85
CI / test (push) Failing after 7m24s
Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 11:54:01 +01:00

194 lines
5.9 KiB
Go
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
package slurm
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"slices"
"strconv"
"strings"
"time"
)
// Slurm's accounts from llmgw's tree: an account for each organization,
// team and project, nested as they are, and on each project's the users
// that may use it, its members and its team's and organization's. A job
// is charged to the project its --account names (`sbatch --account
// acme-ml-training`), and what it used to that project's spend.
// Account is one Slurm account llmgw keeps.
type Account struct {
Name string
Parent string // "root" at the top
// Project is the llmgw project it is for; empty for an organization's
// or a team's.
Project string
// Users are the logins that may run jobs on it.
Users []string
}
// TenancyProject is a project with where it is.
type TenancyProject struct {
ID, Name, TeamID, TeamName, OrgID, OrgName string
}
// TenancyMember is a membership: a person in an organization, team or
// project ("org", "team", "project").
type TenancyMember struct {
UserID, Scope, ScopeID string
}
// Accounts are the accounts for projects, with logins by user id. Parents
// come before their children, as sacctmgr needs them.
func Accounts(projects []TenancyProject, members []TenancyMember, logins map[string]string) []Account {
// By kind and id: an organization, its team and its project may share
// an id (the first install's are all "default").
names := map[string]string{} // kind/id -> name
taken := map[string]string{} // name -> kind/id
name := func(id string, parts ...string) string {
if n, ok := names[id]; ok {
return n
}
n := AccountName(parts...)
if other, ok := taken[n]; ok && other != id {
// Two places whose names come out the same: told apart by id.
h := sha256.Sum256([]byte(id))
n = AccountName(append(parts, hex.EncodeToString(h[:3]))...)
}
names[id], taken[n] = n, id
return n
}
in := map[string][]string{} // kind/id -> logins
for _, m := range members {
if l, ok := logins[m.UserID]; ok {
in[m.Scope+"/"+m.ScopeID] = append(in[m.Scope+"/"+m.ScopeID], l)
}
}
ps := slices.Clone(projects)
slices.SortFunc(ps, func(a, b TenancyProject) int {
return strings.Compare(a.OrgName+"\x00"+a.TeamName+"\x00"+a.Name, b.OrgName+"\x00"+b.TeamName+"\x00"+b.Name)
})
var out []Account
seen := map[string]bool{}
for _, p := range ps {
org := name("org/"+p.OrgID, p.OrgName)
team := name("team/"+p.TeamID, p.OrgName, p.TeamName)
proj := name("project/"+p.ID, p.OrgName, p.TeamName, p.Name)
if !seen[org] {
out = append(out, Account{Name: org, Parent: "root"})
seen[org] = true
}
if !seen[team] {
out = append(out, Account{Name: team, Parent: org})
seen[team] = true
}
var users []string
for _, id := range []string{"project/" + p.ID, "team/" + p.TeamID, "org/" + p.OrgID} {
users = append(users, in[id]...)
}
slices.Sort(users)
out = append(out, Account{Name: proj, Parent: team, Project: p.ID, Users: slices.Compact(users)})
}
return out
}
// AccountName is a Slurm account name from names: lower case, ASCII,
// joined with "-", at most 64 characters.
func AccountName(parts ...string) string {
var segs []string
for _, p := range parts {
var b strings.Builder
for _, r := range strings.ToLower(p) {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_':
b.WriteRune(r)
case strings.ContainsRune("çğıöşüâîû", r):
b.WriteString(map[rune]string{'ç': "c", 'ğ': "g", 'ı': "i", 'ö': "o", 'ş': "s", 'ü': "u",
'â': "a", 'î': "i", 'û': "u"}[r])
default:
b.WriteByte('_')
}
}
if s := strings.Trim(b.String(), "_"); s != "" {
segs = append(segs, s)
}
}
n := strings.Join(segs, "-")
if n == "" {
n = "x"
}
if len(n) > 64 {
h := sha256.Sum256([]byte(n))
n = n[:57] + "-" + hex.EncodeToString(h[:3])
}
return n
}
// Association is one line of `sacctmgr show assoc format=account,user,parentname`.
type Association struct {
Account, User, Parent string
}
// AssocFormat is what ParseAssociations reads.
var AssocFormat = []string{"show", "assoc", "format=account,user,parentname", "-n", "-P"}
// ParseAssociations reads sacctmgr's parsable lines.
func ParseAssociations(out string) []Association {
var as []Association
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
f := strings.Split(line, "|")
if len(f) < 3 || f[0] == "" {
continue
}
as = append(as, Association{Account: f[0], User: f[1], Parent: f[2]})
}
return as
}
// Job is a finished job, as sacct has it.
type Job struct {
ID, User, Account, Partition, State string
Start, End time.Time
Elapsed time.Duration
CPUs, GPUs int
}
// SacctArgs are sacct's for the jobs that ended in the last day.
var SacctArgs = []string{"-a", "-X", "-n", "-P", "--noconvert", "-S", "now-1days", "-E", "now",
"-s", "CD,F,TO,CA,NF,OOM,PR,DL,BF",
"-o", "JobIDRaw,User,Account,Partition,Start,End,ElapsedRaw,AllocTRES,State"}
// ParseJobs reads sacct's lines; a job that has not ended is left out.
func ParseJobs(out string) ([]Job, error) {
var jobs []Job
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
if line == "" {
continue
}
f := strings.Split(line, "|")
if len(f) < 9 {
return nil, fmt.Errorf("sacct said %q, not 9 fields", line)
}
end, err := time.ParseInLocation("2006-01-02T15:04:05", f[5], time.Local)
if err != nil {
continue // Unknown: still running, or never started
}
start, _ := time.ParseInLocation("2006-01-02T15:04:05", f[4], time.Local)
secs, _ := strconv.ParseInt(f[6], 10, 64)
j := Job{ID: f[0], User: f[1], Account: f[2], Partition: f[3], Start: start, End: end,
Elapsed: time.Duration(secs) * time.Second, State: strings.Fields(f[8] + " ")[0]}
for _, t := range strings.Split(f[7], ",") {
k, v, _ := strings.Cut(t, "=")
n, _ := strconv.Atoi(v)
switch k {
case "cpu":
j.CPUs = n
case "gres/gpu":
j.GPUs = n
}
}
jobs = append(jobs, j)
}
return jobs, nil
}