Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s

With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-15 11:54:01 +01:00
1 parent 8547528fb7
commit fcda4d3f85
22 files changed
+1390 -76

No files matched your search

+193
View File
@@ -0,0 +1,193 @@
package slurm
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"slices"
"strconv"
"strings"
"time"
)
// Slurm's accounts from llmgw's tree: an account for each organization,
// team and project, nested as they are, and on each project's the users
// that may use it, its members and its team's and organization's. A job
// is charged to the project its --account names (`sbatch --account
// acme-ml-training`), and what it used to that project's spend.
// Account is one Slurm account llmgw keeps.
type Account struct {
Name string
Parent string // "root" at the top
// Project is the llmgw project it is for; empty for an organization's
// or a team's.
Project string
// Users are the logins that may run jobs on it.
Users []string
}
// TenancyProject is a project with where it is.
type TenancyProject struct {
ID, Name, TeamID, TeamName, OrgID, OrgName string
}
// TenancyMember is a membership: a person in an organization, team or
// project ("org", "team", "project").
type TenancyMember struct {
UserID, Scope, ScopeID string
}
// Accounts are the accounts for projects, with logins by user id. Parents
// come before their children, as sacctmgr needs them.
func Accounts(projects []TenancyProject, members []TenancyMember, logins map[string]string) []Account {
// By kind and id: an organization, its team and its project may share
// an id (the first install's are all "default").
names := map[string]string{} // kind/id -> name
taken := map[string]string{} // name -> kind/id
name := func(id string, parts ...string) string {
if n, ok := names[id]; ok {
return n
}
n := AccountName(parts...)
if other, ok := taken[n]; ok && other != id {
// Two places whose names come out the same: told apart by id.
h := sha256.Sum256([]byte(id))
n = AccountName(append(parts, hex.EncodeToString(h[:3]))...)
}
names[id], taken[n] = n, id
return n
}
in := map[string][]string{} // kind/id -> logins
for _, m := range members {
if l, ok := logins[m.UserID]; ok {
in[m.Scope+"/"+m.ScopeID] = append(in[m.Scope+"/"+m.ScopeID], l)
}
}
ps := slices.Clone(projects)
slices.SortFunc(ps, func(a, b TenancyProject) int {
return strings.Compare(a.OrgName+"\x00"+a.TeamName+"\x00"+a.Name, b.OrgName+"\x00"+b.TeamName+"\x00"+b.Name)
})
var out []Account
seen := map[string]bool{}
for _, p := range ps {
org := name("org/"+p.OrgID, p.OrgName)
team := name("team/"+p.TeamID, p.OrgName, p.TeamName)
proj := name("project/"+p.ID, p.OrgName, p.TeamName, p.Name)
if !seen[org] {
out = append(out, Account{Name: org, Parent: "root"})
seen[org] = true
}
if !seen[team] {
out = append(out, Account{Name: team, Parent: org})
seen[team] = true
}
var users []string
for _, id := range []string{"project/" + p.ID, "team/" + p.TeamID, "org/" + p.OrgID} {
users = append(users, in[id]...)
}
slices.Sort(users)
out = append(out, Account{Name: proj, Parent: team, Project: p.ID, Users: slices.Compact(users)})
}
return out
}
// AccountName is a Slurm account name from names: lower case, ASCII,
// joined with "-", at most 64 characters.
func AccountName(parts ...string) string {
var segs []string
for _, p := range parts {
var b strings.Builder
for _, r := range strings.ToLower(p) {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_':
b.WriteRune(r)
case strings.ContainsRune("çğıöşüâîû", r):
b.WriteString(map[rune]string{'ç': "c", 'ğ': "g", 'ı': "i", 'ö': "o", 'ş': "s", 'ü': "u",
'â': "a", 'î': "i", 'û': "u"}[r])
default:
b.WriteByte('_')
}
}
if s := strings.Trim(b.String(), "_"); s != "" {
segs = append(segs, s)
}
}
n := strings.Join(segs, "-")
if n == "" {
n = "x"
}
if len(n) > 64 {
h := sha256.Sum256([]byte(n))
n = n[:57] + "-" + hex.EncodeToString(h[:3])
}
return n
}
// Association is one line of `sacctmgr show assoc format=account,user,parentname`.
type Association struct {
Account, User, Parent string
}
// AssocFormat is what ParseAssociations reads.
var AssocFormat = []string{"show", "assoc", "format=account,user,parentname", "-n", "-P"}
// ParseAssociations reads sacctmgr's parsable lines.
func ParseAssociations(out string) []Association {
var as []Association
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
f := strings.Split(line, "|")
if len(f) < 3 || f[0] == "" {
continue
}
as = append(as, Association{Account: f[0], User: f[1], Parent: f[2]})
}
return as
}
// Job is a finished job, as sacct has it.
type Job struct {
ID, User, Account, Partition, State string
Start, End time.Time
Elapsed time.Duration
CPUs, GPUs int
}
// SacctArgs are sacct's for the jobs that ended in the last day.
var SacctArgs = []string{"-a", "-X", "-n", "-P", "--noconvert", "-S", "now-1days", "-E", "now",
"-s", "CD,F,TO,CA,NF,OOM,PR,DL,BF",
"-o", "JobIDRaw,User,Account,Partition,Start,End,ElapsedRaw,AllocTRES,State"}
// ParseJobs reads sacct's lines; a job that has not ended is left out.
func ParseJobs(out string) ([]Job, error) {
var jobs []Job
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
if line == "" {
continue
}
f := strings.Split(line, "|")
if len(f) < 9 {
return nil, fmt.Errorf("sacct said %q, not 9 fields", line)
}
end, err := time.ParseInLocation("2006-01-02T15:04:05", f[5], time.Local)
if err != nil {
continue // Unknown: still running, or never started
}
start, _ := time.ParseInLocation("2006-01-02T15:04:05", f[4], time.Local)
secs, _ := strconv.ParseInt(f[6], 10, 64)
j := Job{ID: f[0], User: f[1], Account: f[2], Partition: f[3], Start: start, End: end,
Elapsed: time.Duration(secs) * time.Second, State: strings.Fields(f[8] + " ")[0]}
for _, t := range strings.Split(f[7], ",") {
k, v, _ := strings.Cut(t, "=")
n, _ := strconv.Atoi(v)
switch k {
case "cpu":
j.CPUs = n
case "gres/gpu":
j.GPUs = n
}
}
jobs = append(jobs, j)
}
return jobs, nil
}
+89
View File
@@ -0,0 +1,89 @@
package slurm
import (
"slices"
"testing"
"time"
)
func TestAccountsFollowTheTree(t *testing.T) {
projects := []TenancyProject{
{ID: "p1", Name: "deneme", TeamID: "t1", TeamName: "arge", OrgID: "o1", OrgName: "Test"},
{ID: "p2", Name: "Görüntü İşleme", TeamID: "t1", TeamName: "arge", OrgID: "o1", OrgName: "Test"},
{ID: "p3", Name: "x", TeamID: "t2", TeamName: "ops", OrgID: "o2", OrgName: "Other"},
}
members := []TenancyMember{
{UserID: "u-ayse", Scope: "project", ScopeID: "p1"},
{UserID: "u-ada", Scope: "team", ScopeID: "t1"},
{UserID: "u-bob", Scope: "org", ScopeID: "o2"},
{UserID: "u-nologin", Scope: "project", ScopeID: "p1"},
}
logins := map[string]string{"u-ayse": "ayse", "u-ada": "ada", "u-bob": "bob"}
got := Accounts(projects, members, logins)
var names []string
by := map[string]Account{}
for _, a := range got {
names = append(names, a.Name)
by[a.Name] = a
}
// Parents first, each before its children.
want := []string{"other", "other-ops", "other-ops-x", "test", "test-arge", "test-arge-goruntu_isleme", "test-arge-deneme"}
if !slices.Equal(names, want) {
t.Fatalf("names %v", names)
}
if a := by["test-arge-deneme"]; a.Parent != "test-arge" || a.Project != "p1" || !slices.Equal(a.Users, []string{"ada", "ayse"}) {
t.Errorf("deneme %+v", a)
}
if a := by["other-ops-x"]; !slices.Equal(a.Users, []string{"bob"}) {
t.Errorf("x %+v", a)
}
if a := by["test"]; a.Parent != "root" || a.Project != "" || len(a.Users) != 0 {
t.Errorf("org %+v", a)
}
if n := AccountName("Test", "arge", "Görüntü İşleme"); n != "test-arge-goruntu_isleme" {
t.Errorf("name %q", n)
}
// The first install's tree: one id for all three.
def := Accounts([]TenancyProject{{ID: "default", Name: "default", TeamID: "default", TeamName: "default",
OrgID: "default", OrgName: "default"}}, []TenancyMember{{UserID: "u", Scope: "project", ScopeID: "default"}},
map[string]string{"u": "admin2"})
if len(def) != 3 || def[2].Name != "default-default-default" || def[2].Parent != "default-default" ||
def[1].Parent != "default" || !slices.Equal(def[2].Users, []string{"admin2"}) {
t.Errorf("default tree %+v", def)
}
// Two that come out the same are told apart.
same := Accounts([]TenancyProject{
{ID: "a", Name: "ml", TeamID: "t", TeamName: "x", OrgID: "o", OrgName: "A"},
{ID: "b", Name: "ML", TeamID: "t", TeamName: "x", OrgID: "o", OrgName: "A"},
}, nil, nil)
if same[2].Name == same[3].Name {
t.Errorf("same name twice: %+v", same)
}
}
func TestParseJobs(t *testing.T) {
out := "41|ayse|test-arge-deneme|gpu|2026-11-15T10:00:00|2026-11-15T10:30:00|1800|billing=4,cpu=4,gres/gpu=1,mem=16G,node=1|COMPLETED\n" +
"42|ada|test-arge-deneme|gpu|2026-11-15T10:00:00|Unknown|0|cpu=2|RUNNING\n" +
"43|root|root|gpu|2026-11-15T09:00:00|2026-11-15T09:00:05|5|cpu=1,mem=1G|CANCELLED by 0\n"
jobs, err := ParseJobs(out)
if err != nil {
t.Fatal(err)
}
if len(jobs) != 2 {
t.Fatalf("jobs %+v", jobs)
}
j := jobs[0]
if j.ID != "41" || j.User != "ayse" || j.GPUs != 1 || j.CPUs != 4 || j.Elapsed != 30*time.Minute || j.State != "COMPLETED" {
t.Errorf("job %+v", j)
}
if jobs[1].State != "CANCELLED" {
t.Errorf("state %q", jobs[1].State)
}
if _, err := ParseJobs("short|line\n"); err == nil {
t.Error("a short line taken")
}
as := ParseAssociations("root||\ntest|ayse|test-arge\n")
if len(as) != 2 || as[1] != (Association{"test", "ayse", "test-arge"}) {
t.Errorf("assoc %+v", as)
}
}
+7
View File
@@ -131,6 +131,10 @@ type Cluster struct {
Controller string
ControllerAddr string
Partition string
// EnforceAccounts refuses a job whose user has no account in Slurm's
// accounting: with llmgw's users, each is on the accounts of the
// projects they are in, and a job is charged to one of them.
EnforceAccounts bool
// Nodes without hardware (not reported yet) are left out of the files
// until they are.
Nodes []Node
@@ -199,6 +203,9 @@ func (c Cluster) SlurmConf() string {
// The jobs' records, in slurmdbd beside slurmctld (DbdConf).
fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n",
c.ControllerAddr, DbdPort)
if c.EnforceAccounts {
b.WriteString("# A job runs on an account of a project its user is in.\nAccountingStorageEnforce=associations\n")
}
b.WriteString(`SlurmctldParameters=enable_configless
SlurmctldPort=6817
SlurmdPort=6818