Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s
CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8547528fb7
commit
fcda4d3f85
22 files changed
+1390
-76
No files matched your search
@@ -0,0 +1,193 @@
|
||||
package slurm
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Slurm's accounts from llmgw's tree: an account for each organization,
|
||||
// team and project, nested as they are, and on each project's the users
|
||||
// that may use it, its members and its team's and organization's. A job
|
||||
// is charged to the project its --account names (`sbatch --account
|
||||
// acme-ml-training`), and what it used to that project's spend.
|
||||
|
||||
// Account is one Slurm account llmgw keeps.
|
||||
type Account struct {
|
||||
Name string
|
||||
Parent string // "root" at the top
|
||||
// Project is the llmgw project it is for; empty for an organization's
|
||||
// or a team's.
|
||||
Project string
|
||||
// Users are the logins that may run jobs on it.
|
||||
Users []string
|
||||
}
|
||||
|
||||
// TenancyProject is a project with where it is.
|
||||
type TenancyProject struct {
|
||||
ID, Name, TeamID, TeamName, OrgID, OrgName string
|
||||
}
|
||||
|
||||
// TenancyMember is a membership: a person in an organization, team or
|
||||
// project ("org", "team", "project").
|
||||
type TenancyMember struct {
|
||||
UserID, Scope, ScopeID string
|
||||
}
|
||||
|
||||
// Accounts are the accounts for projects, with logins by user id. Parents
|
||||
// come before their children, as sacctmgr needs them.
|
||||
func Accounts(projects []TenancyProject, members []TenancyMember, logins map[string]string) []Account {
|
||||
// By kind and id: an organization, its team and its project may share
|
||||
// an id (the first install's are all "default").
|
||||
names := map[string]string{} // kind/id -> name
|
||||
taken := map[string]string{} // name -> kind/id
|
||||
name := func(id string, parts ...string) string {
|
||||
if n, ok := names[id]; ok {
|
||||
return n
|
||||
}
|
||||
n := AccountName(parts...)
|
||||
if other, ok := taken[n]; ok && other != id {
|
||||
// Two places whose names come out the same: told apart by id.
|
||||
h := sha256.Sum256([]byte(id))
|
||||
n = AccountName(append(parts, hex.EncodeToString(h[:3]))...)
|
||||
}
|
||||
names[id], taken[n] = n, id
|
||||
return n
|
||||
}
|
||||
in := map[string][]string{} // kind/id -> logins
|
||||
for _, m := range members {
|
||||
if l, ok := logins[m.UserID]; ok {
|
||||
in[m.Scope+"/"+m.ScopeID] = append(in[m.Scope+"/"+m.ScopeID], l)
|
||||
}
|
||||
}
|
||||
ps := slices.Clone(projects)
|
||||
slices.SortFunc(ps, func(a, b TenancyProject) int {
|
||||
return strings.Compare(a.OrgName+"\x00"+a.TeamName+"\x00"+a.Name, b.OrgName+"\x00"+b.TeamName+"\x00"+b.Name)
|
||||
})
|
||||
var out []Account
|
||||
seen := map[string]bool{}
|
||||
for _, p := range ps {
|
||||
org := name("org/"+p.OrgID, p.OrgName)
|
||||
team := name("team/"+p.TeamID, p.OrgName, p.TeamName)
|
||||
proj := name("project/"+p.ID, p.OrgName, p.TeamName, p.Name)
|
||||
if !seen[org] {
|
||||
out = append(out, Account{Name: org, Parent: "root"})
|
||||
seen[org] = true
|
||||
}
|
||||
if !seen[team] {
|
||||
out = append(out, Account{Name: team, Parent: org})
|
||||
seen[team] = true
|
||||
}
|
||||
var users []string
|
||||
for _, id := range []string{"project/" + p.ID, "team/" + p.TeamID, "org/" + p.OrgID} {
|
||||
users = append(users, in[id]...)
|
||||
}
|
||||
slices.Sort(users)
|
||||
out = append(out, Account{Name: proj, Parent: team, Project: p.ID, Users: slices.Compact(users)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AccountName is a Slurm account name from names: lower case, ASCII,
|
||||
// joined with "-", at most 64 characters.
|
||||
func AccountName(parts ...string) string {
|
||||
var segs []string
|
||||
for _, p := range parts {
|
||||
var b strings.Builder
|
||||
for _, r := range strings.ToLower(p) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_':
|
||||
b.WriteRune(r)
|
||||
case strings.ContainsRune("çğıöşüâîû", r):
|
||||
b.WriteString(map[rune]string{'ç': "c", 'ğ': "g", 'ı': "i", 'ö': "o", 'ş': "s", 'ü': "u",
|
||||
'â': "a", 'î': "i", 'û': "u"}[r])
|
||||
default:
|
||||
b.WriteByte('_')
|
||||
}
|
||||
}
|
||||
if s := strings.Trim(b.String(), "_"); s != "" {
|
||||
segs = append(segs, s)
|
||||
}
|
||||
}
|
||||
n := strings.Join(segs, "-")
|
||||
if n == "" {
|
||||
n = "x"
|
||||
}
|
||||
if len(n) > 64 {
|
||||
h := sha256.Sum256([]byte(n))
|
||||
n = n[:57] + "-" + hex.EncodeToString(h[:3])
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Association is one line of `sacctmgr show assoc format=account,user,parentname`.
|
||||
type Association struct {
|
||||
Account, User, Parent string
|
||||
}
|
||||
|
||||
// AssocFormat is what ParseAssociations reads.
|
||||
var AssocFormat = []string{"show", "assoc", "format=account,user,parentname", "-n", "-P"}
|
||||
|
||||
// ParseAssociations reads sacctmgr's parsable lines.
|
||||
func ParseAssociations(out string) []Association {
|
||||
var as []Association
|
||||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||
f := strings.Split(line, "|")
|
||||
if len(f) < 3 || f[0] == "" {
|
||||
continue
|
||||
}
|
||||
as = append(as, Association{Account: f[0], User: f[1], Parent: f[2]})
|
||||
}
|
||||
return as
|
||||
}
|
||||
|
||||
// Job is a finished job, as sacct has it.
|
||||
type Job struct {
|
||||
ID, User, Account, Partition, State string
|
||||
Start, End time.Time
|
||||
Elapsed time.Duration
|
||||
CPUs, GPUs int
|
||||
}
|
||||
|
||||
// SacctArgs are sacct's for the jobs that ended in the last day.
|
||||
var SacctArgs = []string{"-a", "-X", "-n", "-P", "--noconvert", "-S", "now-1days", "-E", "now",
|
||||
"-s", "CD,F,TO,CA,NF,OOM,PR,DL,BF",
|
||||
"-o", "JobIDRaw,User,Account,Partition,Start,End,ElapsedRaw,AllocTRES,State"}
|
||||
|
||||
// ParseJobs reads sacct's lines; a job that has not ended is left out.
|
||||
func ParseJobs(out string) ([]Job, error) {
|
||||
var jobs []Job
|
||||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
f := strings.Split(line, "|")
|
||||
if len(f) < 9 {
|
||||
return nil, fmt.Errorf("sacct said %q, not 9 fields", line)
|
||||
}
|
||||
end, err := time.ParseInLocation("2006-01-02T15:04:05", f[5], time.Local)
|
||||
if err != nil {
|
||||
continue // Unknown: still running, or never started
|
||||
}
|
||||
start, _ := time.ParseInLocation("2006-01-02T15:04:05", f[4], time.Local)
|
||||
secs, _ := strconv.ParseInt(f[6], 10, 64)
|
||||
j := Job{ID: f[0], User: f[1], Account: f[2], Partition: f[3], Start: start, End: end,
|
||||
Elapsed: time.Duration(secs) * time.Second, State: strings.Fields(f[8] + " ")[0]}
|
||||
for _, t := range strings.Split(f[7], ",") {
|
||||
k, v, _ := strings.Cut(t, "=")
|
||||
n, _ := strconv.Atoi(v)
|
||||
switch k {
|
||||
case "cpu":
|
||||
j.CPUs = n
|
||||
case "gres/gpu":
|
||||
j.GPUs = n
|
||||
}
|
||||
}
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package slurm
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAccountsFollowTheTree(t *testing.T) {
|
||||
projects := []TenancyProject{
|
||||
{ID: "p1", Name: "deneme", TeamID: "t1", TeamName: "arge", OrgID: "o1", OrgName: "Test"},
|
||||
{ID: "p2", Name: "Görüntü İşleme", TeamID: "t1", TeamName: "arge", OrgID: "o1", OrgName: "Test"},
|
||||
{ID: "p3", Name: "x", TeamID: "t2", TeamName: "ops", OrgID: "o2", OrgName: "Other"},
|
||||
}
|
||||
members := []TenancyMember{
|
||||
{UserID: "u-ayse", Scope: "project", ScopeID: "p1"},
|
||||
{UserID: "u-ada", Scope: "team", ScopeID: "t1"},
|
||||
{UserID: "u-bob", Scope: "org", ScopeID: "o2"},
|
||||
{UserID: "u-nologin", Scope: "project", ScopeID: "p1"},
|
||||
}
|
||||
logins := map[string]string{"u-ayse": "ayse", "u-ada": "ada", "u-bob": "bob"}
|
||||
got := Accounts(projects, members, logins)
|
||||
var names []string
|
||||
by := map[string]Account{}
|
||||
for _, a := range got {
|
||||
names = append(names, a.Name)
|
||||
by[a.Name] = a
|
||||
}
|
||||
// Parents first, each before its children.
|
||||
want := []string{"other", "other-ops", "other-ops-x", "test", "test-arge", "test-arge-goruntu_isleme", "test-arge-deneme"}
|
||||
if !slices.Equal(names, want) {
|
||||
t.Fatalf("names %v", names)
|
||||
}
|
||||
if a := by["test-arge-deneme"]; a.Parent != "test-arge" || a.Project != "p1" || !slices.Equal(a.Users, []string{"ada", "ayse"}) {
|
||||
t.Errorf("deneme %+v", a)
|
||||
}
|
||||
if a := by["other-ops-x"]; !slices.Equal(a.Users, []string{"bob"}) {
|
||||
t.Errorf("x %+v", a)
|
||||
}
|
||||
if a := by["test"]; a.Parent != "root" || a.Project != "" || len(a.Users) != 0 {
|
||||
t.Errorf("org %+v", a)
|
||||
}
|
||||
if n := AccountName("Test", "arge", "Görüntü İşleme"); n != "test-arge-goruntu_isleme" {
|
||||
t.Errorf("name %q", n)
|
||||
}
|
||||
// The first install's tree: one id for all three.
|
||||
def := Accounts([]TenancyProject{{ID: "default", Name: "default", TeamID: "default", TeamName: "default",
|
||||
OrgID: "default", OrgName: "default"}}, []TenancyMember{{UserID: "u", Scope: "project", ScopeID: "default"}},
|
||||
map[string]string{"u": "admin2"})
|
||||
if len(def) != 3 || def[2].Name != "default-default-default" || def[2].Parent != "default-default" ||
|
||||
def[1].Parent != "default" || !slices.Equal(def[2].Users, []string{"admin2"}) {
|
||||
t.Errorf("default tree %+v", def)
|
||||
}
|
||||
// Two that come out the same are told apart.
|
||||
same := Accounts([]TenancyProject{
|
||||
{ID: "a", Name: "ml", TeamID: "t", TeamName: "x", OrgID: "o", OrgName: "A"},
|
||||
{ID: "b", Name: "ML", TeamID: "t", TeamName: "x", OrgID: "o", OrgName: "A"},
|
||||
}, nil, nil)
|
||||
if same[2].Name == same[3].Name {
|
||||
t.Errorf("same name twice: %+v", same)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJobs(t *testing.T) {
|
||||
out := "41|ayse|test-arge-deneme|gpu|2026-11-15T10:00:00|2026-11-15T10:30:00|1800|billing=4,cpu=4,gres/gpu=1,mem=16G,node=1|COMPLETED\n" +
|
||||
"42|ada|test-arge-deneme|gpu|2026-11-15T10:00:00|Unknown|0|cpu=2|RUNNING\n" +
|
||||
"43|root|root|gpu|2026-11-15T09:00:00|2026-11-15T09:00:05|5|cpu=1,mem=1G|CANCELLED by 0\n"
|
||||
jobs, err := ParseJobs(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(jobs) != 2 {
|
||||
t.Fatalf("jobs %+v", jobs)
|
||||
}
|
||||
j := jobs[0]
|
||||
if j.ID != "41" || j.User != "ayse" || j.GPUs != 1 || j.CPUs != 4 || j.Elapsed != 30*time.Minute || j.State != "COMPLETED" {
|
||||
t.Errorf("job %+v", j)
|
||||
}
|
||||
if jobs[1].State != "CANCELLED" {
|
||||
t.Errorf("state %q", jobs[1].State)
|
||||
}
|
||||
if _, err := ParseJobs("short|line\n"); err == nil {
|
||||
t.Error("a short line taken")
|
||||
}
|
||||
as := ParseAssociations("root||\ntest|ayse|test-arge\n")
|
||||
if len(as) != 2 || as[1] != (Association{"test", "ayse", "test-arge"}) {
|
||||
t.Errorf("assoc %+v", as)
|
||||
}
|
||||
}
|
||||
@@ -131,6 +131,10 @@ type Cluster struct {
|
||||
Controller string
|
||||
ControllerAddr string
|
||||
Partition string
|
||||
// EnforceAccounts refuses a job whose user has no account in Slurm's
|
||||
// accounting: with llmgw's users, each is on the accounts of the
|
||||
// projects they are in, and a job is charged to one of them.
|
||||
EnforceAccounts bool
|
||||
// Nodes without hardware (not reported yet) are left out of the files
|
||||
// until they are.
|
||||
Nodes []Node
|
||||
@@ -199,6 +203,9 @@ func (c Cluster) SlurmConf() string {
|
||||
// The jobs' records, in slurmdbd beside slurmctld (DbdConf).
|
||||
fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n",
|
||||
c.ControllerAddr, DbdPort)
|
||||
if c.EnforceAccounts {
|
||||
b.WriteString("# A job runs on an account of a project its user is in.\nAccountingStorageEnforce=associations\n")
|
||||
}
|
||||
b.WriteString(`SlurmctldParameters=enable_configless
|
||||
SlurmctldPort=6817
|
||||
SlurmdPort=6818
|
||||
|
||||
Reference in new issue
Block a user