Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s

With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-15 11:54:01 +01:00
1 parent 8547528fb7
commit fcda4d3f85
22 files changed
+1390 -76

No files matched your search

+214
View File
@@ -0,0 +1,214 @@
package slurmnode
import (
"context"
"errors"
"fmt"
"slices"
"strings"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// accountsMarker lists what llmgw made in Slurm's accounting, a line each:
// "account NAME" and "user LOGIN NAME". Nothing else is changed or taken
// away: an account a Slurm administrator made is theirs.
func (k *Keeper) accountsMarker() string { return k.marker + "-accounts" }
// maxEnded is the most ended jobs sent at once: a day's on a busy cluster
// would otherwise be a status message of megabytes.
const maxEnded = 2000
// accounts keeps Slurm's accounts and their users as want says, on the
// controller.
func (k *Keeper) accounts(ctx context.Context, want *llmgwv1.Slurm) error {
made := k.madeAccounts()
if !want.GetManageAccounts() {
if len(made) == 0 {
return nil
}
return k.applyAccounts(ctx, nil, made)
}
return k.applyAccounts(ctx, want.GetAccounts(), made)
}
func (k *Keeper) madeAccounts() []string {
b, err := k.host.ReadFile(k.accountsMarker())
if err != nil {
return nil
}
var out []string
for _, l := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if l != "" {
out = append(out, l)
}
}
return out
}
func (k *Keeper) sacctmgr(ctx context.Context, args ...string) error {
out, err := k.host.Run(ctx, nil, "sacctmgr", append([]string{"-i"}, args...)...)
if err != nil {
return fmt.Errorf("sacctmgr %s: %v: %s", strings.Join(args, " "), err, lastLine(out))
}
return nil
}
func (k *Keeper) applyAccounts(ctx context.Context, want []*llmgwv1.SlurmAccount, made []string) error {
out, err := k.host.Run(ctx, nil, "sacctmgr", slurm.AssocFormat...)
if err != nil {
return fmt.Errorf("reading Slurm's accounts: %v: %s", err, lastLine(out))
}
have := map[string]bool{} // "account NAME", "user LOGIN NAME"
for _, a := range slurm.ParseAssociations(string(out)) {
if a.User == "" {
have["account "+a.Account] = true
} else {
have["user "+a.User+" "+a.Account] = true
}
}
keep := map[string]bool{}
var errs []error
ours := slices.Clone(made)
note := func(line string) {
if !slices.Contains(ours, line) {
ours = append(ours, line)
}
}
for _, a := range want {
name, parent := a.GetName(), a.GetParent()
if !validAccount(name) || (parent != "root" && !validAccount(parent)) {
errs = append(errs, fmt.Errorf("the leader sent an account %q under %q", name, parent))
continue
}
line := "account " + name
keep[line] = true
if !have[line] {
k.log.Info("a Slurm account is made, as llmgw's projects say", "account", name, "parent", parent)
if err := k.sacctmgr(ctx, "add", "account", name, "parent="+parent, "description=llmgw"); err != nil {
errs = append(errs, err)
continue
}
note(line)
}
for _, u := range a.GetUsers() {
if !slurm.ValidLogin(u) {
continue
}
ul := "user " + u + " " + name
keep[ul] = true
if have[ul] {
continue
}
if err := k.sacctmgr(ctx, "add", "user", u, "account="+name); err != nil {
errs = append(errs, err)
continue
}
note(ul)
}
}
// Taken away: what llmgw made and no longer wants, users before their
// accounts, children before their parents (made later, listed later).
var left []string
for i := len(ours) - 1; i >= 0; i-- {
line := ours[i]
if keep[line] || !strings.HasPrefix(line, "user ") {
continue
}
f := strings.Fields(line)
if !have[line] {
continue
}
if err := k.removeAssoc(ctx, f[1], f[2], have, keep); err != nil {
errs = append(errs, err)
left = append(left, line)
}
}
for i := len(ours) - 1; i >= 0; i-- {
line := ours[i]
if keep[line] || !strings.HasPrefix(line, "account ") {
continue
}
if have[line] {
k.log.Info("a Slurm account llmgw made is taken away; its jobs' records stay", "account", line[len("account "):])
if err := k.sacctmgr(ctx, "remove", "account", line[len("account "):]); err != nil {
errs = append(errs, err)
left = append(left, line)
}
}
}
var lines []string
for _, l := range ours {
if keep[l] || slices.Contains(left, l) {
lines = append(lines, l)
}
}
if !slices.Equal(lines, made) {
var err error
if len(lines) == 0 {
err = k.host.Remove(k.accountsMarker())
} else {
_, err = k.write(k.accountsMarker(), []byte(strings.Join(lines, "\n")+"\n"), 0o644, "")
}
errs = append(errs, err)
}
return errors.Join(errs...)
}
func validAccount(s string) bool {
return s != "" && len(s) <= 64 && strings.Trim(s, "abcdefghijklmnopqrstuvwxyz0123456789_-") == ""
}
// ended are the jobs that ended in the last day, newest last.
func (k *Keeper) ended(ctx context.Context) ([]*llmgwv1.SlurmJob, error) {
out, err := k.host.Run(ctx, nil, "sacct", slurm.SacctArgs...)
if err != nil {
return nil, fmt.Errorf("sacct: %v: %s", err, lastLine(out))
}
jobs, err := slurm.ParseJobs(string(out))
if err != nil {
return nil, err
}
if len(jobs) > maxEnded {
jobs = jobs[len(jobs)-maxEnded:]
}
ps := make([]*llmgwv1.SlurmJob, 0, len(jobs))
for _, j := range jobs {
ps = append(ps, &llmgwv1.SlurmJob{Id: j.ID, User: j.User, Account: j.Account, Partition: j.Partition,
State: j.State, StartUnix: j.Start.Unix(), EndUnix: j.End.Unix(), ElapsedSeconds: int64(j.Elapsed.Seconds()),
Cpus: int32(j.CPUs), Gpus: int32(j.GPUs)}) //nolint:gosec // counts of a node's
}
return ps, nil
}
// removeAssoc takes login off account. Slurm refuses to take away a
// user's default account, so another of theirs is made the default first,
// one llmgw keeps if there is one; with none left the user goes from the
// accounting entirely.
func (k *Keeper) removeAssoc(ctx context.Context, login, account string, have, keep map[string]bool) error {
var others []string
for line := range have {
f := strings.Fields(line)
if len(f) == 3 && f[0] == "user" && f[1] == login && f[2] != account {
others = append(others, f[2])
}
}
if len(others) == 0 {
return k.sacctmgr(ctx, "remove", "user", login)
}
slices.SortFunc(others, func(a, b string) int {
ka, kb := keep["user "+login+" "+a], keep["user "+login+" "+b]
switch {
case ka && !kb:
return -1
case kb && !ka:
return 1
}
return strings.Compare(a, b)
})
if err := k.sacctmgr(ctx, "modify", "user", login, "set", "defaultaccount="+others[0]); err != nil {
return err
}
return k.sacctmgr(ctx, "remove", "user", login, "account="+account)
}
+83
View File
@@ -0,0 +1,83 @@
package slurmnode
import (
"context"
"slices"
"strings"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
func TestAccountsMadeAndTakenAwayOnlyIfLlmgwsOwn(t *testing.T) {
h := newFake()
k := New(h, nil, "/var/lib/llmgw-agent/slurm", nil)
assoc := "sacctmgr " + strings.Join(slurm.AssocFormat, " ")
// root, and an account a Slurm administrator made.
h.outputs[assoc] = "root||\nroot|root|\nphysics||root\n"
m := &llmgwv1.Slurm{ManageAccounts: true, Accounts: []*llmgwv1.SlurmAccount{
{Name: "test", Parent: "root"}, {Name: "test-arge", Parent: "test"},
{Name: "test-arge-deneme", Parent: "test-arge", Users: []string{"ada", "ayse"}},
}}
if err := k.accounts(context.Background(), m); err != nil {
t.Fatal(err)
}
for _, want := range []string{"sacctmgr -i add account test parent=root description=llmgw",
"sacctmgr -i add account test-arge-deneme parent=test-arge description=llmgw",
"sacctmgr -i add user ayse account=test-arge-deneme"} {
if !slices.Contains(h.ran, want) {
t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n"))
}
}
if !strings.Contains(string(h.files[k.accountsMarker()]), "user ada test-arge-deneme") {
t.Errorf("marker %q", h.files[k.accountsMarker()])
}
// Now there; ayse leaves the project, and the team goes.
h.outputs[assoc] = "root||\nphysics||root\ntest||root\ntest-arge||test\ntest-arge-deneme||test-arge\n" +
"test-arge-deneme|ada|test-arge\ntest-arge-deneme|ayse|test-arge\n"
h.ran = nil
m.Accounts = []*llmgwv1.SlurmAccount{{Name: "test", Parent: "root"}}
if err := k.accounts(context.Background(), m); err != nil {
t.Fatal(err)
}
// Neither has another account: they go from the accounting.
want := []string{"sacctmgr -i remove user ayse", "sacctmgr -i remove user ada",
"sacctmgr -i remove account test-arge-deneme", "sacctmgr -i remove account test-arge"}
var got []string
for _, c := range h.ran {
if strings.Contains(c, "remove") || strings.Contains(c, " add ") {
got = append(got, c)
}
}
if !slices.Equal(got, want) {
t.Errorf("ran:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
if slices.ContainsFunc(h.ran, func(c string) bool { return strings.Contains(c, "physics") }) {
t.Error("touched an account llmgw did not make")
}
// One still on another account: that one is made the default first.
h.outputs[assoc] = "root||\nphysics||root\nphysics|ada|\ntest||root\nx||test\nx|ada|\n"
h.files[k.accountsMarker()] = []byte("account test\naccount x\nuser ada x\n")
h.ran = nil
m.Accounts = []*llmgwv1.SlurmAccount{{Name: "test", Parent: "root"}}
if err := k.accounts(context.Background(), m); err != nil {
t.Fatal(err)
}
if i, j := slices.Index(h.ran, "sacctmgr -i modify user ada set defaultaccount=physics"),
slices.Index(h.ran, "sacctmgr -i remove user ada account=x"); i < 0 || j < i {
t.Errorf("ran:\n%s", strings.Join(h.ran, "\n"))
}
// Off: what llmgw made goes.
h.outputs[assoc] = "root||\nphysics||root\ntest||root\n"
h.ran = nil
if err := k.accounts(context.Background(), &llmgwv1.Slurm{}); err != nil {
t.Fatal(err)
}
if !slices.Contains(h.ran, "sacctmgr -i remove account test") || h.files[k.accountsMarker()] != nil {
t.Errorf("not taken away:\n%s", strings.Join(h.ran, "\n"))
}
}
+14
View File
@@ -267,6 +267,20 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
return fail("slurmctld", err)
}
st.SlurmctldRunning = k.active(ctx, "slurmctld")
// The accounts, and the jobs that ended, need the accounting.
if st.SlurmdbdRunning && st.SlurmctldRunning {
if err := k.accounts(ctx, want); err != nil {
k.log.Warn("keeping Slurm's accounts", "err", err)
if acctErr == nil {
acctErr = fmt.Errorf("accounts: %w", err)
}
}
if ended, err := k.ended(ctx); err != nil {
k.log.Warn("reading the jobs that ended", "err", err)
} else {
st.Ended = ended
}
}
}
if want.GetNode() {
if err := k.node(ctx, want, st); err != nil {
+3
View File
@@ -247,6 +247,9 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
fmt.Fprintf(w, "shared files: %s\n", sharedLine(st.Shared))
if st.Users != "" {
fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount)
if st.Accounts > 0 {
fmt.Fprintf(w, "accounts: %d, one for each project (sbatch --account=ORG-TEAM-PROJECT)\n", st.Accounts)
}
}
if line := containersLine(st); line != "" {
fmt.Fprintf(w, "containers: %s\n", line)
+83
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"log/slog"
"maps"
"math"
"net"
"net/http"
"os"
@@ -487,6 +488,15 @@ func newServerCmd() *cobra.Command {
return out, err
})
api.SetSlurmUsers(st)
// Slurm's accounts from the projects, and the jobs
// that ended priced as the cluster's hours are.
slurmSvc.SetAccounts(func(ctx context.Context) ([]slurm.Account, error) {
return slurmAccounts(ctx, st)
})
slurmSvc.SetJobs(func(ctx context.Context, jobs []slurmsvc.JobRecord) error {
au := current.Load().Auth
return recordSlurmJobs(ctx, st, current.Load().Slurm.Cluster(), au.GPUHourCost, au.CoreHourCost, jobs, log)
})
agentSvc.SetSlurm(slurmSvc)
go slurmSvc.Run(ctx)
api.SetSlurm(slurmSvc)
@@ -1447,3 +1457,76 @@ func accountingPrice(p config.Price) accounting.Price {
return accounting.Price{PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, CachedPer1k: p.CachedPer1k,
PerImage: p.PerImage, PerMesh: p.PerMesh}
}
// slurmAccounts are Slurm's accounts from the store's projects, members and
// Slurm users.
func slurmAccounts(ctx context.Context, st *storepkg.DB) ([]slurm.Account, error) {
ps, err := st.SlurmProjects(ctx)
if err != nil {
return nil, err
}
ms, err := st.AllMemberships(ctx)
if err != nil {
return nil, err
}
us, err := st.SlurmUsers(ctx)
if err != nil {
return nil, err
}
logins := make(map[string]string, len(us))
for _, u := range us {
logins[u.UserID] = u.Login
}
tps := make([]slurm.TenancyProject, 0, len(ps))
for _, p := range ps {
tps = append(tps, slurm.TenancyProject{ID: p.ID, Name: p.Name, TeamID: p.TeamID, TeamName: p.TeamName,
OrgID: p.OrgID, OrgName: p.OrgName})
}
tms := make([]slurm.TenancyMember, 0, len(ms))
for _, m := range ms {
tms = append(tms, slurm.TenancyMember{UserID: m.UserID, Scope: string(m.Scope), ScopeID: m.ScopeID})
}
return slurm.Accounts(tps, tms, logins), nil
}
// recordSlurmJobs records the jobs that ended, each with its project (from
// its account) and person (from its login), priced as the hours the
// cluster's models hold are: a GPU job its cards' hours, any other its
// cores'.
func recordSlurmJobs(ctx context.Context, st *storepkg.DB, cluster string, gpuHour, coreHour float64,
jobs []slurmsvc.JobRecord, log *slog.Logger) error {
as, err := slurmAccounts(ctx, st)
if err != nil {
return err
}
project := map[string]string{}
for _, a := range as {
if a.Project != "" {
project[a.Name] = a.Project
}
}
us, err := st.SlurmUsers(ctx)
if err != nil {
return err
}
person := map[string]string{}
for _, u := range us {
person[u.Login] = u.UserID
}
rows := make([]storepkg.SlurmJob, 0, len(jobs))
for _, j := range jobs {
secs := j.Elapsed.Seconds()
cost := float64(j.CPUs) * secs / 3600 * coreHour
if j.GPUs > 0 {
cost = float64(j.GPUs) * secs / 3600 * gpuHour
}
rows = append(rows, storepkg.SlurmJob{Key: cluster + "/" + j.ID, UserID: person[j.Login], ProjectID: project[j.Account],
Login: j.Login, Account: j.Account, Partition: j.Partition, State: j.State, Start: j.Start, End: j.End,
ElapsedSeconds: int64(secs), CPUs: int64(j.CPUs), GPUs: int64(j.GPUs), CostMicros: int64(math.Ceil(cost * 1e6))})
}
n, err := st.RecordSlurmJobs(ctx, rows)
if n > 0 {
log.Info("Slurm jobs that ended recorded, with their cost", "jobs", n)
}
return err
}
+126
View File
@@ -116,6 +116,8 @@ type Status struct {
// machine has pyxis and enroot, "installing" until then, empty when
// slurm.containers is off.
Containers string `json:"containers,omitempty"`
// Accounts is how many of Slurm's accounts are llmgw's projects'.
Accounts int `json:"accounts,omitempty"`
}
// User is a Linux user made from an llmgw account.
@@ -166,6 +168,53 @@ type Service struct {
users []User
usersAt time.Time
usersOK bool
// accountList reads Slurm's accounts from the projects, with the users;
// accountsOK is false until it has been read, and the controller is
// told nothing of accounts until then: none would take every one away.
accountList func(ctx context.Context) ([]slurm.Account, error)
accounts []slurm.Account
accountsOK bool
// jobSink records the jobs that ended; recorded are those it took,
// recording whether a recording is under way.
jobSink func(ctx context.Context, jobs []JobRecord) error
recorded map[string]time.Time
recording bool
}
// JobRecord is a job that ended, for llmgw's accounting.
type JobRecord struct {
ID, Login, Account, Partition, State string
Start, End time.Time
Elapsed time.Duration
CPUs, GPUs int
}
// SetAccounts has Slurm's accounts read with list, beside the users.
func (s *Service) SetAccounts(list func(ctx context.Context) ([]slurm.Account, error)) {
s.mu.Lock()
s.accountList = list
s.mu.Unlock()
}
// SetJobs has the jobs that ended recorded with record; each is given to it
// once it took it without an error.
func (s *Service) SetJobs(record func(ctx context.Context, jobs []JobRecord) error) {
s.mu.Lock()
s.jobSink = record
s.mu.Unlock()
}
// AccountsOf are the accounts a login may run jobs on.
func (s *Service) AccountsOf(login string) []string {
s.mu.Lock()
defer s.mu.Unlock()
var out []string
for _, a := range s.accounts {
if slices.Contains(a.Users, login) {
out = append(out, a.Name)
}
}
return out
}
// UsersEvery is how often the users are read again without being asked.
@@ -199,6 +248,64 @@ func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
return
}
s.users, s.usersOK = us, true
if s.accountList == nil {
return
}
as, err := s.accountList(ctx)
if err != nil {
s.log.Warn("could not read Slurm's accounts from the projects; the controller keeps the last ones", "err", err)
return
}
s.accounts, s.accountsOK = as, true
}
// takeJobs records the jobs a controller says ended that are not recorded
// yet, away from the agent's stream.
func (s *Service) takeJobs(ended []*llmgwv1.SlurmJob) {
s.mu.Lock()
if s.jobSink == nil || s.recording {
s.mu.Unlock()
return
}
if s.recorded == nil {
s.recorded = map[string]time.Time{}
}
for id, at := range s.recorded {
if time.Since(at) > 3*24*time.Hour {
delete(s.recorded, id)
}
}
var jobs []JobRecord
for _, j := range ended {
if _, ok := s.recorded[j.GetId()]; ok || j.GetId() == "" {
continue
}
jobs = append(jobs, JobRecord{ID: j.GetId(), Login: j.GetUser(), Account: j.GetAccount(), Partition: j.GetPartition(),
State: j.GetState(), Start: time.Unix(j.GetStartUnix(), 0), End: time.Unix(j.GetEndUnix(), 0),
Elapsed: time.Duration(j.GetElapsedSeconds()) * time.Second, CPUs: int(j.GetCpus()), GPUs: int(j.GetGpus())})
}
if len(jobs) == 0 {
s.mu.Unlock()
return
}
sink := s.jobSink
s.recording = true
s.mu.Unlock()
go func() {
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
defer cancel()
err := sink(ctx, jobs)
s.mu.Lock()
defer s.mu.Unlock()
s.recording = false
if err != nil {
s.log.Warn("could not record the Slurm jobs that ended; tried again with the next status", "jobs", len(jobs), "err", err)
return
}
for _, j := range jobs {
s.recorded[j.ID] = time.Now()
}
}()
}
// HoldFor is how long a new leader waits for a node it neither has heard
@@ -276,7 +383,11 @@ func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
s.mu.Lock()
prev := s.status[node]
s.status[node] = r
isCtl := s.settings().Controller == node
s.mu.Unlock()
if isCtl && len(st.GetEnded()) > 0 {
s.takeJobs(st.GetEnded())
}
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
}
@@ -517,6 +628,7 @@ func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
return c, nil
}
c.ControllerAddr = s.controllerAddr(set)
c.EnforceAccounts = set.Users == config.UsersLlmgw && s.accountList != nil
var waiting []string
for _, node := range set.Nodes {
sn, ok := s.reported(node)
@@ -582,6 +694,15 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
return nil
}
if isCtl && set.Users == config.UsersLlmgw && s.accountList != nil {
if !s.accountsOK {
return nil
}
m.ManageAccounts = true
for _, a := range s.accounts {
m.Accounts = append(m.Accounts, &llmgwv1.SlurmAccount{Name: a.Name, Parent: a.Parent, Users: slices.Clone(a.Users)})
}
}
return m
}
@@ -675,6 +796,11 @@ func (s *Service) Status() Status {
Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
if set.Users == config.UsersLlmgw {
out.UserCount = len(s.users)
for _, a := range s.accounts {
if a.Project != "" {
out.Accounts++
}
}
}
if !set.Enabled {
return out
+57
View File
@@ -11,6 +11,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
type fakeInv struct {
@@ -323,3 +324,59 @@ func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
t.Fatal("pyxis still loaded with containers off")
}
}
func TestTheControllerKeepsTheProjectsAccountsAndJobsAreRecordedOnce(t *testing.T) {
r := newRig()
r.set.Users = config.UsersLlmgw
r.svc.SetUsers(func(context.Context) ([]User, error) { return []User{{Login: "ayse", UID: 200001}}, nil })
read := 0
r.svc.SetAccounts(func(context.Context) ([]slurm.Account, error) {
read++
return []slurm.Account{{Name: "test", Parent: "root"},
{Name: "test-arge-deneme", Parent: "test", Project: "p1", Users: []string{"ayse"}}}, nil
})
got := make(chan []JobRecord, 4)
r.svc.SetJobs(func(_ context.Context, jobs []JobRecord) error { got <- jobs; return nil })
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
r.tick()
ctl := r.agents.last("node06")
if !ctl.GetManageAccounts() || len(ctl.GetAccounts()) != 2 || ctl.GetAccounts()[1].GetUsers()[0] != "ayse" {
t.Fatalf("controller told %v", ctl.GetAccounts())
}
if r.agents.last("cuda01").GetManageAccounts() {
t.Error("a node was told the accounts")
}
if !strings.Contains(ctl.GetFiles()["slurm.conf"], "AccountingStorageEnforce=associations") {
t.Error("jobs not held to accounts")
}
if r.svc.Status().Accounts != 1 || len(r.svc.AccountsOf("ayse")) != 1 {
t.Errorf("status %d, ayse's %v", r.svc.Status().Accounts, r.svc.AccountsOf("ayse"))
}
ended := []*llmgwv1.SlurmJob{{Id: "41", User: "ayse", Account: "test-arge-deneme", ElapsedSeconds: 60, Gpus: 1}}
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
select {
case jobs := <-got:
if len(jobs) != 1 || jobs[0].Login != "ayse" || jobs[0].GPUs != 1 {
t.Fatalf("jobs %+v", jobs)
}
case <-time.After(5 * time.Second):
t.Fatal("not recorded")
}
// The next status says it again: not recorded twice.
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
r.svc.mu.Lock()
busy := r.svc.recording
r.svc.mu.Unlock()
if !busy {
break
}
time.Sleep(10 * time.Millisecond)
}
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
select {
case jobs := <-got:
t.Fatalf("recorded again: %+v", jobs)
case <-time.After(200 * time.Millisecond):
}
}
+193
View File
@@ -0,0 +1,193 @@
package slurm
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"slices"
"strconv"
"strings"
"time"
)
// Slurm's accounts from llmgw's tree: an account for each organization,
// team and project, nested as they are, and on each project's the users
// that may use it, its members and its team's and organization's. A job
// is charged to the project its --account names (`sbatch --account
// acme-ml-training`), and what it used to that project's spend.
// Account is one Slurm account llmgw keeps.
type Account struct {
Name string
Parent string // "root" at the top
// Project is the llmgw project it is for; empty for an organization's
// or a team's.
Project string
// Users are the logins that may run jobs on it.
Users []string
}
// TenancyProject is a project with where it is.
type TenancyProject struct {
ID, Name, TeamID, TeamName, OrgID, OrgName string
}
// TenancyMember is a membership: a person in an organization, team or
// project ("org", "team", "project").
type TenancyMember struct {
UserID, Scope, ScopeID string
}
// Accounts are the accounts for projects, with logins by user id. Parents
// come before their children, as sacctmgr needs them.
func Accounts(projects []TenancyProject, members []TenancyMember, logins map[string]string) []Account {
// By kind and id: an organization, its team and its project may share
// an id (the first install's are all "default").
names := map[string]string{} // kind/id -> name
taken := map[string]string{} // name -> kind/id
name := func(id string, parts ...string) string {
if n, ok := names[id]; ok {
return n
}
n := AccountName(parts...)
if other, ok := taken[n]; ok && other != id {
// Two places whose names come out the same: told apart by id.
h := sha256.Sum256([]byte(id))
n = AccountName(append(parts, hex.EncodeToString(h[:3]))...)
}
names[id], taken[n] = n, id
return n
}
in := map[string][]string{} // kind/id -> logins
for _, m := range members {
if l, ok := logins[m.UserID]; ok {
in[m.Scope+"/"+m.ScopeID] = append(in[m.Scope+"/"+m.ScopeID], l)
}
}
ps := slices.Clone(projects)
slices.SortFunc(ps, func(a, b TenancyProject) int {
return strings.Compare(a.OrgName+"\x00"+a.TeamName+"\x00"+a.Name, b.OrgName+"\x00"+b.TeamName+"\x00"+b.Name)
})
var out []Account
seen := map[string]bool{}
for _, p := range ps {
org := name("org/"+p.OrgID, p.OrgName)
team := name("team/"+p.TeamID, p.OrgName, p.TeamName)
proj := name("project/"+p.ID, p.OrgName, p.TeamName, p.Name)
if !seen[org] {
out = append(out, Account{Name: org, Parent: "root"})
seen[org] = true
}
if !seen[team] {
out = append(out, Account{Name: team, Parent: org})
seen[team] = true
}
var users []string
for _, id := range []string{"project/" + p.ID, "team/" + p.TeamID, "org/" + p.OrgID} {
users = append(users, in[id]...)
}
slices.Sort(users)
out = append(out, Account{Name: proj, Parent: team, Project: p.ID, Users: slices.Compact(users)})
}
return out
}
// AccountName is a Slurm account name from names: lower case, ASCII,
// joined with "-", at most 64 characters.
func AccountName(parts ...string) string {
var segs []string
for _, p := range parts {
var b strings.Builder
for _, r := range strings.ToLower(p) {
switch {
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_':
b.WriteRune(r)
case strings.ContainsRune("çğıöşüâîû", r):
b.WriteString(map[rune]string{'ç': "c", 'ğ': "g", 'ı': "i", 'ö': "o", 'ş': "s", 'ü': "u",
'â': "a", 'î': "i", 'û': "u"}[r])
default:
b.WriteByte('_')
}
}
if s := strings.Trim(b.String(), "_"); s != "" {
segs = append(segs, s)
}
}
n := strings.Join(segs, "-")
if n == "" {
n = "x"
}
if len(n) > 64 {
h := sha256.Sum256([]byte(n))
n = n[:57] + "-" + hex.EncodeToString(h[:3])
}
return n
}
// Association is one line of `sacctmgr show assoc format=account,user,parentname`.
type Association struct {
Account, User, Parent string
}
// AssocFormat is what ParseAssociations reads.
var AssocFormat = []string{"show", "assoc", "format=account,user,parentname", "-n", "-P"}
// ParseAssociations reads sacctmgr's parsable lines.
func ParseAssociations(out string) []Association {
var as []Association
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
f := strings.Split(line, "|")
if len(f) < 3 || f[0] == "" {
continue
}
as = append(as, Association{Account: f[0], User: f[1], Parent: f[2]})
}
return as
}
// Job is a finished job, as sacct has it.
type Job struct {
ID, User, Account, Partition, State string
Start, End time.Time
Elapsed time.Duration
CPUs, GPUs int
}
// SacctArgs are sacct's for the jobs that ended in the last day.
var SacctArgs = []string{"-a", "-X", "-n", "-P", "--noconvert", "-S", "now-1days", "-E", "now",
"-s", "CD,F,TO,CA,NF,OOM,PR,DL,BF",
"-o", "JobIDRaw,User,Account,Partition,Start,End,ElapsedRaw,AllocTRES,State"}
// ParseJobs reads sacct's lines; a job that has not ended is left out.
func ParseJobs(out string) ([]Job, error) {
var jobs []Job
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
if line == "" {
continue
}
f := strings.Split(line, "|")
if len(f) < 9 {
return nil, fmt.Errorf("sacct said %q, not 9 fields", line)
}
end, err := time.ParseInLocation("2006-01-02T15:04:05", f[5], time.Local)
if err != nil {
continue // Unknown: still running, or never started
}
start, _ := time.ParseInLocation("2006-01-02T15:04:05", f[4], time.Local)
secs, _ := strconv.ParseInt(f[6], 10, 64)
j := Job{ID: f[0], User: f[1], Account: f[2], Partition: f[3], Start: start, End: end,
Elapsed: time.Duration(secs) * time.Second, State: strings.Fields(f[8] + " ")[0]}
for _, t := range strings.Split(f[7], ",") {
k, v, _ := strings.Cut(t, "=")
n, _ := strconv.Atoi(v)
switch k {
case "cpu":
j.CPUs = n
case "gres/gpu":
j.GPUs = n
}
}
jobs = append(jobs, j)
}
return jobs, nil
}
+89
View File
@@ -0,0 +1,89 @@
package slurm
import (
"slices"
"testing"
"time"
)
func TestAccountsFollowTheTree(t *testing.T) {
projects := []TenancyProject{
{ID: "p1", Name: "deneme", TeamID: "t1", TeamName: "arge", OrgID: "o1", OrgName: "Test"},
{ID: "p2", Name: "Görüntü İşleme", TeamID: "t1", TeamName: "arge", OrgID: "o1", OrgName: "Test"},
{ID: "p3", Name: "x", TeamID: "t2", TeamName: "ops", OrgID: "o2", OrgName: "Other"},
}
members := []TenancyMember{
{UserID: "u-ayse", Scope: "project", ScopeID: "p1"},
{UserID: "u-ada", Scope: "team", ScopeID: "t1"},
{UserID: "u-bob", Scope: "org", ScopeID: "o2"},
{UserID: "u-nologin", Scope: "project", ScopeID: "p1"},
}
logins := map[string]string{"u-ayse": "ayse", "u-ada": "ada", "u-bob": "bob"}
got := Accounts(projects, members, logins)
var names []string
by := map[string]Account{}
for _, a := range got {
names = append(names, a.Name)
by[a.Name] = a
}
// Parents first, each before its children.
want := []string{"other", "other-ops", "other-ops-x", "test", "test-arge", "test-arge-goruntu_isleme", "test-arge-deneme"}
if !slices.Equal(names, want) {
t.Fatalf("names %v", names)
}
if a := by["test-arge-deneme"]; a.Parent != "test-arge" || a.Project != "p1" || !slices.Equal(a.Users, []string{"ada", "ayse"}) {
t.Errorf("deneme %+v", a)
}
if a := by["other-ops-x"]; !slices.Equal(a.Users, []string{"bob"}) {
t.Errorf("x %+v", a)
}
if a := by["test"]; a.Parent != "root" || a.Project != "" || len(a.Users) != 0 {
t.Errorf("org %+v", a)
}
if n := AccountName("Test", "arge", "Görüntü İşleme"); n != "test-arge-goruntu_isleme" {
t.Errorf("name %q", n)
}
// The first install's tree: one id for all three.
def := Accounts([]TenancyProject{{ID: "default", Name: "default", TeamID: "default", TeamName: "default",
OrgID: "default", OrgName: "default"}}, []TenancyMember{{UserID: "u", Scope: "project", ScopeID: "default"}},
map[string]string{"u": "admin2"})
if len(def) != 3 || def[2].Name != "default-default-default" || def[2].Parent != "default-default" ||
def[1].Parent != "default" || !slices.Equal(def[2].Users, []string{"admin2"}) {
t.Errorf("default tree %+v", def)
}
// Two that come out the same are told apart.
same := Accounts([]TenancyProject{
{ID: "a", Name: "ml", TeamID: "t", TeamName: "x", OrgID: "o", OrgName: "A"},
{ID: "b", Name: "ML", TeamID: "t", TeamName: "x", OrgID: "o", OrgName: "A"},
}, nil, nil)
if same[2].Name == same[3].Name {
t.Errorf("same name twice: %+v", same)
}
}
func TestParseJobs(t *testing.T) {
out := "41|ayse|test-arge-deneme|gpu|2026-11-15T10:00:00|2026-11-15T10:30:00|1800|billing=4,cpu=4,gres/gpu=1,mem=16G,node=1|COMPLETED\n" +
"42|ada|test-arge-deneme|gpu|2026-11-15T10:00:00|Unknown|0|cpu=2|RUNNING\n" +
"43|root|root|gpu|2026-11-15T09:00:00|2026-11-15T09:00:05|5|cpu=1,mem=1G|CANCELLED by 0\n"
jobs, err := ParseJobs(out)
if err != nil {
t.Fatal(err)
}
if len(jobs) != 2 {
t.Fatalf("jobs %+v", jobs)
}
j := jobs[0]
if j.ID != "41" || j.User != "ayse" || j.GPUs != 1 || j.CPUs != 4 || j.Elapsed != 30*time.Minute || j.State != "COMPLETED" {
t.Errorf("job %+v", j)
}
if jobs[1].State != "CANCELLED" {
t.Errorf("state %q", jobs[1].State)
}
if _, err := ParseJobs("short|line\n"); err == nil {
t.Error("a short line taken")
}
as := ParseAssociations("root||\ntest|ayse|test-arge\n")
if len(as) != 2 || as[1] != (Association{"test", "ayse", "test-arge"}) {
t.Errorf("assoc %+v", as)
}
}
+7
View File
@@ -131,6 +131,10 @@ type Cluster struct {
Controller string
ControllerAddr string
Partition string
// EnforceAccounts refuses a job whose user has no account in Slurm's
// accounting: with llmgw's users, each is on the accounts of the
// projects they are in, and a job is charged to one of them.
EnforceAccounts bool
// Nodes without hardware (not reported yet) are left out of the files
// until they are.
Nodes []Node
@@ -199,6 +203,9 @@ func (c Cluster) SlurmConf() string {
// The jobs' records, in slurmdbd beside slurmctld (DbdConf).
fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n",
c.ControllerAddr, DbdPort)
if c.EnforceAccounts {
b.WriteString("# A job runs on an account of a project its user is in.\nAccountingStorageEnforce=associations\n")
}
b.WriteString(`SlurmctldParameters=enable_configless
SlurmctldPort=6817
SlurmdPort=6818
+1
View File
@@ -45,6 +45,7 @@ var copyOrder = []string{
"usage_1h",
"usage_1d",
"held_1h",
"slurm_jobs",
"models",
// No foreign key to models, deliberately: a measurement outlives the
// catalog entry it was taken for, and losing it when somebody removes a
@@ -0,0 +1,20 @@
-- Ported from the SQLite migration of the same number.
-- The Slurm jobs that ended, each recorded once (job_key is the cluster's
-- name and the job's id), with what it used and cost; a usage row (model
-- "slurm") carries the cost into the usage pages with the requests'.
CREATE TABLE slurm_jobs (
job_key TEXT PRIMARY KEY,
user_id TEXT NOT NULL DEFAULT '',
project_id TEXT NOT NULL DEFAULT '',
login TEXT NOT NULL DEFAULT '',
account TEXT NOT NULL DEFAULT '',
partition TEXT NOT NULL DEFAULT '',
state TEXT NOT NULL DEFAULT '',
started_at BIGINT NOT NULL DEFAULT 0,
ended_at BIGINT NOT NULL DEFAULT 0,
elapsed_seconds BIGINT NOT NULL DEFAULT 0,
cpus BIGINT NOT NULL DEFAULT 0,
gpus BIGINT NOT NULL DEFAULT 0,
cost_micros BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX slurm_jobs_by_project ON slurm_jobs(project_id, ended_at);
@@ -0,0 +1,19 @@
-- The Slurm jobs that ended, each recorded once (job_key is the cluster's
-- name and the job's id), with what it used and cost; a usage row (model
-- "slurm") carries the cost into the usage pages with the requests'.
CREATE TABLE slurm_jobs (
job_key TEXT PRIMARY KEY,
user_id TEXT NOT NULL DEFAULT '',
project_id TEXT NOT NULL DEFAULT '',
login TEXT NOT NULL DEFAULT '',
account TEXT NOT NULL DEFAULT '',
partition TEXT NOT NULL DEFAULT '',
state TEXT NOT NULL DEFAULT '',
started_at BIGINT NOT NULL DEFAULT 0,
ended_at BIGINT NOT NULL DEFAULT 0,
elapsed_seconds BIGINT NOT NULL DEFAULT 0,
cpus BIGINT NOT NULL DEFAULT 0,
gpus BIGINT NOT NULL DEFAULT 0,
cost_micros BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX slurm_jobs_by_project ON slurm_jobs(project_id, ended_at);
+65
View File
@@ -0,0 +1,65 @@
package store
import (
"context"
"database/sql"
"time"
)
// SlurmProject is a project with where it is, for Slurm's accounts.
type SlurmProject struct {
ID, Name, TeamID, TeamName, OrgID, OrgName string
}
// SlurmProjects are every project, with its team and organization.
func (s *DB) SlurmProjects(ctx context.Context) ([]SlurmProject, error) {
return queryAll(ctx, s.db, `
SELECT p.id, p.name, t.id, t.name, o.id, o.name
FROM projects p JOIN teams t ON t.id = p.team_id JOIN organizations o ON o.id = t.org_id
ORDER BY o.name, t.name, p.name`, nil, func(rows *sql.Rows) (SlurmProject, error) {
var p SlurmProject
return p, rows.Scan(&p.ID, &p.Name, &p.TeamID, &p.TeamName, &p.OrgID, &p.OrgName)
})
}
// SlurmJob is a job that ended, as recorded: once, by its cluster's id.
type SlurmJob struct {
// Key is the cluster's name and the job's id: unique among jobs.
Key string
UserID, ProjectID string
Login, Account, Partition string
State string
Start, End time.Time
ElapsedSeconds, CPUs, GPUs int64
CostMicros int64
}
// RecordSlurmJobs records jobs not recorded before, each with a usage row
// (model "slurm") so its cost is in the usage pages and the exports with
// the requests'. Answers how many were new.
func (s *DB) RecordSlurmJobs(ctx context.Context, jobs []SlurmJob) (int, error) {
n := 0
err := s.inTx(ctx, func(tx conn) error {
var usage []Usage
for _, j := range jobs {
res, err := tx.ExecContext(ctx,
`INSERT INTO slurm_jobs (job_key, user_id, project_id, login, account, partition, state,
started_at, ended_at, elapsed_seconds, cpus, gpus, cost_micros)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(job_key) DO NOTHING`,
j.Key, j.UserID, j.ProjectID, j.Login, j.Account, j.Partition, j.State,
millis(j.Start), millis(j.End), j.ElapsedSeconds, j.CPUs, j.GPUs, j.CostMicros)
if err != nil {
return err
}
if k, _ := res.RowsAffected(); k == 0 {
continue
}
n++
usage = append(usage, Usage{At: j.End, ProjectID: j.ProjectID, UserID: j.UserID, Model: "slurm",
ServedModel: "slurm/" + j.Partition, CostMicros: j.CostMicros, Status: 200,
LatencyMs: j.ElapsedSeconds * 1000, RequestID: "slurm-" + j.Key})
}
return insertUsage(ctx, tx, usage)
})
return n, err
}
+41
View File
@@ -0,0 +1,41 @@
package store
import (
"testing"
"time"
)
func TestASlurmJobIsRecordedOnce(t *testing.T) {
s := capacityStore(t)
ctx := t.Context()
end := time.Now().Add(-time.Hour).Truncate(time.Second)
j := SlurmJob{Key: "llmgw/41", ProjectID: "p1", UserID: "u1", Login: "ayse", Account: "test-arge-deneme",
Partition: "gpu", State: "COMPLETED", Start: end.Add(-30 * time.Minute), End: end, ElapsedSeconds: 1800,
CPUs: 4, GPUs: 1, CostMicros: 675000}
for range 2 {
n, err := s.RecordSlurmJobs(ctx, []SlurmJob{j})
if err != nil {
t.Fatal(err)
}
_ = n
}
byModel, err := s.UsageByModel(ctx, end.Add(-24*time.Hour))
if err != nil {
t.Fatal(err)
}
var found bool
for _, m := range byModel {
if m.Model == "slurm/gpu" {
found = true
if m.Requests != 1 || m.CostMicros != 675000 {
t.Errorf("recorded twice or wrong: %+v", m)
}
}
}
if !found {
t.Fatalf("no usage row: %+v", byModel)
}
if n, _ := s.RecordSlurmJobs(ctx, []SlurmJob{j}); n != 0 {
t.Errorf("recorded again: %d", n)
}
}
+30 -24
View File
@@ -447,32 +447,38 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
if len(records) == 0 {
return nil
}
return s.inTx(ctx, func(tx conn) error {
stmt, err := tx.PrepareContext(ctx,
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
request_id, replica, queue_ms, ttft_ms,
guard_action, guard_policy, guard_category, guard_rule, guard_ms, end_user, tags, cache_hit, images, meshes)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return s.inTx(ctx, func(tx conn) error { return insertUsage(ctx, tx, records) })
}
// insertUsage writes records, and their sums, in tx.
func insertUsage(ctx context.Context, tx conn, records []Usage) error {
if len(records) == 0 {
return nil
}
stmt, err := tx.PrepareContext(ctx,
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
request_id, replica, queue_ms, ttft_ms,
guard_action, guard_policy, guard_category, guard_rule, guard_ms, end_user, tags, cache_hit, images, meshes)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
defer stmt.Close()
for _, r := range records {
hit := 0
if r.CacheHit {
hit = 1
}
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardRule, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit, r.Images, r.Meshes); err != nil {
return err
}
defer stmt.Close()
for _, r := range records {
hit := 0
if r.CacheHit {
hit = 1
}
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardRule, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit, r.Images, r.Meshes); err != nil {
return err
}
}
return writeRollup(ctx, tx, records)
})
}
return writeRollup(ctx, tx, records)
}
// SpendSince totals cost per key since a moment.