Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s

With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-15 11:54:01 +01:00
1 parent 8547528fb7
commit fcda4d3f85
22 files changed
+1390 -76

No files matched your search

+83
View File
@@ -10,6 +10,7 @@ import (
"fmt"
"log/slog"
"maps"
"math"
"net"
"net/http"
"os"
@@ -487,6 +488,15 @@ func newServerCmd() *cobra.Command {
return out, err
})
api.SetSlurmUsers(st)
// Slurm's accounts from the projects, and the jobs
// that ended priced as the cluster's hours are.
slurmSvc.SetAccounts(func(ctx context.Context) ([]slurm.Account, error) {
return slurmAccounts(ctx, st)
})
slurmSvc.SetJobs(func(ctx context.Context, jobs []slurmsvc.JobRecord) error {
au := current.Load().Auth
return recordSlurmJobs(ctx, st, current.Load().Slurm.Cluster(), au.GPUHourCost, au.CoreHourCost, jobs, log)
})
agentSvc.SetSlurm(slurmSvc)
go slurmSvc.Run(ctx)
api.SetSlurm(slurmSvc)
@@ -1447,3 +1457,76 @@ func accountingPrice(p config.Price) accounting.Price {
return accounting.Price{PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, CachedPer1k: p.CachedPer1k,
PerImage: p.PerImage, PerMesh: p.PerMesh}
}
// slurmAccounts are Slurm's accounts from the store's projects, members and
// Slurm users.
func slurmAccounts(ctx context.Context, st *storepkg.DB) ([]slurm.Account, error) {
ps, err := st.SlurmProjects(ctx)
if err != nil {
return nil, err
}
ms, err := st.AllMemberships(ctx)
if err != nil {
return nil, err
}
us, err := st.SlurmUsers(ctx)
if err != nil {
return nil, err
}
logins := make(map[string]string, len(us))
for _, u := range us {
logins[u.UserID] = u.Login
}
tps := make([]slurm.TenancyProject, 0, len(ps))
for _, p := range ps {
tps = append(tps, slurm.TenancyProject{ID: p.ID, Name: p.Name, TeamID: p.TeamID, TeamName: p.TeamName,
OrgID: p.OrgID, OrgName: p.OrgName})
}
tms := make([]slurm.TenancyMember, 0, len(ms))
for _, m := range ms {
tms = append(tms, slurm.TenancyMember{UserID: m.UserID, Scope: string(m.Scope), ScopeID: m.ScopeID})
}
return slurm.Accounts(tps, tms, logins), nil
}
// recordSlurmJobs records the jobs that ended, each with its project (from
// its account) and person (from its login), priced as the hours the
// cluster's models hold are: a GPU job its cards' hours, any other its
// cores'.
func recordSlurmJobs(ctx context.Context, st *storepkg.DB, cluster string, gpuHour, coreHour float64,
jobs []slurmsvc.JobRecord, log *slog.Logger) error {
as, err := slurmAccounts(ctx, st)
if err != nil {
return err
}
project := map[string]string{}
for _, a := range as {
if a.Project != "" {
project[a.Name] = a.Project
}
}
us, err := st.SlurmUsers(ctx)
if err != nil {
return err
}
person := map[string]string{}
for _, u := range us {
person[u.Login] = u.UserID
}
rows := make([]storepkg.SlurmJob, 0, len(jobs))
for _, j := range jobs {
secs := j.Elapsed.Seconds()
cost := float64(j.CPUs) * secs / 3600 * coreHour
if j.GPUs > 0 {
cost = float64(j.GPUs) * secs / 3600 * gpuHour
}
rows = append(rows, storepkg.SlurmJob{Key: cluster + "/" + j.ID, UserID: person[j.Login], ProjectID: project[j.Account],
Login: j.Login, Account: j.Account, Partition: j.Partition, State: j.State, Start: j.Start, End: j.End,
ElapsedSeconds: int64(secs), CPUs: int64(j.CPUs), GPUs: int64(j.GPUs), CostMicros: int64(math.Ceil(cost * 1e6))})
}
n, err := st.RecordSlurmJobs(ctx, rows)
if n > 0 {
log.Info("Slurm jobs that ended recorded, with their cost", "jobs", n)
}
return err
}