Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s
CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8547528fb7
commit
fcda4d3f85
22 files changed
+1390
-76
No files matched your search
@@ -247,6 +247,9 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Fprintf(w, "shared files: %s\n", sharedLine(st.Shared))
|
||||
if st.Users != "" {
|
||||
fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount)
|
||||
if st.Accounts > 0 {
|
||||
fmt.Fprintf(w, "accounts: %d, one for each project (sbatch --account=ORG-TEAM-PROJECT)\n", st.Accounts)
|
||||
}
|
||||
}
|
||||
if line := containersLine(st); line != "" {
|
||||
fmt.Fprintf(w, "containers: %s\n", line)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -487,6 +488,15 @@ func newServerCmd() *cobra.Command {
|
||||
return out, err
|
||||
})
|
||||
api.SetSlurmUsers(st)
|
||||
// Slurm's accounts from the projects, and the jobs
|
||||
// that ended priced as the cluster's hours are.
|
||||
slurmSvc.SetAccounts(func(ctx context.Context) ([]slurm.Account, error) {
|
||||
return slurmAccounts(ctx, st)
|
||||
})
|
||||
slurmSvc.SetJobs(func(ctx context.Context, jobs []slurmsvc.JobRecord) error {
|
||||
au := current.Load().Auth
|
||||
return recordSlurmJobs(ctx, st, current.Load().Slurm.Cluster(), au.GPUHourCost, au.CoreHourCost, jobs, log)
|
||||
})
|
||||
agentSvc.SetSlurm(slurmSvc)
|
||||
go slurmSvc.Run(ctx)
|
||||
api.SetSlurm(slurmSvc)
|
||||
@@ -1447,3 +1457,76 @@ func accountingPrice(p config.Price) accounting.Price {
|
||||
return accounting.Price{PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, CachedPer1k: p.CachedPer1k,
|
||||
PerImage: p.PerImage, PerMesh: p.PerMesh}
|
||||
}
|
||||
|
||||
// slurmAccounts are Slurm's accounts from the store's projects, members and
|
||||
// Slurm users.
|
||||
func slurmAccounts(ctx context.Context, st *storepkg.DB) ([]slurm.Account, error) {
|
||||
ps, err := st.SlurmProjects(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ms, err := st.AllMemberships(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
us, err := st.SlurmUsers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logins := make(map[string]string, len(us))
|
||||
for _, u := range us {
|
||||
logins[u.UserID] = u.Login
|
||||
}
|
||||
tps := make([]slurm.TenancyProject, 0, len(ps))
|
||||
for _, p := range ps {
|
||||
tps = append(tps, slurm.TenancyProject{ID: p.ID, Name: p.Name, TeamID: p.TeamID, TeamName: p.TeamName,
|
||||
OrgID: p.OrgID, OrgName: p.OrgName})
|
||||
}
|
||||
tms := make([]slurm.TenancyMember, 0, len(ms))
|
||||
for _, m := range ms {
|
||||
tms = append(tms, slurm.TenancyMember{UserID: m.UserID, Scope: string(m.Scope), ScopeID: m.ScopeID})
|
||||
}
|
||||
return slurm.Accounts(tps, tms, logins), nil
|
||||
}
|
||||
|
||||
// recordSlurmJobs records the jobs that ended, each with its project (from
|
||||
// its account) and person (from its login), priced as the hours the
|
||||
// cluster's models hold are: a GPU job its cards' hours, any other its
|
||||
// cores'.
|
||||
func recordSlurmJobs(ctx context.Context, st *storepkg.DB, cluster string, gpuHour, coreHour float64,
|
||||
jobs []slurmsvc.JobRecord, log *slog.Logger) error {
|
||||
as, err := slurmAccounts(ctx, st)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
project := map[string]string{}
|
||||
for _, a := range as {
|
||||
if a.Project != "" {
|
||||
project[a.Name] = a.Project
|
||||
}
|
||||
}
|
||||
us, err := st.SlurmUsers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
person := map[string]string{}
|
||||
for _, u := range us {
|
||||
person[u.Login] = u.UserID
|
||||
}
|
||||
rows := make([]storepkg.SlurmJob, 0, len(jobs))
|
||||
for _, j := range jobs {
|
||||
secs := j.Elapsed.Seconds()
|
||||
cost := float64(j.CPUs) * secs / 3600 * coreHour
|
||||
if j.GPUs > 0 {
|
||||
cost = float64(j.GPUs) * secs / 3600 * gpuHour
|
||||
}
|
||||
rows = append(rows, storepkg.SlurmJob{Key: cluster + "/" + j.ID, UserID: person[j.Login], ProjectID: project[j.Account],
|
||||
Login: j.Login, Account: j.Account, Partition: j.Partition, State: j.State, Start: j.Start, End: j.End,
|
||||
ElapsedSeconds: int64(secs), CPUs: int64(j.CPUs), GPUs: int64(j.GPUs), CostMicros: int64(math.Ceil(cost * 1e6))})
|
||||
}
|
||||
n, err := st.RecordSlurmJobs(ctx, rows)
|
||||
if n > 0 {
|
||||
log.Info("Slurm jobs that ended recorded, with their cost", "jobs", n)
|
||||
}
|
||||
return err
|
||||
}
|
||||
Reference in new issue
Block a user