Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s

With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-15 11:54:01 +01:00
1 parent 8547528fb7
commit fcda4d3f85
22 files changed
+1390 -76

No files matched your search

+1
View File
@@ -45,6 +45,7 @@ var copyOrder = []string{
"usage_1h",
"usage_1d",
"held_1h",
"slurm_jobs",
"models",
// No foreign key to models, deliberately: a measurement outlives the
// catalog entry it was taken for, and losing it when somebody removes a
@@ -0,0 +1,20 @@
-- Ported from the SQLite migration of the same number.
-- The Slurm jobs that ended, each recorded once (job_key is the cluster's
-- name and the job's id), with what it used and cost; a usage row (model
-- "slurm") carries the cost into the usage pages with the requests'.
CREATE TABLE slurm_jobs (
job_key TEXT PRIMARY KEY,
user_id TEXT NOT NULL DEFAULT '',
project_id TEXT NOT NULL DEFAULT '',
login TEXT NOT NULL DEFAULT '',
account TEXT NOT NULL DEFAULT '',
partition TEXT NOT NULL DEFAULT '',
state TEXT NOT NULL DEFAULT '',
started_at BIGINT NOT NULL DEFAULT 0,
ended_at BIGINT NOT NULL DEFAULT 0,
elapsed_seconds BIGINT NOT NULL DEFAULT 0,
cpus BIGINT NOT NULL DEFAULT 0,
gpus BIGINT NOT NULL DEFAULT 0,
cost_micros BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX slurm_jobs_by_project ON slurm_jobs(project_id, ended_at);
@@ -0,0 +1,19 @@
-- The Slurm jobs that ended, each recorded once (job_key is the cluster's
-- name and the job's id), with what it used and cost; a usage row (model
-- "slurm") carries the cost into the usage pages with the requests'.
CREATE TABLE slurm_jobs (
job_key TEXT PRIMARY KEY,
user_id TEXT NOT NULL DEFAULT '',
project_id TEXT NOT NULL DEFAULT '',
login TEXT NOT NULL DEFAULT '',
account TEXT NOT NULL DEFAULT '',
partition TEXT NOT NULL DEFAULT '',
state TEXT NOT NULL DEFAULT '',
started_at BIGINT NOT NULL DEFAULT 0,
ended_at BIGINT NOT NULL DEFAULT 0,
elapsed_seconds BIGINT NOT NULL DEFAULT 0,
cpus BIGINT NOT NULL DEFAULT 0,
gpus BIGINT NOT NULL DEFAULT 0,
cost_micros BIGINT NOT NULL DEFAULT 0
);
CREATE INDEX slurm_jobs_by_project ON slurm_jobs(project_id, ended_at);
+65
View File
@@ -0,0 +1,65 @@
package store
import (
"context"
"database/sql"
"time"
)
// SlurmProject is a project with where it is, for Slurm's accounts.
type SlurmProject struct {
ID, Name, TeamID, TeamName, OrgID, OrgName string
}
// SlurmProjects are every project, with its team and organization.
func (s *DB) SlurmProjects(ctx context.Context) ([]SlurmProject, error) {
return queryAll(ctx, s.db, `
SELECT p.id, p.name, t.id, t.name, o.id, o.name
FROM projects p JOIN teams t ON t.id = p.team_id JOIN organizations o ON o.id = t.org_id
ORDER BY o.name, t.name, p.name`, nil, func(rows *sql.Rows) (SlurmProject, error) {
var p SlurmProject
return p, rows.Scan(&p.ID, &p.Name, &p.TeamID, &p.TeamName, &p.OrgID, &p.OrgName)
})
}
// SlurmJob is a job that ended, as recorded: once, by its cluster's id.
type SlurmJob struct {
// Key is the cluster's name and the job's id: unique among jobs.
Key string
UserID, ProjectID string
Login, Account, Partition string
State string
Start, End time.Time
ElapsedSeconds, CPUs, GPUs int64
CostMicros int64
}
// RecordSlurmJobs records jobs not recorded before, each with a usage row
// (model "slurm") so its cost is in the usage pages and the exports with
// the requests'. Answers how many were new.
func (s *DB) RecordSlurmJobs(ctx context.Context, jobs []SlurmJob) (int, error) {
n := 0
err := s.inTx(ctx, func(tx conn) error {
var usage []Usage
for _, j := range jobs {
res, err := tx.ExecContext(ctx,
`INSERT INTO slurm_jobs (job_key, user_id, project_id, login, account, partition, state,
started_at, ended_at, elapsed_seconds, cpus, gpus, cost_micros)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(job_key) DO NOTHING`,
j.Key, j.UserID, j.ProjectID, j.Login, j.Account, j.Partition, j.State,
millis(j.Start), millis(j.End), j.ElapsedSeconds, j.CPUs, j.GPUs, j.CostMicros)
if err != nil {
return err
}
if k, _ := res.RowsAffected(); k == 0 {
continue
}
n++
usage = append(usage, Usage{At: j.End, ProjectID: j.ProjectID, UserID: j.UserID, Model: "slurm",
ServedModel: "slurm/" + j.Partition, CostMicros: j.CostMicros, Status: 200,
LatencyMs: j.ElapsedSeconds * 1000, RequestID: "slurm-" + j.Key})
}
return insertUsage(ctx, tx, usage)
})
return n, err
}
+41
View File
@@ -0,0 +1,41 @@
package store
import (
"testing"
"time"
)
func TestASlurmJobIsRecordedOnce(t *testing.T) {
s := capacityStore(t)
ctx := t.Context()
end := time.Now().Add(-time.Hour).Truncate(time.Second)
j := SlurmJob{Key: "llmgw/41", ProjectID: "p1", UserID: "u1", Login: "ayse", Account: "test-arge-deneme",
Partition: "gpu", State: "COMPLETED", Start: end.Add(-30 * time.Minute), End: end, ElapsedSeconds: 1800,
CPUs: 4, GPUs: 1, CostMicros: 675000}
for range 2 {
n, err := s.RecordSlurmJobs(ctx, []SlurmJob{j})
if err != nil {
t.Fatal(err)
}
_ = n
}
byModel, err := s.UsageByModel(ctx, end.Add(-24*time.Hour))
if err != nil {
t.Fatal(err)
}
var found bool
for _, m := range byModel {
if m.Model == "slurm/gpu" {
found = true
if m.Requests != 1 || m.CostMicros != 675000 {
t.Errorf("recorded twice or wrong: %+v", m)
}
}
}
if !found {
t.Fatalf("no usage row: %+v", byModel)
}
if n, _ := s.RecordSlurmJobs(ctx, []SlurmJob{j}); n != 0 {
t.Errorf("recorded again: %d", n)
}
}
+30 -24
View File
@@ -447,32 +447,38 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
if len(records) == 0 {
return nil
}
return s.inTx(ctx, func(tx conn) error {
stmt, err := tx.PrepareContext(ctx,
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
request_id, replica, queue_ms, ttft_ms,
guard_action, guard_policy, guard_category, guard_rule, guard_ms, end_user, tags, cache_hit, images, meshes)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return s.inTx(ctx, func(tx conn) error { return insertUsage(ctx, tx, records) })
}
// insertUsage writes records, and their sums, in tx.
func insertUsage(ctx context.Context, tx conn, records []Usage) error {
if len(records) == 0 {
return nil
}
stmt, err := tx.PrepareContext(ctx,
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
request_id, replica, queue_ms, ttft_ms,
guard_action, guard_policy, guard_category, guard_rule, guard_ms, end_user, tags, cache_hit, images, meshes)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
if err != nil {
return err
}
defer stmt.Close()
for _, r := range records {
hit := 0
if r.CacheHit {
hit = 1
}
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardRule, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit, r.Images, r.Meshes); err != nil {
return err
}
defer stmt.Close()
for _, r := range records {
hit := 0
if r.CacheHit {
hit = 1
}
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardRule, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit, r.Images, r.Meshes); err != nil {
return err
}
}
return writeRollup(ctx, tx, records)
})
}
return writeRollup(ctx, tx, records)
}
// SpendSince totals cost per key since a moment.