Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s
CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8547528fb7
commit
fcda4d3f85
22 files changed
+1390
-76
No files matched your search
@@ -45,6 +45,7 @@ var copyOrder = []string{
|
||||
"usage_1h",
|
||||
"usage_1d",
|
||||
"held_1h",
|
||||
"slurm_jobs",
|
||||
"models",
|
||||
// No foreign key to models, deliberately: a measurement outlives the
|
||||
// catalog entry it was taken for, and losing it when somebody removes a
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- The Slurm jobs that ended, each recorded once (job_key is the cluster's
|
||||
-- name and the job's id), with what it used and cost; a usage row (model
|
||||
-- "slurm") carries the cost into the usage pages with the requests'.
|
||||
CREATE TABLE slurm_jobs (
|
||||
job_key TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL DEFAULT '',
|
||||
project_id TEXT NOT NULL DEFAULT '',
|
||||
login TEXT NOT NULL DEFAULT '',
|
||||
account TEXT NOT NULL DEFAULT '',
|
||||
partition TEXT NOT NULL DEFAULT '',
|
||||
state TEXT NOT NULL DEFAULT '',
|
||||
started_at BIGINT NOT NULL DEFAULT 0,
|
||||
ended_at BIGINT NOT NULL DEFAULT 0,
|
||||
elapsed_seconds BIGINT NOT NULL DEFAULT 0,
|
||||
cpus BIGINT NOT NULL DEFAULT 0,
|
||||
gpus BIGINT NOT NULL DEFAULT 0,
|
||||
cost_micros BIGINT NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX slurm_jobs_by_project ON slurm_jobs(project_id, ended_at);
|
||||
@@ -0,0 +1,19 @@
|
||||
-- The Slurm jobs that ended, each recorded once (job_key is the cluster's
|
||||
-- name and the job's id), with what it used and cost; a usage row (model
|
||||
-- "slurm") carries the cost into the usage pages with the requests'.
|
||||
CREATE TABLE slurm_jobs (
|
||||
job_key TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL DEFAULT '',
|
||||
project_id TEXT NOT NULL DEFAULT '',
|
||||
login TEXT NOT NULL DEFAULT '',
|
||||
account TEXT NOT NULL DEFAULT '',
|
||||
partition TEXT NOT NULL DEFAULT '',
|
||||
state TEXT NOT NULL DEFAULT '',
|
||||
started_at BIGINT NOT NULL DEFAULT 0,
|
||||
ended_at BIGINT NOT NULL DEFAULT 0,
|
||||
elapsed_seconds BIGINT NOT NULL DEFAULT 0,
|
||||
cpus BIGINT NOT NULL DEFAULT 0,
|
||||
gpus BIGINT NOT NULL DEFAULT 0,
|
||||
cost_micros BIGINT NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX slurm_jobs_by_project ON slurm_jobs(project_id, ended_at);
|
||||
@@ -0,0 +1,65 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SlurmProject is a project with where it is, for Slurm's accounts.
|
||||
type SlurmProject struct {
|
||||
ID, Name, TeamID, TeamName, OrgID, OrgName string
|
||||
}
|
||||
|
||||
// SlurmProjects are every project, with its team and organization.
|
||||
func (s *DB) SlurmProjects(ctx context.Context) ([]SlurmProject, error) {
|
||||
return queryAll(ctx, s.db, `
|
||||
SELECT p.id, p.name, t.id, t.name, o.id, o.name
|
||||
FROM projects p JOIN teams t ON t.id = p.team_id JOIN organizations o ON o.id = t.org_id
|
||||
ORDER BY o.name, t.name, p.name`, nil, func(rows *sql.Rows) (SlurmProject, error) {
|
||||
var p SlurmProject
|
||||
return p, rows.Scan(&p.ID, &p.Name, &p.TeamID, &p.TeamName, &p.OrgID, &p.OrgName)
|
||||
})
|
||||
}
|
||||
|
||||
// SlurmJob is a job that ended, as recorded: once, by its cluster's id.
|
||||
type SlurmJob struct {
|
||||
// Key is the cluster's name and the job's id: unique among jobs.
|
||||
Key string
|
||||
UserID, ProjectID string
|
||||
Login, Account, Partition string
|
||||
State string
|
||||
Start, End time.Time
|
||||
ElapsedSeconds, CPUs, GPUs int64
|
||||
CostMicros int64
|
||||
}
|
||||
|
||||
// RecordSlurmJobs records jobs not recorded before, each with a usage row
|
||||
// (model "slurm") so its cost is in the usage pages and the exports with
|
||||
// the requests'. Answers how many were new.
|
||||
func (s *DB) RecordSlurmJobs(ctx context.Context, jobs []SlurmJob) (int, error) {
|
||||
n := 0
|
||||
err := s.inTx(ctx, func(tx conn) error {
|
||||
var usage []Usage
|
||||
for _, j := range jobs {
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`INSERT INTO slurm_jobs (job_key, user_id, project_id, login, account, partition, state,
|
||||
started_at, ended_at, elapsed_seconds, cpus, gpus, cost_micros)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(job_key) DO NOTHING`,
|
||||
j.Key, j.UserID, j.ProjectID, j.Login, j.Account, j.Partition, j.State,
|
||||
millis(j.Start), millis(j.End), j.ElapsedSeconds, j.CPUs, j.GPUs, j.CostMicros)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if k, _ := res.RowsAffected(); k == 0 {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
usage = append(usage, Usage{At: j.End, ProjectID: j.ProjectID, UserID: j.UserID, Model: "slurm",
|
||||
ServedModel: "slurm/" + j.Partition, CostMicros: j.CostMicros, Status: 200,
|
||||
LatencyMs: j.ElapsedSeconds * 1000, RequestID: "slurm-" + j.Key})
|
||||
}
|
||||
return insertUsage(ctx, tx, usage)
|
||||
})
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestASlurmJobIsRecordedOnce(t *testing.T) {
|
||||
s := capacityStore(t)
|
||||
ctx := t.Context()
|
||||
end := time.Now().Add(-time.Hour).Truncate(time.Second)
|
||||
j := SlurmJob{Key: "llmgw/41", ProjectID: "p1", UserID: "u1", Login: "ayse", Account: "test-arge-deneme",
|
||||
Partition: "gpu", State: "COMPLETED", Start: end.Add(-30 * time.Minute), End: end, ElapsedSeconds: 1800,
|
||||
CPUs: 4, GPUs: 1, CostMicros: 675000}
|
||||
for range 2 {
|
||||
n, err := s.RecordSlurmJobs(ctx, []SlurmJob{j})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = n
|
||||
}
|
||||
byModel, err := s.UsageByModel(ctx, end.Add(-24*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, m := range byModel {
|
||||
if m.Model == "slurm/gpu" {
|
||||
found = true
|
||||
if m.Requests != 1 || m.CostMicros != 675000 {
|
||||
t.Errorf("recorded twice or wrong: %+v", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("no usage row: %+v", byModel)
|
||||
}
|
||||
if n, _ := s.RecordSlurmJobs(ctx, []SlurmJob{j}); n != 0 {
|
||||
t.Errorf("recorded again: %d", n)
|
||||
}
|
||||
}
|
||||
+30
-24
@@ -447,32 +447,38 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
return s.inTx(ctx, func(tx conn) error {
|
||||
stmt, err := tx.PrepareContext(ctx,
|
||||
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
|
||||
request_id, replica, queue_ms, ttft_ms,
|
||||
guard_action, guard_policy, guard_category, guard_rule, guard_ms, end_user, tags, cache_hit, images, meshes)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return s.inTx(ctx, func(tx conn) error { return insertUsage(ctx, tx, records) })
|
||||
}
|
||||
|
||||
// insertUsage writes records, and their sums, in tx.
|
||||
func insertUsage(ctx context.Context, tx conn, records []Usage) error {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
stmt, err := tx.PrepareContext(ctx,
|
||||
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
|
||||
request_id, replica, queue_ms, ttft_ms,
|
||||
guard_action, guard_policy, guard_category, guard_rule, guard_ms, end_user, tags, cache_hit, images, meshes)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
for _, r := range records {
|
||||
hit := 0
|
||||
if r.CacheHit {
|
||||
hit = 1
|
||||
}
|
||||
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
|
||||
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
|
||||
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
|
||||
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardRule, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit, r.Images, r.Meshes); err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
for _, r := range records {
|
||||
hit := 0
|
||||
if r.CacheHit {
|
||||
hit = 1
|
||||
}
|
||||
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
|
||||
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
|
||||
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
|
||||
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardRule, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit, r.Images, r.Meshes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writeRollup(ctx, tx, records)
|
||||
})
|
||||
}
|
||||
return writeRollup(ctx, tx, records)
|
||||
}
|
||||
|
||||
// SpendSince totals cost per key since a moment.
|
||||
|
||||
Reference in new issue
Block a user