Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s
CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8547528fb7
commit
fcda4d3f85
22 files changed
+1390
-76
No files matched your search
@@ -0,0 +1,214 @@
|
||||
package slurmnode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
)
|
||||
|
||||
// accountsMarker lists what llmgw made in Slurm's accounting, a line each:
|
||||
// "account NAME" and "user LOGIN NAME". Nothing else is changed or taken
|
||||
// away: an account a Slurm administrator made is theirs.
|
||||
func (k *Keeper) accountsMarker() string { return k.marker + "-accounts" }
|
||||
|
||||
// maxEnded is the most ended jobs sent at once: a day's on a busy cluster
|
||||
// would otherwise be a status message of megabytes.
|
||||
const maxEnded = 2000
|
||||
|
||||
// accounts keeps Slurm's accounts and their users as want says, on the
|
||||
// controller.
|
||||
func (k *Keeper) accounts(ctx context.Context, want *llmgwv1.Slurm) error {
|
||||
made := k.madeAccounts()
|
||||
if !want.GetManageAccounts() {
|
||||
if len(made) == 0 {
|
||||
return nil
|
||||
}
|
||||
return k.applyAccounts(ctx, nil, made)
|
||||
}
|
||||
return k.applyAccounts(ctx, want.GetAccounts(), made)
|
||||
}
|
||||
|
||||
func (k *Keeper) madeAccounts() []string {
|
||||
b, err := k.host.ReadFile(k.accountsMarker())
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, l := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
||||
if l != "" {
|
||||
out = append(out, l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (k *Keeper) sacctmgr(ctx context.Context, args ...string) error {
|
||||
out, err := k.host.Run(ctx, nil, "sacctmgr", append([]string{"-i"}, args...)...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sacctmgr %s: %v: %s", strings.Join(args, " "), err, lastLine(out))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *Keeper) applyAccounts(ctx context.Context, want []*llmgwv1.SlurmAccount, made []string) error {
|
||||
out, err := k.host.Run(ctx, nil, "sacctmgr", slurm.AssocFormat...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading Slurm's accounts: %v: %s", err, lastLine(out))
|
||||
}
|
||||
have := map[string]bool{} // "account NAME", "user LOGIN NAME"
|
||||
for _, a := range slurm.ParseAssociations(string(out)) {
|
||||
if a.User == "" {
|
||||
have["account "+a.Account] = true
|
||||
} else {
|
||||
have["user "+a.User+" "+a.Account] = true
|
||||
}
|
||||
}
|
||||
keep := map[string]bool{}
|
||||
var errs []error
|
||||
ours := slices.Clone(made)
|
||||
note := func(line string) {
|
||||
if !slices.Contains(ours, line) {
|
||||
ours = append(ours, line)
|
||||
}
|
||||
}
|
||||
for _, a := range want {
|
||||
name, parent := a.GetName(), a.GetParent()
|
||||
if !validAccount(name) || (parent != "root" && !validAccount(parent)) {
|
||||
errs = append(errs, fmt.Errorf("the leader sent an account %q under %q", name, parent))
|
||||
continue
|
||||
}
|
||||
line := "account " + name
|
||||
keep[line] = true
|
||||
if !have[line] {
|
||||
k.log.Info("a Slurm account is made, as llmgw's projects say", "account", name, "parent", parent)
|
||||
if err := k.sacctmgr(ctx, "add", "account", name, "parent="+parent, "description=llmgw"); err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
note(line)
|
||||
}
|
||||
for _, u := range a.GetUsers() {
|
||||
if !slurm.ValidLogin(u) {
|
||||
continue
|
||||
}
|
||||
ul := "user " + u + " " + name
|
||||
keep[ul] = true
|
||||
if have[ul] {
|
||||
continue
|
||||
}
|
||||
if err := k.sacctmgr(ctx, "add", "user", u, "account="+name); err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
note(ul)
|
||||
}
|
||||
}
|
||||
// Taken away: what llmgw made and no longer wants, users before their
|
||||
// accounts, children before their parents (made later, listed later).
|
||||
var left []string
|
||||
for i := len(ours) - 1; i >= 0; i-- {
|
||||
line := ours[i]
|
||||
if keep[line] || !strings.HasPrefix(line, "user ") {
|
||||
continue
|
||||
}
|
||||
f := strings.Fields(line)
|
||||
if !have[line] {
|
||||
continue
|
||||
}
|
||||
if err := k.removeAssoc(ctx, f[1], f[2], have, keep); err != nil {
|
||||
errs = append(errs, err)
|
||||
left = append(left, line)
|
||||
}
|
||||
}
|
||||
for i := len(ours) - 1; i >= 0; i-- {
|
||||
line := ours[i]
|
||||
if keep[line] || !strings.HasPrefix(line, "account ") {
|
||||
continue
|
||||
}
|
||||
if have[line] {
|
||||
k.log.Info("a Slurm account llmgw made is taken away; its jobs' records stay", "account", line[len("account "):])
|
||||
if err := k.sacctmgr(ctx, "remove", "account", line[len("account "):]); err != nil {
|
||||
errs = append(errs, err)
|
||||
left = append(left, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
for _, l := range ours {
|
||||
if keep[l] || slices.Contains(left, l) {
|
||||
lines = append(lines, l)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(lines, made) {
|
||||
var err error
|
||||
if len(lines) == 0 {
|
||||
err = k.host.Remove(k.accountsMarker())
|
||||
} else {
|
||||
_, err = k.write(k.accountsMarker(), []byte(strings.Join(lines, "\n")+"\n"), 0o644, "")
|
||||
}
|
||||
errs = append(errs, err)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func validAccount(s string) bool {
|
||||
return s != "" && len(s) <= 64 && strings.Trim(s, "abcdefghijklmnopqrstuvwxyz0123456789_-") == ""
|
||||
}
|
||||
|
||||
// ended are the jobs that ended in the last day, newest last.
|
||||
func (k *Keeper) ended(ctx context.Context) ([]*llmgwv1.SlurmJob, error) {
|
||||
out, err := k.host.Run(ctx, nil, "sacct", slurm.SacctArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sacct: %v: %s", err, lastLine(out))
|
||||
}
|
||||
jobs, err := slurm.ParseJobs(string(out))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(jobs) > maxEnded {
|
||||
jobs = jobs[len(jobs)-maxEnded:]
|
||||
}
|
||||
ps := make([]*llmgwv1.SlurmJob, 0, len(jobs))
|
||||
for _, j := range jobs {
|
||||
ps = append(ps, &llmgwv1.SlurmJob{Id: j.ID, User: j.User, Account: j.Account, Partition: j.Partition,
|
||||
State: j.State, StartUnix: j.Start.Unix(), EndUnix: j.End.Unix(), ElapsedSeconds: int64(j.Elapsed.Seconds()),
|
||||
Cpus: int32(j.CPUs), Gpus: int32(j.GPUs)}) //nolint:gosec // counts of a node's
|
||||
}
|
||||
return ps, nil
|
||||
}
|
||||
|
||||
// removeAssoc takes login off account. Slurm refuses to take away a
|
||||
// user's default account, so another of theirs is made the default first,
|
||||
// one llmgw keeps if there is one; with none left the user goes from the
|
||||
// accounting entirely.
|
||||
func (k *Keeper) removeAssoc(ctx context.Context, login, account string, have, keep map[string]bool) error {
|
||||
var others []string
|
||||
for line := range have {
|
||||
f := strings.Fields(line)
|
||||
if len(f) == 3 && f[0] == "user" && f[1] == login && f[2] != account {
|
||||
others = append(others, f[2])
|
||||
}
|
||||
}
|
||||
if len(others) == 0 {
|
||||
return k.sacctmgr(ctx, "remove", "user", login)
|
||||
}
|
||||
slices.SortFunc(others, func(a, b string) int {
|
||||
ka, kb := keep["user "+login+" "+a], keep["user "+login+" "+b]
|
||||
switch {
|
||||
case ka && !kb:
|
||||
return -1
|
||||
case kb && !ka:
|
||||
return 1
|
||||
}
|
||||
return strings.Compare(a, b)
|
||||
})
|
||||
if err := k.sacctmgr(ctx, "modify", "user", login, "set", "defaultaccount="+others[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
return k.sacctmgr(ctx, "remove", "user", login, "account="+account)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package slurmnode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
)
|
||||
|
||||
func TestAccountsMadeAndTakenAwayOnlyIfLlmgwsOwn(t *testing.T) {
|
||||
h := newFake()
|
||||
k := New(h, nil, "/var/lib/llmgw-agent/slurm", nil)
|
||||
assoc := "sacctmgr " + strings.Join(slurm.AssocFormat, " ")
|
||||
// root, and an account a Slurm administrator made.
|
||||
h.outputs[assoc] = "root||\nroot|root|\nphysics||root\n"
|
||||
m := &llmgwv1.Slurm{ManageAccounts: true, Accounts: []*llmgwv1.SlurmAccount{
|
||||
{Name: "test", Parent: "root"}, {Name: "test-arge", Parent: "test"},
|
||||
{Name: "test-arge-deneme", Parent: "test-arge", Users: []string{"ada", "ayse"}},
|
||||
}}
|
||||
if err := k.accounts(context.Background(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"sacctmgr -i add account test parent=root description=llmgw",
|
||||
"sacctmgr -i add account test-arge-deneme parent=test-arge description=llmgw",
|
||||
"sacctmgr -i add user ayse account=test-arge-deneme"} {
|
||||
if !slices.Contains(h.ran, want) {
|
||||
t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n"))
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(h.files[k.accountsMarker()]), "user ada test-arge-deneme") {
|
||||
t.Errorf("marker %q", h.files[k.accountsMarker()])
|
||||
}
|
||||
|
||||
// Now there; ayse leaves the project, and the team goes.
|
||||
h.outputs[assoc] = "root||\nphysics||root\ntest||root\ntest-arge||test\ntest-arge-deneme||test-arge\n" +
|
||||
"test-arge-deneme|ada|test-arge\ntest-arge-deneme|ayse|test-arge\n"
|
||||
h.ran = nil
|
||||
m.Accounts = []*llmgwv1.SlurmAccount{{Name: "test", Parent: "root"}}
|
||||
if err := k.accounts(context.Background(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Neither has another account: they go from the accounting.
|
||||
want := []string{"sacctmgr -i remove user ayse", "sacctmgr -i remove user ada",
|
||||
"sacctmgr -i remove account test-arge-deneme", "sacctmgr -i remove account test-arge"}
|
||||
var got []string
|
||||
for _, c := range h.ran {
|
||||
if strings.Contains(c, "remove") || strings.Contains(c, " add ") {
|
||||
got = append(got, c)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("ran:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if slices.ContainsFunc(h.ran, func(c string) bool { return strings.Contains(c, "physics") }) {
|
||||
t.Error("touched an account llmgw did not make")
|
||||
}
|
||||
|
||||
// One still on another account: that one is made the default first.
|
||||
h.outputs[assoc] = "root||\nphysics||root\nphysics|ada|\ntest||root\nx||test\nx|ada|\n"
|
||||
h.files[k.accountsMarker()] = []byte("account test\naccount x\nuser ada x\n")
|
||||
h.ran = nil
|
||||
m.Accounts = []*llmgwv1.SlurmAccount{{Name: "test", Parent: "root"}}
|
||||
if err := k.accounts(context.Background(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if i, j := slices.Index(h.ran, "sacctmgr -i modify user ada set defaultaccount=physics"),
|
||||
slices.Index(h.ran, "sacctmgr -i remove user ada account=x"); i < 0 || j < i {
|
||||
t.Errorf("ran:\n%s", strings.Join(h.ran, "\n"))
|
||||
}
|
||||
|
||||
// Off: what llmgw made goes.
|
||||
h.outputs[assoc] = "root||\nphysics||root\ntest||root\n"
|
||||
h.ran = nil
|
||||
if err := k.accounts(context.Background(), &llmgwv1.Slurm{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(h.ran, "sacctmgr -i remove account test") || h.files[k.accountsMarker()] != nil {
|
||||
t.Errorf("not taken away:\n%s", strings.Join(h.ran, "\n"))
|
||||
}
|
||||
}
|
||||
@@ -267,6 +267,20 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
|
||||
return fail("slurmctld", err)
|
||||
}
|
||||
st.SlurmctldRunning = k.active(ctx, "slurmctld")
|
||||
// The accounts, and the jobs that ended, need the accounting.
|
||||
if st.SlurmdbdRunning && st.SlurmctldRunning {
|
||||
if err := k.accounts(ctx, want); err != nil {
|
||||
k.log.Warn("keeping Slurm's accounts", "err", err)
|
||||
if acctErr == nil {
|
||||
acctErr = fmt.Errorf("accounts: %w", err)
|
||||
}
|
||||
}
|
||||
if ended, err := k.ended(ctx); err != nil {
|
||||
k.log.Warn("reading the jobs that ended", "err", err)
|
||||
} else {
|
||||
st.Ended = ended
|
||||
}
|
||||
}
|
||||
}
|
||||
if want.GetNode() {
|
||||
if err := k.node(ctx, want, st); err != nil {
|
||||
|
||||
@@ -247,6 +247,9 @@ func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
fmt.Fprintf(w, "shared files: %s\n", sharedLine(st.Shared))
|
||||
if st.Users != "" {
|
||||
fmt.Fprintf(w, "users: %d, made from llmgw's accounts\n", st.UserCount)
|
||||
if st.Accounts > 0 {
|
||||
fmt.Fprintf(w, "accounts: %d, one for each project (sbatch --account=ORG-TEAM-PROJECT)\n", st.Accounts)
|
||||
}
|
||||
}
|
||||
if line := containersLine(st); line != "" {
|
||||
fmt.Fprintf(w, "containers: %s\n", line)
|
||||
|
||||
@@ -10,6 +10,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"maps"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
@@ -487,6 +488,15 @@ func newServerCmd() *cobra.Command {
|
||||
return out, err
|
||||
})
|
||||
api.SetSlurmUsers(st)
|
||||
// Slurm's accounts from the projects, and the jobs
|
||||
// that ended priced as the cluster's hours are.
|
||||
slurmSvc.SetAccounts(func(ctx context.Context) ([]slurm.Account, error) {
|
||||
return slurmAccounts(ctx, st)
|
||||
})
|
||||
slurmSvc.SetJobs(func(ctx context.Context, jobs []slurmsvc.JobRecord) error {
|
||||
au := current.Load().Auth
|
||||
return recordSlurmJobs(ctx, st, current.Load().Slurm.Cluster(), au.GPUHourCost, au.CoreHourCost, jobs, log)
|
||||
})
|
||||
agentSvc.SetSlurm(slurmSvc)
|
||||
go slurmSvc.Run(ctx)
|
||||
api.SetSlurm(slurmSvc)
|
||||
@@ -1447,3 +1457,76 @@ func accountingPrice(p config.Price) accounting.Price {
|
||||
return accounting.Price{PromptPer1k: p.PromptPer1k, CompletionPer1k: p.CompletionPer1k, CachedPer1k: p.CachedPer1k,
|
||||
PerImage: p.PerImage, PerMesh: p.PerMesh}
|
||||
}
|
||||
|
||||
// slurmAccounts are Slurm's accounts from the store's projects, members and
|
||||
// Slurm users.
|
||||
func slurmAccounts(ctx context.Context, st *storepkg.DB) ([]slurm.Account, error) {
|
||||
ps, err := st.SlurmProjects(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
ms, err := st.AllMemberships(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
us, err := st.SlurmUsers(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
logins := make(map[string]string, len(us))
|
||||
for _, u := range us {
|
||||
logins[u.UserID] = u.Login
|
||||
}
|
||||
tps := make([]slurm.TenancyProject, 0, len(ps))
|
||||
for _, p := range ps {
|
||||
tps = append(tps, slurm.TenancyProject{ID: p.ID, Name: p.Name, TeamID: p.TeamID, TeamName: p.TeamName,
|
||||
OrgID: p.OrgID, OrgName: p.OrgName})
|
||||
}
|
||||
tms := make([]slurm.TenancyMember, 0, len(ms))
|
||||
for _, m := range ms {
|
||||
tms = append(tms, slurm.TenancyMember{UserID: m.UserID, Scope: string(m.Scope), ScopeID: m.ScopeID})
|
||||
}
|
||||
return slurm.Accounts(tps, tms, logins), nil
|
||||
}
|
||||
|
||||
// recordSlurmJobs records the jobs that ended, each with its project (from
|
||||
// its account) and person (from its login), priced as the hours the
|
||||
// cluster's models hold are: a GPU job its cards' hours, any other its
|
||||
// cores'.
|
||||
func recordSlurmJobs(ctx context.Context, st *storepkg.DB, cluster string, gpuHour, coreHour float64,
|
||||
jobs []slurmsvc.JobRecord, log *slog.Logger) error {
|
||||
as, err := slurmAccounts(ctx, st)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
project := map[string]string{}
|
||||
for _, a := range as {
|
||||
if a.Project != "" {
|
||||
project[a.Name] = a.Project
|
||||
}
|
||||
}
|
||||
us, err := st.SlurmUsers(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
person := map[string]string{}
|
||||
for _, u := range us {
|
||||
person[u.Login] = u.UserID
|
||||
}
|
||||
rows := make([]storepkg.SlurmJob, 0, len(jobs))
|
||||
for _, j := range jobs {
|
||||
secs := j.Elapsed.Seconds()
|
||||
cost := float64(j.CPUs) * secs / 3600 * coreHour
|
||||
if j.GPUs > 0 {
|
||||
cost = float64(j.GPUs) * secs / 3600 * gpuHour
|
||||
}
|
||||
rows = append(rows, storepkg.SlurmJob{Key: cluster + "/" + j.ID, UserID: person[j.Login], ProjectID: project[j.Account],
|
||||
Login: j.Login, Account: j.Account, Partition: j.Partition, State: j.State, Start: j.Start, End: j.End,
|
||||
ElapsedSeconds: int64(secs), CPUs: int64(j.CPUs), GPUs: int64(j.GPUs), CostMicros: int64(math.Ceil(cost * 1e6))})
|
||||
}
|
||||
n, err := st.RecordSlurmJobs(ctx, rows)
|
||||
if n > 0 {
|
||||
log.Info("Slurm jobs that ended recorded, with their cost", "jobs", n)
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -116,6 +116,8 @@ type Status struct {
|
||||
// machine has pyxis and enroot, "installing" until then, empty when
|
||||
// slurm.containers is off.
|
||||
Containers string `json:"containers,omitempty"`
|
||||
// Accounts is how many of Slurm's accounts are llmgw's projects'.
|
||||
Accounts int `json:"accounts,omitempty"`
|
||||
}
|
||||
|
||||
// User is a Linux user made from an llmgw account.
|
||||
@@ -166,6 +168,53 @@ type Service struct {
|
||||
users []User
|
||||
usersAt time.Time
|
||||
usersOK bool
|
||||
// accountList reads Slurm's accounts from the projects, with the users;
|
||||
// accountsOK is false until it has been read, and the controller is
|
||||
// told nothing of accounts until then: none would take every one away.
|
||||
accountList func(ctx context.Context) ([]slurm.Account, error)
|
||||
accounts []slurm.Account
|
||||
accountsOK bool
|
||||
// jobSink records the jobs that ended; recorded are those it took,
|
||||
// recording whether a recording is under way.
|
||||
jobSink func(ctx context.Context, jobs []JobRecord) error
|
||||
recorded map[string]time.Time
|
||||
recording bool
|
||||
}
|
||||
|
||||
// JobRecord is a job that ended, for llmgw's accounting.
|
||||
type JobRecord struct {
|
||||
ID, Login, Account, Partition, State string
|
||||
Start, End time.Time
|
||||
Elapsed time.Duration
|
||||
CPUs, GPUs int
|
||||
}
|
||||
|
||||
// SetAccounts has Slurm's accounts read with list, beside the users.
|
||||
func (s *Service) SetAccounts(list func(ctx context.Context) ([]slurm.Account, error)) {
|
||||
s.mu.Lock()
|
||||
s.accountList = list
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// SetJobs has the jobs that ended recorded with record; each is given to it
|
||||
// once it took it without an error.
|
||||
func (s *Service) SetJobs(record func(ctx context.Context, jobs []JobRecord) error) {
|
||||
s.mu.Lock()
|
||||
s.jobSink = record
|
||||
s.mu.Unlock()
|
||||
}
|
||||
|
||||
// AccountsOf are the accounts a login may run jobs on.
|
||||
func (s *Service) AccountsOf(login string) []string {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
var out []string
|
||||
for _, a := range s.accounts {
|
||||
if slices.Contains(a.Users, login) {
|
||||
out = append(out, a.Name)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// UsersEvery is how often the users are read again without being asked.
|
||||
@@ -199,6 +248,64 @@ func (s *Service) readUsers(ctx context.Context, set config.SlurmSettings) {
|
||||
return
|
||||
}
|
||||
s.users, s.usersOK = us, true
|
||||
if s.accountList == nil {
|
||||
return
|
||||
}
|
||||
as, err := s.accountList(ctx)
|
||||
if err != nil {
|
||||
s.log.Warn("could not read Slurm's accounts from the projects; the controller keeps the last ones", "err", err)
|
||||
return
|
||||
}
|
||||
s.accounts, s.accountsOK = as, true
|
||||
}
|
||||
|
||||
// takeJobs records the jobs a controller says ended that are not recorded
|
||||
// yet, away from the agent's stream.
|
||||
func (s *Service) takeJobs(ended []*llmgwv1.SlurmJob) {
|
||||
s.mu.Lock()
|
||||
if s.jobSink == nil || s.recording {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
if s.recorded == nil {
|
||||
s.recorded = map[string]time.Time{}
|
||||
}
|
||||
for id, at := range s.recorded {
|
||||
if time.Since(at) > 3*24*time.Hour {
|
||||
delete(s.recorded, id)
|
||||
}
|
||||
}
|
||||
var jobs []JobRecord
|
||||
for _, j := range ended {
|
||||
if _, ok := s.recorded[j.GetId()]; ok || j.GetId() == "" {
|
||||
continue
|
||||
}
|
||||
jobs = append(jobs, JobRecord{ID: j.GetId(), Login: j.GetUser(), Account: j.GetAccount(), Partition: j.GetPartition(),
|
||||
State: j.GetState(), Start: time.Unix(j.GetStartUnix(), 0), End: time.Unix(j.GetEndUnix(), 0),
|
||||
Elapsed: time.Duration(j.GetElapsedSeconds()) * time.Second, CPUs: int(j.GetCpus()), GPUs: int(j.GetGpus())})
|
||||
}
|
||||
if len(jobs) == 0 {
|
||||
s.mu.Unlock()
|
||||
return
|
||||
}
|
||||
sink := s.jobSink
|
||||
s.recording = true
|
||||
s.mu.Unlock()
|
||||
go func() {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Minute)
|
||||
defer cancel()
|
||||
err := sink(ctx, jobs)
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.recording = false
|
||||
if err != nil {
|
||||
s.log.Warn("could not record the Slurm jobs that ended; tried again with the next status", "jobs", len(jobs), "err", err)
|
||||
return
|
||||
}
|
||||
for _, j := range jobs {
|
||||
s.recorded[j.ID] = time.Now()
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
// HoldFor is how long a new leader waits for a node it neither has heard
|
||||
@@ -276,7 +383,11 @@ func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) {
|
||||
s.mu.Lock()
|
||||
prev := s.status[node]
|
||||
s.status[node] = r
|
||||
isCtl := s.settings().Controller == node
|
||||
s.mu.Unlock()
|
||||
if isCtl && len(st.GetEnded()) > 0 {
|
||||
s.takeJobs(st.GetEnded())
|
||||
}
|
||||
if r.Error != "" && (prev == nil || prev.Error != r.Error) {
|
||||
s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error)
|
||||
}
|
||||
@@ -517,6 +628,7 @@ func (s *Service) cluster(set config.SlurmSettings) (slurm.Cluster, []string) {
|
||||
return c, nil
|
||||
}
|
||||
c.ControllerAddr = s.controllerAddr(set)
|
||||
c.EnforceAccounts = set.Users == config.UsersLlmgw && s.accountList != nil
|
||||
var waiting []string
|
||||
for _, node := range set.Nodes {
|
||||
sn, ok := s.reported(node)
|
||||
@@ -582,6 +694,15 @@ func (s *Service) message(set config.SlurmSettings, node string, lent bool, key
|
||||
if !s.sharedPart(set, node, m) || !s.usersPart(set, node, m) {
|
||||
return nil
|
||||
}
|
||||
if isCtl && set.Users == config.UsersLlmgw && s.accountList != nil {
|
||||
if !s.accountsOK {
|
||||
return nil
|
||||
}
|
||||
m.ManageAccounts = true
|
||||
for _, a := range s.accounts {
|
||||
m.Accounts = append(m.Accounts, &llmgwv1.SlurmAccount{Name: a.Name, Parent: a.Parent, Users: slices.Clone(a.Users)})
|
||||
}
|
||||
}
|
||||
return m
|
||||
}
|
||||
|
||||
@@ -675,6 +796,11 @@ func (s *Service) Status() Status {
|
||||
Partition: set.PartitionName(), Ctld: s.status[set.Controller], Shared: set.Shared, Users: set.Users}
|
||||
if set.Users == config.UsersLlmgw {
|
||||
out.UserCount = len(s.users)
|
||||
for _, a := range s.accounts {
|
||||
if a.Project != "" {
|
||||
out.Accounts++
|
||||
}
|
||||
}
|
||||
}
|
||||
if !set.Enabled {
|
||||
return out
|
||||
|
||||
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
)
|
||||
|
||||
type fakeInv struct {
|
||||
@@ -323,3 +324,59 @@ func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
|
||||
t.Fatal("pyxis still loaded with containers off")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllerKeepsTheProjectsAccountsAndJobsAreRecordedOnce(t *testing.T) {
|
||||
r := newRig()
|
||||
r.set.Users = config.UsersLlmgw
|
||||
r.svc.SetUsers(func(context.Context) ([]User, error) { return []User{{Login: "ayse", UID: 200001}}, nil })
|
||||
read := 0
|
||||
r.svc.SetAccounts(func(context.Context) ([]slurm.Account, error) {
|
||||
read++
|
||||
return []slurm.Account{{Name: "test", Parent: "root"},
|
||||
{Name: "test-arge-deneme", Parent: "test", Project: "p1", Users: []string{"ayse"}}}, nil
|
||||
})
|
||||
got := make(chan []JobRecord, 4)
|
||||
r.svc.SetJobs(func(_ context.Context, jobs []JobRecord) error { got <- jobs; return nil })
|
||||
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
|
||||
r.tick()
|
||||
ctl := r.agents.last("node06")
|
||||
if !ctl.GetManageAccounts() || len(ctl.GetAccounts()) != 2 || ctl.GetAccounts()[1].GetUsers()[0] != "ayse" {
|
||||
t.Fatalf("controller told %v", ctl.GetAccounts())
|
||||
}
|
||||
if r.agents.last("cuda01").GetManageAccounts() {
|
||||
t.Error("a node was told the accounts")
|
||||
}
|
||||
if !strings.Contains(ctl.GetFiles()["slurm.conf"], "AccountingStorageEnforce=associations") {
|
||||
t.Error("jobs not held to accounts")
|
||||
}
|
||||
if r.svc.Status().Accounts != 1 || len(r.svc.AccountsOf("ayse")) != 1 {
|
||||
t.Errorf("status %d, ayse's %v", r.svc.Status().Accounts, r.svc.AccountsOf("ayse"))
|
||||
}
|
||||
ended := []*llmgwv1.SlurmJob{{Id: "41", User: "ayse", Account: "test-arge-deneme", ElapsedSeconds: 60, Gpus: 1}}
|
||||
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
|
||||
select {
|
||||
case jobs := <-got:
|
||||
if len(jobs) != 1 || jobs[0].Login != "ayse" || jobs[0].GPUs != 1 {
|
||||
t.Fatalf("jobs %+v", jobs)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("not recorded")
|
||||
}
|
||||
// The next status says it again: not recorded twice.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
r.svc.mu.Lock()
|
||||
busy := r.svc.recording
|
||||
r.svc.mu.Unlock()
|
||||
if !busy {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
|
||||
select {
|
||||
case jobs := <-got:
|
||||
t.Fatalf("recorded again: %+v", jobs)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,193 @@
|
||||
package slurm
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Slurm's accounts from llmgw's tree: an account for each organization,
|
||||
// team and project, nested as they are, and on each project's the users
|
||||
// that may use it, its members and its team's and organization's. A job
|
||||
// is charged to the project its --account names (`sbatch --account
|
||||
// acme-ml-training`), and what it used to that project's spend.
|
||||
|
||||
// Account is one Slurm account llmgw keeps.
|
||||
type Account struct {
|
||||
Name string
|
||||
Parent string // "root" at the top
|
||||
// Project is the llmgw project it is for; empty for an organization's
|
||||
// or a team's.
|
||||
Project string
|
||||
// Users are the logins that may run jobs on it.
|
||||
Users []string
|
||||
}
|
||||
|
||||
// TenancyProject is a project with where it is.
|
||||
type TenancyProject struct {
|
||||
ID, Name, TeamID, TeamName, OrgID, OrgName string
|
||||
}
|
||||
|
||||
// TenancyMember is a membership: a person in an organization, team or
|
||||
// project ("org", "team", "project").
|
||||
type TenancyMember struct {
|
||||
UserID, Scope, ScopeID string
|
||||
}
|
||||
|
||||
// Accounts are the accounts for projects, with logins by user id. Parents
|
||||
// come before their children, as sacctmgr needs them.
|
||||
func Accounts(projects []TenancyProject, members []TenancyMember, logins map[string]string) []Account {
|
||||
// By kind and id: an organization, its team and its project may share
|
||||
// an id (the first install's are all "default").
|
||||
names := map[string]string{} // kind/id -> name
|
||||
taken := map[string]string{} // name -> kind/id
|
||||
name := func(id string, parts ...string) string {
|
||||
if n, ok := names[id]; ok {
|
||||
return n
|
||||
}
|
||||
n := AccountName(parts...)
|
||||
if other, ok := taken[n]; ok && other != id {
|
||||
// Two places whose names come out the same: told apart by id.
|
||||
h := sha256.Sum256([]byte(id))
|
||||
n = AccountName(append(parts, hex.EncodeToString(h[:3]))...)
|
||||
}
|
||||
names[id], taken[n] = n, id
|
||||
return n
|
||||
}
|
||||
in := map[string][]string{} // kind/id -> logins
|
||||
for _, m := range members {
|
||||
if l, ok := logins[m.UserID]; ok {
|
||||
in[m.Scope+"/"+m.ScopeID] = append(in[m.Scope+"/"+m.ScopeID], l)
|
||||
}
|
||||
}
|
||||
ps := slices.Clone(projects)
|
||||
slices.SortFunc(ps, func(a, b TenancyProject) int {
|
||||
return strings.Compare(a.OrgName+"\x00"+a.TeamName+"\x00"+a.Name, b.OrgName+"\x00"+b.TeamName+"\x00"+b.Name)
|
||||
})
|
||||
var out []Account
|
||||
seen := map[string]bool{}
|
||||
for _, p := range ps {
|
||||
org := name("org/"+p.OrgID, p.OrgName)
|
||||
team := name("team/"+p.TeamID, p.OrgName, p.TeamName)
|
||||
proj := name("project/"+p.ID, p.OrgName, p.TeamName, p.Name)
|
||||
if !seen[org] {
|
||||
out = append(out, Account{Name: org, Parent: "root"})
|
||||
seen[org] = true
|
||||
}
|
||||
if !seen[team] {
|
||||
out = append(out, Account{Name: team, Parent: org})
|
||||
seen[team] = true
|
||||
}
|
||||
var users []string
|
||||
for _, id := range []string{"project/" + p.ID, "team/" + p.TeamID, "org/" + p.OrgID} {
|
||||
users = append(users, in[id]...)
|
||||
}
|
||||
slices.Sort(users)
|
||||
out = append(out, Account{Name: proj, Parent: team, Project: p.ID, Users: slices.Compact(users)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// AccountName is a Slurm account name from names: lower case, ASCII,
|
||||
// joined with "-", at most 64 characters.
|
||||
func AccountName(parts ...string) string {
|
||||
var segs []string
|
||||
for _, p := range parts {
|
||||
var b strings.Builder
|
||||
for _, r := range strings.ToLower(p) {
|
||||
switch {
|
||||
case r >= 'a' && r <= 'z', r >= '0' && r <= '9', r == '_':
|
||||
b.WriteRune(r)
|
||||
case strings.ContainsRune("çğıöşüâîû", r):
|
||||
b.WriteString(map[rune]string{'ç': "c", 'ğ': "g", 'ı': "i", 'ö': "o", 'ş': "s", 'ü': "u",
|
||||
'â': "a", 'î': "i", 'û': "u"}[r])
|
||||
default:
|
||||
b.WriteByte('_')
|
||||
}
|
||||
}
|
||||
if s := strings.Trim(b.String(), "_"); s != "" {
|
||||
segs = append(segs, s)
|
||||
}
|
||||
}
|
||||
n := strings.Join(segs, "-")
|
||||
if n == "" {
|
||||
n = "x"
|
||||
}
|
||||
if len(n) > 64 {
|
||||
h := sha256.Sum256([]byte(n))
|
||||
n = n[:57] + "-" + hex.EncodeToString(h[:3])
|
||||
}
|
||||
return n
|
||||
}
|
||||
|
||||
// Association is one line of `sacctmgr show assoc format=account,user,parentname`.
|
||||
type Association struct {
|
||||
Account, User, Parent string
|
||||
}
|
||||
|
||||
// AssocFormat is what ParseAssociations reads.
|
||||
var AssocFormat = []string{"show", "assoc", "format=account,user,parentname", "-n", "-P"}
|
||||
|
||||
// ParseAssociations reads sacctmgr's parsable lines.
|
||||
func ParseAssociations(out string) []Association {
|
||||
var as []Association
|
||||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||
f := strings.Split(line, "|")
|
||||
if len(f) < 3 || f[0] == "" {
|
||||
continue
|
||||
}
|
||||
as = append(as, Association{Account: f[0], User: f[1], Parent: f[2]})
|
||||
}
|
||||
return as
|
||||
}
|
||||
|
||||
// Job is a finished job, as sacct has it.
|
||||
type Job struct {
|
||||
ID, User, Account, Partition, State string
|
||||
Start, End time.Time
|
||||
Elapsed time.Duration
|
||||
CPUs, GPUs int
|
||||
}
|
||||
|
||||
// SacctArgs are sacct's for the jobs that ended in the last day.
|
||||
var SacctArgs = []string{"-a", "-X", "-n", "-P", "--noconvert", "-S", "now-1days", "-E", "now",
|
||||
"-s", "CD,F,TO,CA,NF,OOM,PR,DL,BF",
|
||||
"-o", "JobIDRaw,User,Account,Partition,Start,End,ElapsedRaw,AllocTRES,State"}
|
||||
|
||||
// ParseJobs reads sacct's lines; a job that has not ended is left out.
|
||||
func ParseJobs(out string) ([]Job, error) {
|
||||
var jobs []Job
|
||||
for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
f := strings.Split(line, "|")
|
||||
if len(f) < 9 {
|
||||
return nil, fmt.Errorf("sacct said %q, not 9 fields", line)
|
||||
}
|
||||
end, err := time.ParseInLocation("2006-01-02T15:04:05", f[5], time.Local)
|
||||
if err != nil {
|
||||
continue // Unknown: still running, or never started
|
||||
}
|
||||
start, _ := time.ParseInLocation("2006-01-02T15:04:05", f[4], time.Local)
|
||||
secs, _ := strconv.ParseInt(f[6], 10, 64)
|
||||
j := Job{ID: f[0], User: f[1], Account: f[2], Partition: f[3], Start: start, End: end,
|
||||
Elapsed: time.Duration(secs) * time.Second, State: strings.Fields(f[8] + " ")[0]}
|
||||
for _, t := range strings.Split(f[7], ",") {
|
||||
k, v, _ := strings.Cut(t, "=")
|
||||
n, _ := strconv.Atoi(v)
|
||||
switch k {
|
||||
case "cpu":
|
||||
j.CPUs = n
|
||||
case "gres/gpu":
|
||||
j.GPUs = n
|
||||
}
|
||||
}
|
||||
jobs = append(jobs, j)
|
||||
}
|
||||
return jobs, nil
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package slurm
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestAccountsFollowTheTree(t *testing.T) {
|
||||
projects := []TenancyProject{
|
||||
{ID: "p1", Name: "deneme", TeamID: "t1", TeamName: "arge", OrgID: "o1", OrgName: "Test"},
|
||||
{ID: "p2", Name: "Görüntü İşleme", TeamID: "t1", TeamName: "arge", OrgID: "o1", OrgName: "Test"},
|
||||
{ID: "p3", Name: "x", TeamID: "t2", TeamName: "ops", OrgID: "o2", OrgName: "Other"},
|
||||
}
|
||||
members := []TenancyMember{
|
||||
{UserID: "u-ayse", Scope: "project", ScopeID: "p1"},
|
||||
{UserID: "u-ada", Scope: "team", ScopeID: "t1"},
|
||||
{UserID: "u-bob", Scope: "org", ScopeID: "o2"},
|
||||
{UserID: "u-nologin", Scope: "project", ScopeID: "p1"},
|
||||
}
|
||||
logins := map[string]string{"u-ayse": "ayse", "u-ada": "ada", "u-bob": "bob"}
|
||||
got := Accounts(projects, members, logins)
|
||||
var names []string
|
||||
by := map[string]Account{}
|
||||
for _, a := range got {
|
||||
names = append(names, a.Name)
|
||||
by[a.Name] = a
|
||||
}
|
||||
// Parents first, each before its children.
|
||||
want := []string{"other", "other-ops", "other-ops-x", "test", "test-arge", "test-arge-goruntu_isleme", "test-arge-deneme"}
|
||||
if !slices.Equal(names, want) {
|
||||
t.Fatalf("names %v", names)
|
||||
}
|
||||
if a := by["test-arge-deneme"]; a.Parent != "test-arge" || a.Project != "p1" || !slices.Equal(a.Users, []string{"ada", "ayse"}) {
|
||||
t.Errorf("deneme %+v", a)
|
||||
}
|
||||
if a := by["other-ops-x"]; !slices.Equal(a.Users, []string{"bob"}) {
|
||||
t.Errorf("x %+v", a)
|
||||
}
|
||||
if a := by["test"]; a.Parent != "root" || a.Project != "" || len(a.Users) != 0 {
|
||||
t.Errorf("org %+v", a)
|
||||
}
|
||||
if n := AccountName("Test", "arge", "Görüntü İşleme"); n != "test-arge-goruntu_isleme" {
|
||||
t.Errorf("name %q", n)
|
||||
}
|
||||
// The first install's tree: one id for all three.
|
||||
def := Accounts([]TenancyProject{{ID: "default", Name: "default", TeamID: "default", TeamName: "default",
|
||||
OrgID: "default", OrgName: "default"}}, []TenancyMember{{UserID: "u", Scope: "project", ScopeID: "default"}},
|
||||
map[string]string{"u": "admin2"})
|
||||
if len(def) != 3 || def[2].Name != "default-default-default" || def[2].Parent != "default-default" ||
|
||||
def[1].Parent != "default" || !slices.Equal(def[2].Users, []string{"admin2"}) {
|
||||
t.Errorf("default tree %+v", def)
|
||||
}
|
||||
// Two that come out the same are told apart.
|
||||
same := Accounts([]TenancyProject{
|
||||
{ID: "a", Name: "ml", TeamID: "t", TeamName: "x", OrgID: "o", OrgName: "A"},
|
||||
{ID: "b", Name: "ML", TeamID: "t", TeamName: "x", OrgID: "o", OrgName: "A"},
|
||||
}, nil, nil)
|
||||
if same[2].Name == same[3].Name {
|
||||
t.Errorf("same name twice: %+v", same)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseJobs(t *testing.T) {
|
||||
out := "41|ayse|test-arge-deneme|gpu|2026-11-15T10:00:00|2026-11-15T10:30:00|1800|billing=4,cpu=4,gres/gpu=1,mem=16G,node=1|COMPLETED\n" +
|
||||
"42|ada|test-arge-deneme|gpu|2026-11-15T10:00:00|Unknown|0|cpu=2|RUNNING\n" +
|
||||
"43|root|root|gpu|2026-11-15T09:00:00|2026-11-15T09:00:05|5|cpu=1,mem=1G|CANCELLED by 0\n"
|
||||
jobs, err := ParseJobs(out)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(jobs) != 2 {
|
||||
t.Fatalf("jobs %+v", jobs)
|
||||
}
|
||||
j := jobs[0]
|
||||
if j.ID != "41" || j.User != "ayse" || j.GPUs != 1 || j.CPUs != 4 || j.Elapsed != 30*time.Minute || j.State != "COMPLETED" {
|
||||
t.Errorf("job %+v", j)
|
||||
}
|
||||
if jobs[1].State != "CANCELLED" {
|
||||
t.Errorf("state %q", jobs[1].State)
|
||||
}
|
||||
if _, err := ParseJobs("short|line\n"); err == nil {
|
||||
t.Error("a short line taken")
|
||||
}
|
||||
as := ParseAssociations("root||\ntest|ayse|test-arge\n")
|
||||
if len(as) != 2 || as[1] != (Association{"test", "ayse", "test-arge"}) {
|
||||
t.Errorf("assoc %+v", as)
|
||||
}
|
||||
}
|
||||
@@ -131,6 +131,10 @@ type Cluster struct {
|
||||
Controller string
|
||||
ControllerAddr string
|
||||
Partition string
|
||||
// EnforceAccounts refuses a job whose user has no account in Slurm's
|
||||
// accounting: with llmgw's users, each is on the accounts of the
|
||||
// projects they are in, and a job is charged to one of them.
|
||||
EnforceAccounts bool
|
||||
// Nodes without hardware (not reported yet) are left out of the files
|
||||
// until they are.
|
||||
Nodes []Node
|
||||
@@ -199,6 +203,9 @@ func (c Cluster) SlurmConf() string {
|
||||
// The jobs' records, in slurmdbd beside slurmctld (DbdConf).
|
||||
fmt.Fprintf(&b, "AccountingStorageType=accounting_storage/slurmdbd\nAccountingStorageHost=%s\nAccountingStoragePort=%d\n",
|
||||
c.ControllerAddr, DbdPort)
|
||||
if c.EnforceAccounts {
|
||||
b.WriteString("# A job runs on an account of a project its user is in.\nAccountingStorageEnforce=associations\n")
|
||||
}
|
||||
b.WriteString(`SlurmctldParameters=enable_configless
|
||||
SlurmctldPort=6817
|
||||
SlurmdPort=6818
|
||||
|
||||
@@ -45,6 +45,7 @@ var copyOrder = []string{
|
||||
"usage_1h",
|
||||
"usage_1d",
|
||||
"held_1h",
|
||||
"slurm_jobs",
|
||||
"models",
|
||||
// No foreign key to models, deliberately: a measurement outlives the
|
||||
// catalog entry it was taken for, and losing it when somebody removes a
|
||||
|
||||
@@ -0,0 +1,20 @@
|
||||
-- Ported from the SQLite migration of the same number.
|
||||
-- The Slurm jobs that ended, each recorded once (job_key is the cluster's
|
||||
-- name and the job's id), with what it used and cost; a usage row (model
|
||||
-- "slurm") carries the cost into the usage pages with the requests'.
|
||||
CREATE TABLE slurm_jobs (
|
||||
job_key TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL DEFAULT '',
|
||||
project_id TEXT NOT NULL DEFAULT '',
|
||||
login TEXT NOT NULL DEFAULT '',
|
||||
account TEXT NOT NULL DEFAULT '',
|
||||
partition TEXT NOT NULL DEFAULT '',
|
||||
state TEXT NOT NULL DEFAULT '',
|
||||
started_at BIGINT NOT NULL DEFAULT 0,
|
||||
ended_at BIGINT NOT NULL DEFAULT 0,
|
||||
elapsed_seconds BIGINT NOT NULL DEFAULT 0,
|
||||
cpus BIGINT NOT NULL DEFAULT 0,
|
||||
gpus BIGINT NOT NULL DEFAULT 0,
|
||||
cost_micros BIGINT NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX slurm_jobs_by_project ON slurm_jobs(project_id, ended_at);
|
||||
@@ -0,0 +1,19 @@
|
||||
-- The Slurm jobs that ended, each recorded once (job_key is the cluster's
|
||||
-- name and the job's id), with what it used and cost; a usage row (model
|
||||
-- "slurm") carries the cost into the usage pages with the requests'.
|
||||
CREATE TABLE slurm_jobs (
|
||||
job_key TEXT PRIMARY KEY,
|
||||
user_id TEXT NOT NULL DEFAULT '',
|
||||
project_id TEXT NOT NULL DEFAULT '',
|
||||
login TEXT NOT NULL DEFAULT '',
|
||||
account TEXT NOT NULL DEFAULT '',
|
||||
partition TEXT NOT NULL DEFAULT '',
|
||||
state TEXT NOT NULL DEFAULT '',
|
||||
started_at BIGINT NOT NULL DEFAULT 0,
|
||||
ended_at BIGINT NOT NULL DEFAULT 0,
|
||||
elapsed_seconds BIGINT NOT NULL DEFAULT 0,
|
||||
cpus BIGINT NOT NULL DEFAULT 0,
|
||||
gpus BIGINT NOT NULL DEFAULT 0,
|
||||
cost_micros BIGINT NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE INDEX slurm_jobs_by_project ON slurm_jobs(project_id, ended_at);
|
||||
@@ -0,0 +1,65 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SlurmProject is a project with where it is, for Slurm's accounts.
|
||||
type SlurmProject struct {
|
||||
ID, Name, TeamID, TeamName, OrgID, OrgName string
|
||||
}
|
||||
|
||||
// SlurmProjects are every project, with its team and organization.
|
||||
func (s *DB) SlurmProjects(ctx context.Context) ([]SlurmProject, error) {
|
||||
return queryAll(ctx, s.db, `
|
||||
SELECT p.id, p.name, t.id, t.name, o.id, o.name
|
||||
FROM projects p JOIN teams t ON t.id = p.team_id JOIN organizations o ON o.id = t.org_id
|
||||
ORDER BY o.name, t.name, p.name`, nil, func(rows *sql.Rows) (SlurmProject, error) {
|
||||
var p SlurmProject
|
||||
return p, rows.Scan(&p.ID, &p.Name, &p.TeamID, &p.TeamName, &p.OrgID, &p.OrgName)
|
||||
})
|
||||
}
|
||||
|
||||
// SlurmJob is a job that ended, as recorded: once, by its cluster's id.
|
||||
type SlurmJob struct {
|
||||
// Key is the cluster's name and the job's id: unique among jobs.
|
||||
Key string
|
||||
UserID, ProjectID string
|
||||
Login, Account, Partition string
|
||||
State string
|
||||
Start, End time.Time
|
||||
ElapsedSeconds, CPUs, GPUs int64
|
||||
CostMicros int64
|
||||
}
|
||||
|
||||
// RecordSlurmJobs records jobs not recorded before, each with a usage row
|
||||
// (model "slurm") so its cost is in the usage pages and the exports with
|
||||
// the requests'. Answers how many were new.
|
||||
func (s *DB) RecordSlurmJobs(ctx context.Context, jobs []SlurmJob) (int, error) {
|
||||
n := 0
|
||||
err := s.inTx(ctx, func(tx conn) error {
|
||||
var usage []Usage
|
||||
for _, j := range jobs {
|
||||
res, err := tx.ExecContext(ctx,
|
||||
`INSERT INTO slurm_jobs (job_key, user_id, project_id, login, account, partition, state,
|
||||
started_at, ended_at, elapsed_seconds, cpus, gpus, cost_micros)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT(job_key) DO NOTHING`,
|
||||
j.Key, j.UserID, j.ProjectID, j.Login, j.Account, j.Partition, j.State,
|
||||
millis(j.Start), millis(j.End), j.ElapsedSeconds, j.CPUs, j.GPUs, j.CostMicros)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if k, _ := res.RowsAffected(); k == 0 {
|
||||
continue
|
||||
}
|
||||
n++
|
||||
usage = append(usage, Usage{At: j.End, ProjectID: j.ProjectID, UserID: j.UserID, Model: "slurm",
|
||||
ServedModel: "slurm/" + j.Partition, CostMicros: j.CostMicros, Status: 200,
|
||||
LatencyMs: j.ElapsedSeconds * 1000, RequestID: "slurm-" + j.Key})
|
||||
}
|
||||
return insertUsage(ctx, tx, usage)
|
||||
})
|
||||
return n, err
|
||||
}
|
||||
@@ -0,0 +1,41 @@
|
||||
package store
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestASlurmJobIsRecordedOnce(t *testing.T) {
|
||||
s := capacityStore(t)
|
||||
ctx := t.Context()
|
||||
end := time.Now().Add(-time.Hour).Truncate(time.Second)
|
||||
j := SlurmJob{Key: "llmgw/41", ProjectID: "p1", UserID: "u1", Login: "ayse", Account: "test-arge-deneme",
|
||||
Partition: "gpu", State: "COMPLETED", Start: end.Add(-30 * time.Minute), End: end, ElapsedSeconds: 1800,
|
||||
CPUs: 4, GPUs: 1, CostMicros: 675000}
|
||||
for range 2 {
|
||||
n, err := s.RecordSlurmJobs(ctx, []SlurmJob{j})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
_ = n
|
||||
}
|
||||
byModel, err := s.UsageByModel(ctx, end.Add(-24*time.Hour))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var found bool
|
||||
for _, m := range byModel {
|
||||
if m.Model == "slurm/gpu" {
|
||||
found = true
|
||||
if m.Requests != 1 || m.CostMicros != 675000 {
|
||||
t.Errorf("recorded twice or wrong: %+v", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("no usage row: %+v", byModel)
|
||||
}
|
||||
if n, _ := s.RecordSlurmJobs(ctx, []SlurmJob{j}); n != 0 {
|
||||
t.Errorf("recorded again: %d", n)
|
||||
}
|
||||
}
|
||||
+30
-24
@@ -447,32 +447,38 @@ func (s *DB) WriteUsage(ctx context.Context, records []Usage) error {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
return s.inTx(ctx, func(tx conn) error {
|
||||
stmt, err := tx.PrepareContext(ctx,
|
||||
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
|
||||
request_id, replica, queue_ms, ttft_ms,
|
||||
guard_action, guard_policy, guard_category, guard_rule, guard_ms, end_user, tags, cache_hit, images, meshes)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return s.inTx(ctx, func(tx conn) error { return insertUsage(ctx, tx, records) })
|
||||
}
|
||||
|
||||
// insertUsage writes records, and their sums, in tx.
|
||||
func insertUsage(ctx context.Context, tx conn, records []Usage) error {
|
||||
if len(records) == 0 {
|
||||
return nil
|
||||
}
|
||||
stmt, err := tx.PrepareContext(ctx,
|
||||
`INSERT INTO usage (ts, key_id, project_id, model, served_model,
|
||||
prompt_tokens, completion_tokens, cost_micros, status, latency_ms, user_id,
|
||||
request_id, replica, queue_ms, ttft_ms,
|
||||
guard_action, guard_policy, guard_category, guard_rule, guard_ms, end_user, tags, cache_hit, images, meshes)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
for _, r := range records {
|
||||
hit := 0
|
||||
if r.CacheHit {
|
||||
hit = 1
|
||||
}
|
||||
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
|
||||
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
|
||||
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
|
||||
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardRule, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit, r.Images, r.Meshes); err != nil {
|
||||
return err
|
||||
}
|
||||
defer stmt.Close()
|
||||
for _, r := range records {
|
||||
hit := 0
|
||||
if r.CacheHit {
|
||||
hit = 1
|
||||
}
|
||||
if _, err := stmt.ExecContext(ctx, millis(r.At), r.KeyID, r.ProjectID,
|
||||
r.Model, r.ServedModel, r.PromptTokens, r.CompletionTokens,
|
||||
r.CostMicros, r.Status, r.LatencyMs, r.UserID,
|
||||
r.RequestID, r.Replica, r.QueueMs, r.TTFTMs,
|
||||
r.GuardAction, r.GuardPolicy, r.GuardCategory, r.GuardRule, r.GuardMs, r.EndUser, tagsJSON(r.Tags), hit, r.Images, r.Meshes); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return writeRollup(ctx, tx, records)
|
||||
})
|
||||
}
|
||||
return writeRollup(ctx, tx, records)
|
||||
}
|
||||
|
||||
// SpendSince totals cost per key since a moment.
|
||||
|
||||
Reference in new issue
Block a user