Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s

With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-15 11:54:01 +01:00
1 parent 8547528fb7
commit fcda4d3f85
22 files changed
+1390 -76

No files matched your search

+214
View File
@@ -0,0 +1,214 @@
package slurmnode
import (
"context"
"errors"
"fmt"
"slices"
"strings"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// accountsMarker lists what llmgw made in Slurm's accounting, a line each:
// "account NAME" and "user LOGIN NAME". Nothing else is changed or taken
// away: an account a Slurm administrator made is theirs.
func (k *Keeper) accountsMarker() string { return k.marker + "-accounts" }
// maxEnded is the most ended jobs sent at once: a day's on a busy cluster
// would otherwise be a status message of megabytes.
const maxEnded = 2000
// accounts keeps Slurm's accounts and their users as want says, on the
// controller.
func (k *Keeper) accounts(ctx context.Context, want *llmgwv1.Slurm) error {
made := k.madeAccounts()
if !want.GetManageAccounts() {
if len(made) == 0 {
return nil
}
return k.applyAccounts(ctx, nil, made)
}
return k.applyAccounts(ctx, want.GetAccounts(), made)
}
func (k *Keeper) madeAccounts() []string {
b, err := k.host.ReadFile(k.accountsMarker())
if err != nil {
return nil
}
var out []string
for _, l := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if l != "" {
out = append(out, l)
}
}
return out
}
func (k *Keeper) sacctmgr(ctx context.Context, args ...string) error {
out, err := k.host.Run(ctx, nil, "sacctmgr", append([]string{"-i"}, args...)...)
if err != nil {
return fmt.Errorf("sacctmgr %s: %v: %s", strings.Join(args, " "), err, lastLine(out))
}
return nil
}
func (k *Keeper) applyAccounts(ctx context.Context, want []*llmgwv1.SlurmAccount, made []string) error {
out, err := k.host.Run(ctx, nil, "sacctmgr", slurm.AssocFormat...)
if err != nil {
return fmt.Errorf("reading Slurm's accounts: %v: %s", err, lastLine(out))
}
have := map[string]bool{} // "account NAME", "user LOGIN NAME"
for _, a := range slurm.ParseAssociations(string(out)) {
if a.User == "" {
have["account "+a.Account] = true
} else {
have["user "+a.User+" "+a.Account] = true
}
}
keep := map[string]bool{}
var errs []error
ours := slices.Clone(made)
note := func(line string) {
if !slices.Contains(ours, line) {
ours = append(ours, line)
}
}
for _, a := range want {
name, parent := a.GetName(), a.GetParent()
if !validAccount(name) || (parent != "root" && !validAccount(parent)) {
errs = append(errs, fmt.Errorf("the leader sent an account %q under %q", name, parent))
continue
}
line := "account " + name
keep[line] = true
if !have[line] {
k.log.Info("a Slurm account is made, as llmgw's projects say", "account", name, "parent", parent)
if err := k.sacctmgr(ctx, "add", "account", name, "parent="+parent, "description=llmgw"); err != nil {
errs = append(errs, err)
continue
}
note(line)
}
for _, u := range a.GetUsers() {
if !slurm.ValidLogin(u) {
continue
}
ul := "user " + u + " " + name
keep[ul] = true
if have[ul] {
continue
}
if err := k.sacctmgr(ctx, "add", "user", u, "account="+name); err != nil {
errs = append(errs, err)
continue
}
note(ul)
}
}
// Taken away: what llmgw made and no longer wants, users before their
// accounts, children before their parents (made later, listed later).
var left []string
for i := len(ours) - 1; i >= 0; i-- {
line := ours[i]
if keep[line] || !strings.HasPrefix(line, "user ") {
continue
}
f := strings.Fields(line)
if !have[line] {
continue
}
if err := k.removeAssoc(ctx, f[1], f[2], have, keep); err != nil {
errs = append(errs, err)
left = append(left, line)
}
}
for i := len(ours) - 1; i >= 0; i-- {
line := ours[i]
if keep[line] || !strings.HasPrefix(line, "account ") {
continue
}
if have[line] {
k.log.Info("a Slurm account llmgw made is taken away; its jobs' records stay", "account", line[len("account "):])
if err := k.sacctmgr(ctx, "remove", "account", line[len("account "):]); err != nil {
errs = append(errs, err)
left = append(left, line)
}
}
}
var lines []string
for _, l := range ours {
if keep[l] || slices.Contains(left, l) {
lines = append(lines, l)
}
}
if !slices.Equal(lines, made) {
var err error
if len(lines) == 0 {
err = k.host.Remove(k.accountsMarker())
} else {
_, err = k.write(k.accountsMarker(), []byte(strings.Join(lines, "\n")+"\n"), 0o644, "")
}
errs = append(errs, err)
}
return errors.Join(errs...)
}
func validAccount(s string) bool {
return s != "" && len(s) <= 64 && strings.Trim(s, "abcdefghijklmnopqrstuvwxyz0123456789_-") == ""
}
// ended are the jobs that ended in the last day, newest last.
func (k *Keeper) ended(ctx context.Context) ([]*llmgwv1.SlurmJob, error) {
out, err := k.host.Run(ctx, nil, "sacct", slurm.SacctArgs...)
if err != nil {
return nil, fmt.Errorf("sacct: %v: %s", err, lastLine(out))
}
jobs, err := slurm.ParseJobs(string(out))
if err != nil {
return nil, err
}
if len(jobs) > maxEnded {
jobs = jobs[len(jobs)-maxEnded:]
}
ps := make([]*llmgwv1.SlurmJob, 0, len(jobs))
for _, j := range jobs {
ps = append(ps, &llmgwv1.SlurmJob{Id: j.ID, User: j.User, Account: j.Account, Partition: j.Partition,
State: j.State, StartUnix: j.Start.Unix(), EndUnix: j.End.Unix(), ElapsedSeconds: int64(j.Elapsed.Seconds()),
Cpus: int32(j.CPUs), Gpus: int32(j.GPUs)}) //nolint:gosec // counts of a node's
}
return ps, nil
}
// removeAssoc takes login off account. Slurm refuses to take away a
// user's default account, so another of theirs is made the default first,
// one llmgw keeps if there is one; with none left the user goes from the
// accounting entirely.
func (k *Keeper) removeAssoc(ctx context.Context, login, account string, have, keep map[string]bool) error {
var others []string
for line := range have {
f := strings.Fields(line)
if len(f) == 3 && f[0] == "user" && f[1] == login && f[2] != account {
others = append(others, f[2])
}
}
if len(others) == 0 {
return k.sacctmgr(ctx, "remove", "user", login)
}
slices.SortFunc(others, func(a, b string) int {
ka, kb := keep["user "+login+" "+a], keep["user "+login+" "+b]
switch {
case ka && !kb:
return -1
case kb && !ka:
return 1
}
return strings.Compare(a, b)
})
if err := k.sacctmgr(ctx, "modify", "user", login, "set", "defaultaccount="+others[0]); err != nil {
return err
}
return k.sacctmgr(ctx, "remove", "user", login, "account="+account)
}
+83
View File
@@ -0,0 +1,83 @@
package slurmnode
import (
"context"
"slices"
"strings"
"testing"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
func TestAccountsMadeAndTakenAwayOnlyIfLlmgwsOwn(t *testing.T) {
h := newFake()
k := New(h, nil, "/var/lib/llmgw-agent/slurm", nil)
assoc := "sacctmgr " + strings.Join(slurm.AssocFormat, " ")
// root, and an account a Slurm administrator made.
h.outputs[assoc] = "root||\nroot|root|\nphysics||root\n"
m := &llmgwv1.Slurm{ManageAccounts: true, Accounts: []*llmgwv1.SlurmAccount{
{Name: "test", Parent: "root"}, {Name: "test-arge", Parent: "test"},
{Name: "test-arge-deneme", Parent: "test-arge", Users: []string{"ada", "ayse"}},
}}
if err := k.accounts(context.Background(), m); err != nil {
t.Fatal(err)
}
for _, want := range []string{"sacctmgr -i add account test parent=root description=llmgw",
"sacctmgr -i add account test-arge-deneme parent=test-arge description=llmgw",
"sacctmgr -i add user ayse account=test-arge-deneme"} {
if !slices.Contains(h.ran, want) {
t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n"))
}
}
if !strings.Contains(string(h.files[k.accountsMarker()]), "user ada test-arge-deneme") {
t.Errorf("marker %q", h.files[k.accountsMarker()])
}
// Now there; ayse leaves the project, and the team goes.
h.outputs[assoc] = "root||\nphysics||root\ntest||root\ntest-arge||test\ntest-arge-deneme||test-arge\n" +
"test-arge-deneme|ada|test-arge\ntest-arge-deneme|ayse|test-arge\n"
h.ran = nil
m.Accounts = []*llmgwv1.SlurmAccount{{Name: "test", Parent: "root"}}
if err := k.accounts(context.Background(), m); err != nil {
t.Fatal(err)
}
// Neither has another account: they go from the accounting.
want := []string{"sacctmgr -i remove user ayse", "sacctmgr -i remove user ada",
"sacctmgr -i remove account test-arge-deneme", "sacctmgr -i remove account test-arge"}
var got []string
for _, c := range h.ran {
if strings.Contains(c, "remove") || strings.Contains(c, " add ") {
got = append(got, c)
}
}
if !slices.Equal(got, want) {
t.Errorf("ran:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
}
if slices.ContainsFunc(h.ran, func(c string) bool { return strings.Contains(c, "physics") }) {
t.Error("touched an account llmgw did not make")
}
// One still on another account: that one is made the default first.
h.outputs[assoc] = "root||\nphysics||root\nphysics|ada|\ntest||root\nx||test\nx|ada|\n"
h.files[k.accountsMarker()] = []byte("account test\naccount x\nuser ada x\n")
h.ran = nil
m.Accounts = []*llmgwv1.SlurmAccount{{Name: "test", Parent: "root"}}
if err := k.accounts(context.Background(), m); err != nil {
t.Fatal(err)
}
if i, j := slices.Index(h.ran, "sacctmgr -i modify user ada set defaultaccount=physics"),
slices.Index(h.ran, "sacctmgr -i remove user ada account=x"); i < 0 || j < i {
t.Errorf("ran:\n%s", strings.Join(h.ran, "\n"))
}
// Off: what llmgw made goes.
h.outputs[assoc] = "root||\nphysics||root\ntest||root\n"
h.ran = nil
if err := k.accounts(context.Background(), &llmgwv1.Slurm{}); err != nil {
t.Fatal(err)
}
if !slices.Contains(h.ran, "sacctmgr -i remove account test") || h.files[k.accountsMarker()] != nil {
t.Errorf("not taken away:\n%s", strings.Join(h.ran, "\n"))
}
}
+14
View File
@@ -267,6 +267,20 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
return fail("slurmctld", err)
}
st.SlurmctldRunning = k.active(ctx, "slurmctld")
// The accounts, and the jobs that ended, need the accounting.
if st.SlurmdbdRunning && st.SlurmctldRunning {
if err := k.accounts(ctx, want); err != nil {
k.log.Warn("keeping Slurm's accounts", "err", err)
if acctErr == nil {
acctErr = fmt.Errorf("accounts: %w", err)
}
}
if ended, err := k.ended(ctx); err != nil {
k.log.Warn("reading the jobs that ended", "err", err)
} else {
st.Ended = ended
}
}
}
if want.GetNode() {
if err := k.node(ctx, want, st); err != nil {