Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s
CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8547528fb7
commit
fcda4d3f85
22 files changed
+1390
-76
No files matched your search
@@ -0,0 +1,214 @@
|
||||
package slurmnode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"slices"
|
||||
"strings"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
)
|
||||
|
||||
// accountsMarker lists what llmgw made in Slurm's accounting, a line each:
|
||||
// "account NAME" and "user LOGIN NAME". Nothing else is changed or taken
|
||||
// away: an account a Slurm administrator made is theirs.
|
||||
func (k *Keeper) accountsMarker() string { return k.marker + "-accounts" }
|
||||
|
||||
// maxEnded is the most ended jobs sent at once: a day's on a busy cluster
|
||||
// would otherwise be a status message of megabytes.
|
||||
const maxEnded = 2000
|
||||
|
||||
// accounts keeps Slurm's accounts and their users as want says, on the
|
||||
// controller.
|
||||
func (k *Keeper) accounts(ctx context.Context, want *llmgwv1.Slurm) error {
|
||||
made := k.madeAccounts()
|
||||
if !want.GetManageAccounts() {
|
||||
if len(made) == 0 {
|
||||
return nil
|
||||
}
|
||||
return k.applyAccounts(ctx, nil, made)
|
||||
}
|
||||
return k.applyAccounts(ctx, want.GetAccounts(), made)
|
||||
}
|
||||
|
||||
func (k *Keeper) madeAccounts() []string {
|
||||
b, err := k.host.ReadFile(k.accountsMarker())
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, l := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
||||
if l != "" {
|
||||
out = append(out, l)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func (k *Keeper) sacctmgr(ctx context.Context, args ...string) error {
|
||||
out, err := k.host.Run(ctx, nil, "sacctmgr", append([]string{"-i"}, args...)...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("sacctmgr %s: %v: %s", strings.Join(args, " "), err, lastLine(out))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (k *Keeper) applyAccounts(ctx context.Context, want []*llmgwv1.SlurmAccount, made []string) error {
|
||||
out, err := k.host.Run(ctx, nil, "sacctmgr", slurm.AssocFormat...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("reading Slurm's accounts: %v: %s", err, lastLine(out))
|
||||
}
|
||||
have := map[string]bool{} // "account NAME", "user LOGIN NAME"
|
||||
for _, a := range slurm.ParseAssociations(string(out)) {
|
||||
if a.User == "" {
|
||||
have["account "+a.Account] = true
|
||||
} else {
|
||||
have["user "+a.User+" "+a.Account] = true
|
||||
}
|
||||
}
|
||||
keep := map[string]bool{}
|
||||
var errs []error
|
||||
ours := slices.Clone(made)
|
||||
note := func(line string) {
|
||||
if !slices.Contains(ours, line) {
|
||||
ours = append(ours, line)
|
||||
}
|
||||
}
|
||||
for _, a := range want {
|
||||
name, parent := a.GetName(), a.GetParent()
|
||||
if !validAccount(name) || (parent != "root" && !validAccount(parent)) {
|
||||
errs = append(errs, fmt.Errorf("the leader sent an account %q under %q", name, parent))
|
||||
continue
|
||||
}
|
||||
line := "account " + name
|
||||
keep[line] = true
|
||||
if !have[line] {
|
||||
k.log.Info("a Slurm account is made, as llmgw's projects say", "account", name, "parent", parent)
|
||||
if err := k.sacctmgr(ctx, "add", "account", name, "parent="+parent, "description=llmgw"); err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
note(line)
|
||||
}
|
||||
for _, u := range a.GetUsers() {
|
||||
if !slurm.ValidLogin(u) {
|
||||
continue
|
||||
}
|
||||
ul := "user " + u + " " + name
|
||||
keep[ul] = true
|
||||
if have[ul] {
|
||||
continue
|
||||
}
|
||||
if err := k.sacctmgr(ctx, "add", "user", u, "account="+name); err != nil {
|
||||
errs = append(errs, err)
|
||||
continue
|
||||
}
|
||||
note(ul)
|
||||
}
|
||||
}
|
||||
// Taken away: what llmgw made and no longer wants, users before their
|
||||
// accounts, children before their parents (made later, listed later).
|
||||
var left []string
|
||||
for i := len(ours) - 1; i >= 0; i-- {
|
||||
line := ours[i]
|
||||
if keep[line] || !strings.HasPrefix(line, "user ") {
|
||||
continue
|
||||
}
|
||||
f := strings.Fields(line)
|
||||
if !have[line] {
|
||||
continue
|
||||
}
|
||||
if err := k.removeAssoc(ctx, f[1], f[2], have, keep); err != nil {
|
||||
errs = append(errs, err)
|
||||
left = append(left, line)
|
||||
}
|
||||
}
|
||||
for i := len(ours) - 1; i >= 0; i-- {
|
||||
line := ours[i]
|
||||
if keep[line] || !strings.HasPrefix(line, "account ") {
|
||||
continue
|
||||
}
|
||||
if have[line] {
|
||||
k.log.Info("a Slurm account llmgw made is taken away; its jobs' records stay", "account", line[len("account "):])
|
||||
if err := k.sacctmgr(ctx, "remove", "account", line[len("account "):]); err != nil {
|
||||
errs = append(errs, err)
|
||||
left = append(left, line)
|
||||
}
|
||||
}
|
||||
}
|
||||
var lines []string
|
||||
for _, l := range ours {
|
||||
if keep[l] || slices.Contains(left, l) {
|
||||
lines = append(lines, l)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(lines, made) {
|
||||
var err error
|
||||
if len(lines) == 0 {
|
||||
err = k.host.Remove(k.accountsMarker())
|
||||
} else {
|
||||
_, err = k.write(k.accountsMarker(), []byte(strings.Join(lines, "\n")+"\n"), 0o644, "")
|
||||
}
|
||||
errs = append(errs, err)
|
||||
}
|
||||
return errors.Join(errs...)
|
||||
}
|
||||
|
||||
func validAccount(s string) bool {
|
||||
return s != "" && len(s) <= 64 && strings.Trim(s, "abcdefghijklmnopqrstuvwxyz0123456789_-") == ""
|
||||
}
|
||||
|
||||
// ended are the jobs that ended in the last day, newest last.
|
||||
func (k *Keeper) ended(ctx context.Context) ([]*llmgwv1.SlurmJob, error) {
|
||||
out, err := k.host.Run(ctx, nil, "sacct", slurm.SacctArgs...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("sacct: %v: %s", err, lastLine(out))
|
||||
}
|
||||
jobs, err := slurm.ParseJobs(string(out))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(jobs) > maxEnded {
|
||||
jobs = jobs[len(jobs)-maxEnded:]
|
||||
}
|
||||
ps := make([]*llmgwv1.SlurmJob, 0, len(jobs))
|
||||
for _, j := range jobs {
|
||||
ps = append(ps, &llmgwv1.SlurmJob{Id: j.ID, User: j.User, Account: j.Account, Partition: j.Partition,
|
||||
State: j.State, StartUnix: j.Start.Unix(), EndUnix: j.End.Unix(), ElapsedSeconds: int64(j.Elapsed.Seconds()),
|
||||
Cpus: int32(j.CPUs), Gpus: int32(j.GPUs)}) //nolint:gosec // counts of a node's
|
||||
}
|
||||
return ps, nil
|
||||
}
|
||||
|
||||
// removeAssoc takes login off account. Slurm refuses to take away a
|
||||
// user's default account, so another of theirs is made the default first,
|
||||
// one llmgw keeps if there is one; with none left the user goes from the
|
||||
// accounting entirely.
|
||||
func (k *Keeper) removeAssoc(ctx context.Context, login, account string, have, keep map[string]bool) error {
|
||||
var others []string
|
||||
for line := range have {
|
||||
f := strings.Fields(line)
|
||||
if len(f) == 3 && f[0] == "user" && f[1] == login && f[2] != account {
|
||||
others = append(others, f[2])
|
||||
}
|
||||
}
|
||||
if len(others) == 0 {
|
||||
return k.sacctmgr(ctx, "remove", "user", login)
|
||||
}
|
||||
slices.SortFunc(others, func(a, b string) int {
|
||||
ka, kb := keep["user "+login+" "+a], keep["user "+login+" "+b]
|
||||
switch {
|
||||
case ka && !kb:
|
||||
return -1
|
||||
case kb && !ka:
|
||||
return 1
|
||||
}
|
||||
return strings.Compare(a, b)
|
||||
})
|
||||
if err := k.sacctmgr(ctx, "modify", "user", login, "set", "defaultaccount="+others[0]); err != nil {
|
||||
return err
|
||||
}
|
||||
return k.sacctmgr(ctx, "remove", "user", login, "account="+account)
|
||||
}
|
||||
@@ -0,0 +1,83 @@
|
||||
package slurmnode
|
||||
|
||||
import (
|
||||
"context"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
)
|
||||
|
||||
func TestAccountsMadeAndTakenAwayOnlyIfLlmgwsOwn(t *testing.T) {
|
||||
h := newFake()
|
||||
k := New(h, nil, "/var/lib/llmgw-agent/slurm", nil)
|
||||
assoc := "sacctmgr " + strings.Join(slurm.AssocFormat, " ")
|
||||
// root, and an account a Slurm administrator made.
|
||||
h.outputs[assoc] = "root||\nroot|root|\nphysics||root\n"
|
||||
m := &llmgwv1.Slurm{ManageAccounts: true, Accounts: []*llmgwv1.SlurmAccount{
|
||||
{Name: "test", Parent: "root"}, {Name: "test-arge", Parent: "test"},
|
||||
{Name: "test-arge-deneme", Parent: "test-arge", Users: []string{"ada", "ayse"}},
|
||||
}}
|
||||
if err := k.accounts(context.Background(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, want := range []string{"sacctmgr -i add account test parent=root description=llmgw",
|
||||
"sacctmgr -i add account test-arge-deneme parent=test-arge description=llmgw",
|
||||
"sacctmgr -i add user ayse account=test-arge-deneme"} {
|
||||
if !slices.Contains(h.ran, want) {
|
||||
t.Errorf("did not run %q:\n%s", want, strings.Join(h.ran, "\n"))
|
||||
}
|
||||
}
|
||||
if !strings.Contains(string(h.files[k.accountsMarker()]), "user ada test-arge-deneme") {
|
||||
t.Errorf("marker %q", h.files[k.accountsMarker()])
|
||||
}
|
||||
|
||||
// Now there; ayse leaves the project, and the team goes.
|
||||
h.outputs[assoc] = "root||\nphysics||root\ntest||root\ntest-arge||test\ntest-arge-deneme||test-arge\n" +
|
||||
"test-arge-deneme|ada|test-arge\ntest-arge-deneme|ayse|test-arge\n"
|
||||
h.ran = nil
|
||||
m.Accounts = []*llmgwv1.SlurmAccount{{Name: "test", Parent: "root"}}
|
||||
if err := k.accounts(context.Background(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Neither has another account: they go from the accounting.
|
||||
want := []string{"sacctmgr -i remove user ayse", "sacctmgr -i remove user ada",
|
||||
"sacctmgr -i remove account test-arge-deneme", "sacctmgr -i remove account test-arge"}
|
||||
var got []string
|
||||
for _, c := range h.ran {
|
||||
if strings.Contains(c, "remove") || strings.Contains(c, " add ") {
|
||||
got = append(got, c)
|
||||
}
|
||||
}
|
||||
if !slices.Equal(got, want) {
|
||||
t.Errorf("ran:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n"))
|
||||
}
|
||||
if slices.ContainsFunc(h.ran, func(c string) bool { return strings.Contains(c, "physics") }) {
|
||||
t.Error("touched an account llmgw did not make")
|
||||
}
|
||||
|
||||
// One still on another account: that one is made the default first.
|
||||
h.outputs[assoc] = "root||\nphysics||root\nphysics|ada|\ntest||root\nx||test\nx|ada|\n"
|
||||
h.files[k.accountsMarker()] = []byte("account test\naccount x\nuser ada x\n")
|
||||
h.ran = nil
|
||||
m.Accounts = []*llmgwv1.SlurmAccount{{Name: "test", Parent: "root"}}
|
||||
if err := k.accounts(context.Background(), m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if i, j := slices.Index(h.ran, "sacctmgr -i modify user ada set defaultaccount=physics"),
|
||||
slices.Index(h.ran, "sacctmgr -i remove user ada account=x"); i < 0 || j < i {
|
||||
t.Errorf("ran:\n%s", strings.Join(h.ran, "\n"))
|
||||
}
|
||||
|
||||
// Off: what llmgw made goes.
|
||||
h.outputs[assoc] = "root||\nphysics||root\ntest||root\n"
|
||||
h.ran = nil
|
||||
if err := k.accounts(context.Background(), &llmgwv1.Slurm{}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !slices.Contains(h.ran, "sacctmgr -i remove account test") || h.files[k.accountsMarker()] != nil {
|
||||
t.Errorf("not taken away:\n%s", strings.Join(h.ran, "\n"))
|
||||
}
|
||||
}
|
||||
@@ -267,6 +267,20 @@ func (k *Keeper) Reconcile(ctx context.Context, want *llmgwv1.Slurm) *llmgwv1.Sl
|
||||
return fail("slurmctld", err)
|
||||
}
|
||||
st.SlurmctldRunning = k.active(ctx, "slurmctld")
|
||||
// The accounts, and the jobs that ended, need the accounting.
|
||||
if st.SlurmdbdRunning && st.SlurmctldRunning {
|
||||
if err := k.accounts(ctx, want); err != nil {
|
||||
k.log.Warn("keeping Slurm's accounts", "err", err)
|
||||
if acctErr == nil {
|
||||
acctErr = fmt.Errorf("accounts: %w", err)
|
||||
}
|
||||
}
|
||||
if ended, err := k.ended(ctx); err != nil {
|
||||
k.log.Warn("reading the jobs that ended", "err", err)
|
||||
} else {
|
||||
st.Ended = ended
|
||||
}
|
||||
}
|
||||
}
|
||||
if want.GetNode() {
|
||||
if err := k.node(ctx, want, st); err != nil {
|
||||
|
||||
Reference in new issue
Block a user