CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
215 lines
6.3 KiB
Go
215 lines
6.3 KiB
Go
package slurmnode
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"slices"
|
|
"strings"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
|
)
|
|
|
|
// accountsMarker lists what llmgw made in Slurm's accounting, a line each:
|
|
// "account NAME" and "user LOGIN NAME". Nothing else is changed or taken
|
|
// away: an account a Slurm administrator made is theirs.
|
|
func (k *Keeper) accountsMarker() string { return k.marker + "-accounts" }
|
|
|
|
// maxEnded is the most ended jobs sent at once: a day's on a busy cluster
|
|
// would otherwise be a status message of megabytes.
|
|
const maxEnded = 2000
|
|
|
|
// accounts keeps Slurm's accounts and their users as want says, on the
|
|
// controller.
|
|
func (k *Keeper) accounts(ctx context.Context, want *llmgwv1.Slurm) error {
|
|
made := k.madeAccounts()
|
|
if !want.GetManageAccounts() {
|
|
if len(made) == 0 {
|
|
return nil
|
|
}
|
|
return k.applyAccounts(ctx, nil, made)
|
|
}
|
|
return k.applyAccounts(ctx, want.GetAccounts(), made)
|
|
}
|
|
|
|
func (k *Keeper) madeAccounts() []string {
|
|
b, err := k.host.ReadFile(k.accountsMarker())
|
|
if err != nil {
|
|
return nil
|
|
}
|
|
var out []string
|
|
for _, l := range strings.Split(strings.TrimSpace(string(b)), "\n") {
|
|
if l != "" {
|
|
out = append(out, l)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
func (k *Keeper) sacctmgr(ctx context.Context, args ...string) error {
|
|
out, err := k.host.Run(ctx, nil, "sacctmgr", append([]string{"-i"}, args...)...)
|
|
if err != nil {
|
|
return fmt.Errorf("sacctmgr %s: %v: %s", strings.Join(args, " "), err, lastLine(out))
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (k *Keeper) applyAccounts(ctx context.Context, want []*llmgwv1.SlurmAccount, made []string) error {
|
|
out, err := k.host.Run(ctx, nil, "sacctmgr", slurm.AssocFormat...)
|
|
if err != nil {
|
|
return fmt.Errorf("reading Slurm's accounts: %v: %s", err, lastLine(out))
|
|
}
|
|
have := map[string]bool{} // "account NAME", "user LOGIN NAME"
|
|
for _, a := range slurm.ParseAssociations(string(out)) {
|
|
if a.User == "" {
|
|
have["account "+a.Account] = true
|
|
} else {
|
|
have["user "+a.User+" "+a.Account] = true
|
|
}
|
|
}
|
|
keep := map[string]bool{}
|
|
var errs []error
|
|
ours := slices.Clone(made)
|
|
note := func(line string) {
|
|
if !slices.Contains(ours, line) {
|
|
ours = append(ours, line)
|
|
}
|
|
}
|
|
for _, a := range want {
|
|
name, parent := a.GetName(), a.GetParent()
|
|
if !validAccount(name) || (parent != "root" && !validAccount(parent)) {
|
|
errs = append(errs, fmt.Errorf("the leader sent an account %q under %q", name, parent))
|
|
continue
|
|
}
|
|
line := "account " + name
|
|
keep[line] = true
|
|
if !have[line] {
|
|
k.log.Info("a Slurm account is made, as llmgw's projects say", "account", name, "parent", parent)
|
|
if err := k.sacctmgr(ctx, "add", "account", name, "parent="+parent, "description=llmgw"); err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
note(line)
|
|
}
|
|
for _, u := range a.GetUsers() {
|
|
if !slurm.ValidLogin(u) {
|
|
continue
|
|
}
|
|
ul := "user " + u + " " + name
|
|
keep[ul] = true
|
|
if have[ul] {
|
|
continue
|
|
}
|
|
if err := k.sacctmgr(ctx, "add", "user", u, "account="+name); err != nil {
|
|
errs = append(errs, err)
|
|
continue
|
|
}
|
|
note(ul)
|
|
}
|
|
}
|
|
// Taken away: what llmgw made and no longer wants, users before their
|
|
// accounts, children before their parents (made later, listed later).
|
|
var left []string
|
|
for i := len(ours) - 1; i >= 0; i-- {
|
|
line := ours[i]
|
|
if keep[line] || !strings.HasPrefix(line, "user ") {
|
|
continue
|
|
}
|
|
f := strings.Fields(line)
|
|
if !have[line] {
|
|
continue
|
|
}
|
|
if err := k.removeAssoc(ctx, f[1], f[2], have, keep); err != nil {
|
|
errs = append(errs, err)
|
|
left = append(left, line)
|
|
}
|
|
}
|
|
for i := len(ours) - 1; i >= 0; i-- {
|
|
line := ours[i]
|
|
if keep[line] || !strings.HasPrefix(line, "account ") {
|
|
continue
|
|
}
|
|
if have[line] {
|
|
k.log.Info("a Slurm account llmgw made is taken away; its jobs' records stay", "account", line[len("account "):])
|
|
if err := k.sacctmgr(ctx, "remove", "account", line[len("account "):]); err != nil {
|
|
errs = append(errs, err)
|
|
left = append(left, line)
|
|
}
|
|
}
|
|
}
|
|
var lines []string
|
|
for _, l := range ours {
|
|
if keep[l] || slices.Contains(left, l) {
|
|
lines = append(lines, l)
|
|
}
|
|
}
|
|
if !slices.Equal(lines, made) {
|
|
var err error
|
|
if len(lines) == 0 {
|
|
err = k.host.Remove(k.accountsMarker())
|
|
} else {
|
|
_, err = k.write(k.accountsMarker(), []byte(strings.Join(lines, "\n")+"\n"), 0o644, "")
|
|
}
|
|
errs = append(errs, err)
|
|
}
|
|
return errors.Join(errs...)
|
|
}
|
|
|
|
func validAccount(s string) bool {
|
|
return s != "" && len(s) <= 64 && strings.Trim(s, "abcdefghijklmnopqrstuvwxyz0123456789_-") == ""
|
|
}
|
|
|
|
// ended are the jobs that ended in the last day, newest last.
|
|
func (k *Keeper) ended(ctx context.Context) ([]*llmgwv1.SlurmJob, error) {
|
|
out, err := k.host.Run(ctx, nil, "sacct", slurm.SacctArgs...)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("sacct: %v: %s", err, lastLine(out))
|
|
}
|
|
jobs, err := slurm.ParseJobs(string(out))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
if len(jobs) > maxEnded {
|
|
jobs = jobs[len(jobs)-maxEnded:]
|
|
}
|
|
ps := make([]*llmgwv1.SlurmJob, 0, len(jobs))
|
|
for _, j := range jobs {
|
|
ps = append(ps, &llmgwv1.SlurmJob{Id: j.ID, User: j.User, Account: j.Account, Partition: j.Partition,
|
|
State: j.State, StartUnix: j.Start.Unix(), EndUnix: j.End.Unix(), ElapsedSeconds: int64(j.Elapsed.Seconds()),
|
|
Cpus: int32(j.CPUs), Gpus: int32(j.GPUs)}) //nolint:gosec // counts of a node's
|
|
}
|
|
return ps, nil
|
|
}
|
|
|
|
// removeAssoc takes login off account. Slurm refuses to take away a
|
|
// user's default account, so another of theirs is made the default first,
|
|
// one llmgw keeps if there is one; with none left the user goes from the
|
|
// accounting entirely.
|
|
func (k *Keeper) removeAssoc(ctx context.Context, login, account string, have, keep map[string]bool) error {
|
|
var others []string
|
|
for line := range have {
|
|
f := strings.Fields(line)
|
|
if len(f) == 3 && f[0] == "user" && f[1] == login && f[2] != account {
|
|
others = append(others, f[2])
|
|
}
|
|
}
|
|
if len(others) == 0 {
|
|
return k.sacctmgr(ctx, "remove", "user", login)
|
|
}
|
|
slices.SortFunc(others, func(a, b string) int {
|
|
ka, kb := keep["user "+login+" "+a], keep["user "+login+" "+b]
|
|
switch {
|
|
case ka && !kb:
|
|
return -1
|
|
case kb && !ka:
|
|
return 1
|
|
}
|
|
return strings.Compare(a, b)
|
|
})
|
|
if err := k.sacctmgr(ctx, "modify", "user", login, "set", "defaultaccount="+others[0]); err != nil {
|
|
return err
|
|
}
|
|
return k.sacctmgr(ctx, "remove", "user", login, "account="+account)
|
|
}
|