Files
llmgw/internal/agent/slurmnode/accounts.go
T
Ilker ManapandClaude Opus 5.5 fcda4d3f85
CI / test (push) Failing after 7m24s
Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-11-15 11:54:01 +01:00

215 lines
6.3 KiB
Go

package slurmnode
import (
"context"
"errors"
"fmt"
"slices"
"strings"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
// accountsMarker lists what llmgw made in Slurm's accounting, a line each:
// "account NAME" and "user LOGIN NAME". Nothing else is changed or taken
// away: an account a Slurm administrator made is theirs.
func (k *Keeper) accountsMarker() string { return k.marker + "-accounts" }
// maxEnded is the most ended jobs sent at once: a day's on a busy cluster
// would otherwise be a status message of megabytes.
const maxEnded = 2000
// accounts keeps Slurm's accounts and their users as want says, on the
// controller.
func (k *Keeper) accounts(ctx context.Context, want *llmgwv1.Slurm) error {
made := k.madeAccounts()
if !want.GetManageAccounts() {
if len(made) == 0 {
return nil
}
return k.applyAccounts(ctx, nil, made)
}
return k.applyAccounts(ctx, want.GetAccounts(), made)
}
func (k *Keeper) madeAccounts() []string {
b, err := k.host.ReadFile(k.accountsMarker())
if err != nil {
return nil
}
var out []string
for _, l := range strings.Split(strings.TrimSpace(string(b)), "\n") {
if l != "" {
out = append(out, l)
}
}
return out
}
func (k *Keeper) sacctmgr(ctx context.Context, args ...string) error {
out, err := k.host.Run(ctx, nil, "sacctmgr", append([]string{"-i"}, args...)...)
if err != nil {
return fmt.Errorf("sacctmgr %s: %v: %s", strings.Join(args, " "), err, lastLine(out))
}
return nil
}
func (k *Keeper) applyAccounts(ctx context.Context, want []*llmgwv1.SlurmAccount, made []string) error {
out, err := k.host.Run(ctx, nil, "sacctmgr", slurm.AssocFormat...)
if err != nil {
return fmt.Errorf("reading Slurm's accounts: %v: %s", err, lastLine(out))
}
have := map[string]bool{} // "account NAME", "user LOGIN NAME"
for _, a := range slurm.ParseAssociations(string(out)) {
if a.User == "" {
have["account "+a.Account] = true
} else {
have["user "+a.User+" "+a.Account] = true
}
}
keep := map[string]bool{}
var errs []error
ours := slices.Clone(made)
note := func(line string) {
if !slices.Contains(ours, line) {
ours = append(ours, line)
}
}
for _, a := range want {
name, parent := a.GetName(), a.GetParent()
if !validAccount(name) || (parent != "root" && !validAccount(parent)) {
errs = append(errs, fmt.Errorf("the leader sent an account %q under %q", name, parent))
continue
}
line := "account " + name
keep[line] = true
if !have[line] {
k.log.Info("a Slurm account is made, as llmgw's projects say", "account", name, "parent", parent)
if err := k.sacctmgr(ctx, "add", "account", name, "parent="+parent, "description=llmgw"); err != nil {
errs = append(errs, err)
continue
}
note(line)
}
for _, u := range a.GetUsers() {
if !slurm.ValidLogin(u) {
continue
}
ul := "user " + u + " " + name
keep[ul] = true
if have[ul] {
continue
}
if err := k.sacctmgr(ctx, "add", "user", u, "account="+name); err != nil {
errs = append(errs, err)
continue
}
note(ul)
}
}
// Taken away: what llmgw made and no longer wants, users before their
// accounts, children before their parents (made later, listed later).
var left []string
for i := len(ours) - 1; i >= 0; i-- {
line := ours[i]
if keep[line] || !strings.HasPrefix(line, "user ") {
continue
}
f := strings.Fields(line)
if !have[line] {
continue
}
if err := k.removeAssoc(ctx, f[1], f[2], have, keep); err != nil {
errs = append(errs, err)
left = append(left, line)
}
}
for i := len(ours) - 1; i >= 0; i-- {
line := ours[i]
if keep[line] || !strings.HasPrefix(line, "account ") {
continue
}
if have[line] {
k.log.Info("a Slurm account llmgw made is taken away; its jobs' records stay", "account", line[len("account "):])
if err := k.sacctmgr(ctx, "remove", "account", line[len("account "):]); err != nil {
errs = append(errs, err)
left = append(left, line)
}
}
}
var lines []string
for _, l := range ours {
if keep[l] || slices.Contains(left, l) {
lines = append(lines, l)
}
}
if !slices.Equal(lines, made) {
var err error
if len(lines) == 0 {
err = k.host.Remove(k.accountsMarker())
} else {
_, err = k.write(k.accountsMarker(), []byte(strings.Join(lines, "\n")+"\n"), 0o644, "")
}
errs = append(errs, err)
}
return errors.Join(errs...)
}
func validAccount(s string) bool {
return s != "" && len(s) <= 64 && strings.Trim(s, "abcdefghijklmnopqrstuvwxyz0123456789_-") == ""
}
// ended are the jobs that ended in the last day, newest last.
func (k *Keeper) ended(ctx context.Context) ([]*llmgwv1.SlurmJob, error) {
out, err := k.host.Run(ctx, nil, "sacct", slurm.SacctArgs...)
if err != nil {
return nil, fmt.Errorf("sacct: %v: %s", err, lastLine(out))
}
jobs, err := slurm.ParseJobs(string(out))
if err != nil {
return nil, err
}
if len(jobs) > maxEnded {
jobs = jobs[len(jobs)-maxEnded:]
}
ps := make([]*llmgwv1.SlurmJob, 0, len(jobs))
for _, j := range jobs {
ps = append(ps, &llmgwv1.SlurmJob{Id: j.ID, User: j.User, Account: j.Account, Partition: j.Partition,
State: j.State, StartUnix: j.Start.Unix(), EndUnix: j.End.Unix(), ElapsedSeconds: int64(j.Elapsed.Seconds()),
Cpus: int32(j.CPUs), Gpus: int32(j.GPUs)}) //nolint:gosec // counts of a node's
}
return ps, nil
}
// removeAssoc takes login off account. Slurm refuses to take away a
// user's default account, so another of theirs is made the default first,
// one llmgw keeps if there is one; with none left the user goes from the
// accounting entirely.
func (k *Keeper) removeAssoc(ctx context.Context, login, account string, have, keep map[string]bool) error {
var others []string
for line := range have {
f := strings.Fields(line)
if len(f) == 3 && f[0] == "user" && f[1] == login && f[2] != account {
others = append(others, f[2])
}
}
if len(others) == 0 {
return k.sacctmgr(ctx, "remove", "user", login)
}
slices.SortFunc(others, func(a, b string) int {
ka, kb := keep["user "+login+" "+a], keep["user "+login+" "+b]
switch {
case ka && !kb:
return -1
case kb && !ka:
return 1
}
return strings.Compare(a, b)
})
if err := k.sacctmgr(ctx, "modify", "user", login, "set", "defaultaccount="+others[0]); err != nil {
return err
}
return k.sacctmgr(ctx, "remove", "user", login, "account="+account)
}