package slurmnode import ( "context" "errors" "fmt" "slices" "strings" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" ) // accountsMarker lists what llmgw made in Slurm's accounting, a line each: // "account NAME" and "user LOGIN NAME". Nothing else is changed or taken // away: an account a Slurm administrator made is theirs. func (k *Keeper) accountsMarker() string { return k.marker + "-accounts" } // maxEnded is the most ended jobs sent at once: a day's on a busy cluster // would otherwise be a status message of megabytes. const maxEnded = 2000 // accounts keeps Slurm's accounts and their users as want says, on the // controller. func (k *Keeper) accounts(ctx context.Context, want *llmgwv1.Slurm) error { made := k.madeAccounts() if !want.GetManageAccounts() { if len(made) == 0 { return nil } return k.applyAccounts(ctx, nil, made) } return k.applyAccounts(ctx, want.GetAccounts(), made) } func (k *Keeper) madeAccounts() []string { b, err := k.host.ReadFile(k.accountsMarker()) if err != nil { return nil } var out []string for _, l := range strings.Split(strings.TrimSpace(string(b)), "\n") { if l != "" { out = append(out, l) } } return out } func (k *Keeper) sacctmgr(ctx context.Context, args ...string) error { out, err := k.host.Run(ctx, nil, "sacctmgr", append([]string{"-i"}, args...)...) if err != nil { return fmt.Errorf("sacctmgr %s: %v: %s", strings.Join(args, " "), err, lastLine(out)) } return nil } func (k *Keeper) applyAccounts(ctx context.Context, want []*llmgwv1.SlurmAccount, made []string) error { out, err := k.host.Run(ctx, nil, "sacctmgr", slurm.AssocFormat...) if err != nil { return fmt.Errorf("reading Slurm's accounts: %v: %s", err, lastLine(out)) } have := map[string]bool{} // "account NAME", "user LOGIN NAME" for _, a := range slurm.ParseAssociations(string(out)) { if a.User == "" { have["account "+a.Account] = true } else { have["user "+a.User+" "+a.Account] = true } } keep := map[string]bool{} var errs []error ours := slices.Clone(made) note := func(line string) { if !slices.Contains(ours, line) { ours = append(ours, line) } } for _, a := range want { name, parent := a.GetName(), a.GetParent() if !validAccount(name) || (parent != "root" && !validAccount(parent)) { errs = append(errs, fmt.Errorf("the leader sent an account %q under %q", name, parent)) continue } line := "account " + name keep[line] = true if !have[line] { k.log.Info("a Slurm account is made, as llmgw's projects say", "account", name, "parent", parent) if err := k.sacctmgr(ctx, "add", "account", name, "parent="+parent, "description=llmgw"); err != nil { errs = append(errs, err) continue } note(line) } for _, u := range a.GetUsers() { if !slurm.ValidLogin(u) { continue } ul := "user " + u + " " + name keep[ul] = true if have[ul] { continue } if err := k.sacctmgr(ctx, "add", "user", u, "account="+name); err != nil { errs = append(errs, err) continue } note(ul) } } // Taken away: what llmgw made and no longer wants, users before their // accounts, children before their parents (made later, listed later). var left []string for i := len(ours) - 1; i >= 0; i-- { line := ours[i] if keep[line] || !strings.HasPrefix(line, "user ") { continue } f := strings.Fields(line) if !have[line] { continue } if err := k.removeAssoc(ctx, f[1], f[2], have, keep); err != nil { errs = append(errs, err) left = append(left, line) } } for i := len(ours) - 1; i >= 0; i-- { line := ours[i] if keep[line] || !strings.HasPrefix(line, "account ") { continue } if have[line] { k.log.Info("a Slurm account llmgw made is taken away; its jobs' records stay", "account", line[len("account "):]) if err := k.sacctmgr(ctx, "remove", "account", line[len("account "):]); err != nil { errs = append(errs, err) left = append(left, line) } } } var lines []string for _, l := range ours { if keep[l] || slices.Contains(left, l) { lines = append(lines, l) } } if !slices.Equal(lines, made) { var err error if len(lines) == 0 { err = k.host.Remove(k.accountsMarker()) } else { _, err = k.write(k.accountsMarker(), []byte(strings.Join(lines, "\n")+"\n"), 0o644, "") } errs = append(errs, err) } return errors.Join(errs...) } func validAccount(s string) bool { return s != "" && len(s) <= 64 && strings.Trim(s, "abcdefghijklmnopqrstuvwxyz0123456789_-") == "" } // ended are the jobs that ended in the last day, newest last. func (k *Keeper) ended(ctx context.Context) ([]*llmgwv1.SlurmJob, error) { out, err := k.host.Run(ctx, nil, "sacct", slurm.SacctArgs...) if err != nil { return nil, fmt.Errorf("sacct: %v: %s", err, lastLine(out)) } jobs, err := slurm.ParseJobs(string(out)) if err != nil { return nil, err } if len(jobs) > maxEnded { jobs = jobs[len(jobs)-maxEnded:] } ps := make([]*llmgwv1.SlurmJob, 0, len(jobs)) for _, j := range jobs { ps = append(ps, &llmgwv1.SlurmJob{Id: j.ID, User: j.User, Account: j.Account, Partition: j.Partition, State: j.State, StartUnix: j.Start.Unix(), EndUnix: j.End.Unix(), ElapsedSeconds: int64(j.Elapsed.Seconds()), Cpus: int32(j.CPUs), Gpus: int32(j.GPUs)}) //nolint:gosec // counts of a node's } return ps, nil } // removeAssoc takes login off account. Slurm refuses to take away a // user's default account, so another of theirs is made the default first, // one llmgw keeps if there is one; with none left the user goes from the // accounting entirely. func (k *Keeper) removeAssoc(ctx context.Context, login, account string, have, keep map[string]bool) error { var others []string for line := range have { f := strings.Fields(line) if len(f) == 3 && f[0] == "user" && f[1] == login && f[2] != account { others = append(others, f[2]) } } if len(others) == 0 { return k.sacctmgr(ctx, "remove", "user", login) } slices.SortFunc(others, func(a, b string) int { ka, kb := keep["user "+login+" "+a], keep["user "+login+" "+b] switch { case ka && !kb: return -1 case kb && !ka: return 1 } return strings.Compare(a, b) }) if err := k.sacctmgr(ctx, "modify", "user", login, "set", "defaultaccount="+others[0]); err != nil { return err } return k.sacctmgr(ctx, "remove", "user", login, "account="+account) }