Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s

With slurm.users llmgw, each project is a Slurm account, its organization's
and team's above it, with the project's members and its team's and
organization's on it; slurm.conf holds a job to its user's accounts. The
controller's agent keeps them with sacctmgr, only what it made, and reports
the jobs that ended in the last day; the leader records each once, its
GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its
project's usage as model slurm/PARTITION.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-15 11:54:01 +01:00
1 parent 8547528fb7
commit fcda4d3f85
22 files changed
+1390 -76

No files matched your search

+57
View File
@@ -11,6 +11,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
)
type fakeInv struct {
@@ -323,3 +324,59 @@ func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
t.Fatal("pyxis still loaded with containers off")
}
}
func TestTheControllerKeepsTheProjectsAccountsAndJobsAreRecordedOnce(t *testing.T) {
r := newRig()
r.set.Users = config.UsersLlmgw
r.svc.SetUsers(func(context.Context) ([]User, error) { return []User{{Login: "ayse", UID: 200001}}, nil })
read := 0
r.svc.SetAccounts(func(context.Context) ([]slurm.Account, error) {
read++
return []slurm.Account{{Name: "test", Parent: "root"},
{Name: "test-arge-deneme", Parent: "test", Project: "p1", Users: []string{"ayse"}}}, nil
})
got := make(chan []JobRecord, 4)
r.svc.SetJobs(func(_ context.Context, jobs []JobRecord) error { got <- jobs; return nil })
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
r.tick()
ctl := r.agents.last("node06")
if !ctl.GetManageAccounts() || len(ctl.GetAccounts()) != 2 || ctl.GetAccounts()[1].GetUsers()[0] != "ayse" {
t.Fatalf("controller told %v", ctl.GetAccounts())
}
if r.agents.last("cuda01").GetManageAccounts() {
t.Error("a node was told the accounts")
}
if !strings.Contains(ctl.GetFiles()["slurm.conf"], "AccountingStorageEnforce=associations") {
t.Error("jobs not held to accounts")
}
if r.svc.Status().Accounts != 1 || len(r.svc.AccountsOf("ayse")) != 1 {
t.Errorf("status %d, ayse's %v", r.svc.Status().Accounts, r.svc.AccountsOf("ayse"))
}
ended := []*llmgwv1.SlurmJob{{Id: "41", User: "ayse", Account: "test-arge-deneme", ElapsedSeconds: 60, Gpus: 1}}
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
select {
case jobs := <-got:
if len(jobs) != 1 || jobs[0].Login != "ayse" || jobs[0].GPUs != 1 {
t.Fatalf("jobs %+v", jobs)
}
case <-time.After(5 * time.Second):
t.Fatal("not recorded")
}
// The next status says it again: not recorded twice.
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) {
r.svc.mu.Lock()
busy := r.svc.recording
r.svc.mu.Unlock()
if !busy {
break
}
time.Sleep(10 * time.Millisecond)
}
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
select {
case jobs := <-got:
t.Fatalf("recorded again: %+v", jobs)
case <-time.After(200 * time.Millisecond):
}
}