Slurm's jobs in llmgw's accounting: projects as accounts, jobs priced
CI / test (push) Failing after 7m24s
CI / test (push) Failing after 7m24s
With slurm.users llmgw, each project is a Slurm account, its organization's and team's above it, with the project's members and its team's and organization's on it; slurm.conf holds a job to its user's accounts. The controller's agent keeps them with sacctmgr, only what it made, and reports the jobs that ended in the last day; the leader records each once, its GPU-hours (core-hours without cards) priced at auth.gpu_hour_cost into its project's usage as model slurm/PARTITION. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
8547528fb7
commit
fcda4d3f85
22 files changed
+1390
-76
No files matched your search
@@ -11,6 +11,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||
)
|
||||
|
||||
type fakeInv struct {
|
||||
@@ -323,3 +324,59 @@ func TestPyxisIsLoadedOnlyOnceEveryMachineHasIt(t *testing.T) {
|
||||
t.Fatal("pyxis still loaded with containers off")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheControllerKeepsTheProjectsAccountsAndJobsAreRecordedOnce(t *testing.T) {
|
||||
r := newRig()
|
||||
r.set.Users = config.UsersLlmgw
|
||||
r.svc.SetUsers(func(context.Context) ([]User, error) { return []User{{Login: "ayse", UID: 200001}}, nil })
|
||||
read := 0
|
||||
r.svc.SetAccounts(func(context.Context) ([]slurm.Account, error) {
|
||||
read++
|
||||
return []slurm.Account{{Name: "test", Parent: "root"},
|
||||
{Name: "test-arge-deneme", Parent: "test", Project: "p1", Users: []string{"ayse"}}}, nil
|
||||
})
|
||||
got := make(chan []JobRecord, 4)
|
||||
r.svc.SetJobs(func(_ context.Context, jobs []JobRecord) error { got <- jobs; return nil })
|
||||
r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw})
|
||||
r.tick()
|
||||
ctl := r.agents.last("node06")
|
||||
if !ctl.GetManageAccounts() || len(ctl.GetAccounts()) != 2 || ctl.GetAccounts()[1].GetUsers()[0] != "ayse" {
|
||||
t.Fatalf("controller told %v", ctl.GetAccounts())
|
||||
}
|
||||
if r.agents.last("cuda01").GetManageAccounts() {
|
||||
t.Error("a node was told the accounts")
|
||||
}
|
||||
if !strings.Contains(ctl.GetFiles()["slurm.conf"], "AccountingStorageEnforce=associations") {
|
||||
t.Error("jobs not held to accounts")
|
||||
}
|
||||
if r.svc.Status().Accounts != 1 || len(r.svc.AccountsOf("ayse")) != 1 {
|
||||
t.Errorf("status %d, ayse's %v", r.svc.Status().Accounts, r.svc.AccountsOf("ayse"))
|
||||
}
|
||||
ended := []*llmgwv1.SlurmJob{{Id: "41", User: "ayse", Account: "test-arge-deneme", ElapsedSeconds: 60, Gpus: 1}}
|
||||
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
|
||||
select {
|
||||
case jobs := <-got:
|
||||
if len(jobs) != 1 || jobs[0].Login != "ayse" || jobs[0].GPUs != 1 {
|
||||
t.Fatalf("jobs %+v", jobs)
|
||||
}
|
||||
case <-time.After(5 * time.Second):
|
||||
t.Fatal("not recorded")
|
||||
}
|
||||
// The next status says it again: not recorded twice.
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
r.svc.mu.Lock()
|
||||
busy := r.svc.recording
|
||||
r.svc.mu.Unlock()
|
||||
if !busy {
|
||||
break
|
||||
}
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
}
|
||||
r.report("node06", &llmgwv1.SlurmStatus{Ended: ended})
|
||||
select {
|
||||
case jobs := <-got:
|
||||
t.Fatalf("recorded again: %+v", jobs)
|
||||
case <-time.After(200 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user