diff --git a/BACKLOG.md b/BACKLOG.md index 16c467c..b74c9e1 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2028,6 +2028,15 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t slurmd/slurmctld, status); P3 the leader side (conf from reports, lend/reclaim); P4 `ctl slurm` (status, lend, reclaim) and a TUI screen; P5 `ctl train import` of a Slurm job's output into the mirror. Later: a backup slurmctld on a second node, accounting (slurmdbd on the Patroni PG), users from LDAP/AD. +- Built 2026-11-11, not tried on a machine yet: P1 (`internal/slurm`, `slurm.*` settings), P2 + (`internal/agent/slurmnode`, proto Slurm/SlurmStatus), P3 (`internal/control/slurmsvc`, wired in the + leader; `GET /api/v1/slurm`), P4's commands (`ctl slurm status|setup|lend|reclaim|off`). Decisions on the + way: a node not heard from or with Slurm jobs is cordoned, not drained (nothing of llmgw's to move, and a + new leader must not move models off every Slurm node); jobs are counted from the node's own slurmstepd, + so a controller that is down neither hides them nor, with none, keeps the node from llmgw; a drain whose + reason does not start with "llmgw" is a Slurm administrator's and never resumed. Left: the TUI screen, P5, + a try on the PoC (needs agents built from this; slurm-wlm 23.11 on Ubuntu 24.04), GPU minor numbers are + taken to be nvidia-smi's index. ## D. Facts worth knowing before starting diff --git a/api/gen/llmgw/v1/agent.pb.go b/api/gen/llmgw/v1/agent.pb.go index b0e2adf..a89ce7f 100644 --- a/api/gen/llmgw/v1/agent.pb.go +++ b/api/gen/llmgw/v1/agent.pb.go @@ -5401,7 +5401,9 @@ type SlurmStatus struct { // state and reason are the controller's for this node, from scontrol. State string `protobuf:"bytes,5,opt,name=state,proto3" json:"state,omitempty"` Reason string `protobuf:"bytes,6,opt,name=reason,proto3" json:"reason,omitempty"` - // jobs running on this node now. + // jobs is how many job steps run on this node now, counted from their + // slurmstepd so a controller that is down does not hide them; -1 when the + // agent cannot tell. Jobs int32 `protobuf:"varint,7,opt,name=jobs,proto3" json:"jobs,omitempty"` // error is the last thing that went wrong keeping Slurm, empty when // nothing did. diff --git a/api/proto/llmgw/v1/agent.proto b/api/proto/llmgw/v1/agent.proto index e9b095c..42a5881 100644 --- a/api/proto/llmgw/v1/agent.proto +++ b/api/proto/llmgw/v1/agent.proto @@ -1024,7 +1024,9 @@ message SlurmStatus { // state and reason are the controller's for this node, from scontrol. string state = 5; string reason = 6; - // jobs running on this node now. + // jobs is how many job steps run on this node now, counted from their + // slurmstepd so a controller that is down does not hide them; -1 when the + // agent cannot tell. int32 jobs = 7; // error is the last thing that went wrong keeping Slurm, empty when // nothing did. diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index cc16492..670e184 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -116,6 +116,9 @@ { "name": "settings" }, + { + "name": "slurm" + }, { "name": "summary" }, @@ -5078,6 +5081,34 @@ } } }, + "/api/v1/slurm": { + "get": { + "operationId": "getSlurm", + "summary": "The Slurm cluster beside llmgw, and where each of its nodes is", + "description": "From the cluster's settings (slurm.*) and what each node's agent last said. A node's phase is llmgw (drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs still running) or unknown (its agent has not said).", + "tags": [ + "slurm" + ], + "responses": { + "200": { + "description": "OK", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/slurmsvc.Status" + } + } + } + }, + "401": { + "$ref": "#/components/responses/Unauthorized" + }, + "default": { + "$ref": "#/components/responses/Error" + } + } + } + }, "/api/v1/summary": { "get": { "operationId": "getSummary", @@ -12178,6 +12209,98 @@ } } }, + "slurmsvc.NodeStatus": { + "type": "object", + "description": "NodeStatus is one node as the service sees it.", + "properties": { + "node": { + "type": "string" + }, + "phase": { + "type": "string" + }, + "replicas": { + "type": "integer" + }, + "slurm": { + "$ref": "#/components/schemas/slurmsvc.Reported" + }, + "connected": { + "type": "boolean" + } + } + }, + "slurmsvc.Reported": { + "type": "object", + "description": "Reported is the agent's SlurmStatus, for the admin API.", + "properties": { + "version": { + "type": "string" + }, + "hardware": { + "type": "string" + }, + "slurmd_running": { + "type": "boolean" + }, + "slurmctld_running": { + "type": "boolean" + }, + "state": { + "type": "string" + }, + "reason": { + "type": "string" + }, + "jobs": { + "type": "integer" + }, + "error": { + "type": "string" + }, + "files_sha256": { + "type": "string" + }, + "at": { + "type": "string", + "format": "date-time" + } + } + }, + "slurmsvc.Status": { + "type": "object", + "description": "Status is the whole Slurm cluster, for `ctl slurm status`.", + "properties": { + "enabled": { + "type": "boolean" + }, + "cluster": { + "type": "string" + }, + "controller": { + "type": "string" + }, + "partition": { + "type": "string" + }, + "controller_status": { + "$ref": "#/components/schemas/slurmsvc.Reported" + }, + "nodes": { + "type": "array", + "items": { + "$ref": "#/components/schemas/slurmsvc.NodeStatus" + } + }, + "waiting": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Waiting are nodes left out of slurm.conf until they report their hardware." + } + } + }, "store.AuditEvent": { "type": "object", "description": "AuditEvent is one change: who made it, through what, to what, and how it ended. Detail never holds a secret; the writer takes them out.", diff --git a/internal/agent/slurmnode/slurmnode.go b/internal/agent/slurmnode/slurmnode.go index f6c699e..975a262 100644 --- a/internal/agent/slurmnode/slurmnode.go +++ b/internal/agent/slurmnode/slurmnode.go @@ -390,6 +390,7 @@ func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.Slur } st.SlurmdRunning = k.active(ctx, "slurmd") + st.Jobs = k.localJobs(ctx) env := k.clientEnv() out, err := k.host.Run(ctx, env, "scontrol", "show", "node", "-o", name) if err != nil { @@ -421,14 +422,23 @@ func (k *Keeper) node(ctx context.Context, want *llmgwv1.Slurm, st *llmgwv1.Slur } } st.State, st.Reason = ns.State, ns.Reason - if out, err := k.host.Run(ctx, env, "squeue", "-h", "-w", name, "-o", "%i"); err == nil { - st.Jobs = int32(len(strings.Fields(string(out)))) //nolint:gosec // a count of jobs - } else { - st.Jobs = -1 - } return nil } +// localJobs counts the job steps running here, from their slurmstepd, not +// from the controller: a job outlives a controller that is down, and is +// what decides whether llmgw may have the cards back. -1 when it cannot +// tell. +func (k *Keeper) localJobs(ctx context.Context) int32 { + // pgrep -c prints 0 and exits 1 when there is none. + out, _ := k.host.Run(ctx, nil, "pgrep", "-c", "-x", "slurmstepd") + n, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil { + return -1 + } + return int32(n) //nolint:gosec // a count of processes +} + // clientEnv points scontrol at slurmd's copy of the files on a node that is // not the controller, which has none of its own. func (k *Keeper) clientEnv() []string { diff --git a/internal/agent/slurmnode/slurmnode_test.go b/internal/agent/slurmnode/slurmnode_test.go index 75887b3..fb62b97 100644 --- a/internal/agent/slurmnode/slurmnode_test.go +++ b/internal/agent/slurmnode/slurmnode_test.go @@ -68,8 +68,11 @@ func (f *fakeHost) Run(_ context.Context, env []string, name string, args ...str f.state, f.reason = "IDLE+DRAIN", ReasonServing } return nil, nil - case strings.HasPrefix(c, "squeue"): - return []byte(f.outputs["squeue"]), nil + case strings.HasPrefix(c, "pgrep"): + if f.outputs["jobs"] == "" { + return []byte("0\n"), errors.New("exit status 1") + } + return []byte(f.outputs["jobs"]), nil } return []byte(f.outputs[c]), nil } @@ -175,7 +178,7 @@ func TestLendAndReclaim(t *testing.T) { if h.count("scontrol update NodeName=cuda01 State=RESUME") != 1 || st.GetState() != "IDLE" { t.Fatalf("a lent node was not resumed: %v %v", h.ran, st) } - h.outputs["squeue"] = "101\n102\n" + h.outputs["jobs"] = "2\n" st = k.Reconcile(context.Background(), nodeMsg(false)) if h.count("scontrol update NodeName=cuda01 State=DRAIN Reason="+ReasonServing) != 1 { t.Fatalf("a node taken back was not drained: %v", h.ran) diff --git a/internal/cli/clustersettings.go b/internal/cli/clustersettings.go index a12f421..1b885aa 100644 --- a/internal/cli/clustersettings.go +++ b/internal/cli/clustersettings.go @@ -247,6 +247,8 @@ var liveSettings = []string{ "presets", // Handed to the reconciler as they change. "on_demand", + // Read by the Slurm service each pass, kicked when they change. + "slurm", // Read by server init, node add and backups, not by the server. "backup_recipients", "backup_copy_to", } diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go index 3b949ab..feba2c1 100644 --- a/internal/cli/ctl.go +++ b/internal/cli/ctl.go @@ -189,6 +189,7 @@ func newCtlCmd() *cobra.Command { newMCPCmd(client, &asJSON), newKBCmd(client, &asJSON), newTrainCmd(client, &asJSON), + newSlurmCmd(client, &asJSON), newConfigCmd(client, &asJSON), newAuditCmd(client, &asJSON), newNotifyCmd(client, &asJSON), diff --git a/internal/cli/ctl_slurm.go b/internal/cli/ctl_slurm.go new file mode 100644 index 0000000..79ac4ec --- /dev/null +++ b/internal/cli/ctl_slurm.go @@ -0,0 +1,226 @@ +package cli + +import ( + "encoding/json" + "fmt" + "slices" + "strings" + "text/tabwriter" + + "github.com/spf13/cobra" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" + "gitea.mynodes.xyz/ilker/llmgw/internal/settings" +) + +// newSlurmCmd is the Slurm cluster run beside llmgw (BACKLOG P): set up on +// nodes llmgw already has, and nodes handed between the two. +func newSlurmCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + cmd := &cobra.Command{ + Use: "slurm", + Short: "A Slurm cluster beside llmgw, for training and other batch work", + Long: "Slurm runs on llmgw's own nodes, installed and kept by their agents: slurmctld on one node,\n" + + "slurmd on those named. A node is llmgw's, drained in Slurm, until it is lent; lent, its\n" + + "replicas move off first, then Slurm may run jobs there. Taken back, Slurm drains it and\n" + + "llmgw has it again when its last job ends. Jobs are submitted on the controller node\n" + + "(sbatch, srun) by its own users, as on any Slurm cluster.", + } + cmd.AddCommand(newSlurmStatusCmd(client, asJSON), newSlurmSetupCmd(client), newSlurmOffCmd(client), + newSlurmListCmd(client, "lend", "Let Slurm run jobs on nodes; llmgw's replicas move off first", true), + newSlurmListCmd(client, "reclaim", "Take nodes back from Slurm; llmgw has them when their jobs end", false)) + return cmd +} + +func newSlurmStatusCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { + return &cobra.Command{ + Use: "status", + Short: "The Slurm cluster, and where each of its nodes is", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + var st slurmsvc.Status + if err := client().get(cmd.Context(), "/api/v1/slurm", &st); err != nil { + return err + } + w := cmd.OutOrStdout() + if *asJSON { + return json.NewEncoder(w).Encode(st) + } + if !st.Enabled { + fmt.Fprintln(w, "Slurm is off. `llmgw ctl slurm setup --controller NODE --nodes NODE,...` turns it on.") + return nil + } + fmt.Fprintf(w, "cluster %s, partition %s, controller %s", st.Cluster, st.Partition, st.Controller) + switch c := st.Ctld; { + case c == nil: + fmt.Fprintln(w, ": not heard from yet") + case c.Error != "": + fmt.Fprintf(w, ": %s\n", c.Error) + case c.SlurmctldRunning: + fmt.Fprintf(w, ": slurmctld %s running\n", c.Version) + default: + fmt.Fprintln(w, ": slurmctld not running") + } + tw := tabwriter.NewWriter(w, 0, 4, 2, ' ', 0) + fmt.Fprintln(tw, "NODE\tPHASE\tSLURM\tJOBS\tSLURMD\tNOTE") + for _, n := range st.Nodes { + state, jobs, slurmd, note := "-", "-", "-", n.Describe() + if r := n.Status; r != nil { + state = or(r.State, "-") + jobs = fmt.Sprint(r.Jobs) + slurmd = map[bool]string{true: "running", false: "stopped"}[r.SlurmdRunning] + if r.Error != "" { + note = r.Error + } + } + if !n.Connected { + note = "agent not connected; " + note + } + fmt.Fprintf(tw, "%s\t%s\t%s\t%s\t%s\t%s\n", n.Node, n.Phase, state, jobs, slurmd, note) + } + if err := tw.Flush(); err != nil { + return err + } + if len(st.Waiting) > 0 { + fmt.Fprintf(w, "not in slurm.conf until they report their hardware: %s\n", strings.Join(st.Waiting, ", ")) + } + return nil + }, + } +} + +func newSlurmSetupCmd(client func() *ctlClient) *cobra.Command { + var controller, cluster, partition, note string + var nodes []string + c := &cobra.Command{ + Use: "setup --controller NODE --nodes NODE,...", + Short: "Run Slurm on llmgw's nodes: slurmctld on one, slurmd on those named", + Long: "Turns slurm.enabled on and names the nodes. Their agents install munge and Slurm (22.05 or\n" + + "newer) from the machines' own package sources. Every node starts drained in Slurm and stays\n" + + "llmgw's until `llmgw ctl slurm lend` gives it to Slurm.", + Example: " llmgw ctl slurm setup --controller node06 --nodes cuda01,cuda02", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + if controller == "" || len(nodes) == 0 { + return fmt.Errorf("give --controller and --nodes") + } + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + if err := doc.SetYAML([]string{"slurm", "enabled"}, "true"); err != nil { + return err + } + if err := doc.SetYAML([]string{"slurm", "controller"}, yamlString(controller)); err != nil { + return err + } + if err := doc.SetYAML([]string{"slurm", "nodes"}, yamlList(nodes)); err != nil { + return err + } + // Lent nodes no longer named are not lent. + lent := slices.DeleteFunc(docList(doc, []string{"slurm", "lent"}), + func(n string) bool { return !slices.Contains(nodes, n) }) + if err := setList(doc, []string{"slurm", "lent"}, lent); err != nil { + return err + } + for k, v := range map[string]string{"cluster_name": cluster, "partition": partition} { + if v != "" { + if err := doc.SetYAML([]string{"slurm", k}, yamlString(v)); err != nil { + return err + } + } + } + return nil + }) + }, + } + c.Flags().StringVar(&controller, "controller", "", "the node that runs slurmctld") + c.Flags().StringSliceVar(&nodes, "nodes", nil, "the nodes that run slurmd") + c.Flags().StringVar(&cluster, "cluster-name", "", "Slurm's ClusterName (llmgw)") + c.Flags().StringVar(&partition, "partition", "", "the partition every node is in (gpu)") + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c +} + +func newSlurmOffCmd(client func() *ctlClient) *cobra.Command { + var note string + var force bool + c := &cobra.Command{ + Use: "off", + Short: "Stop running Slurm; every node is llmgw's again", + Long: "slurmd and slurmctld are stopped where llmgw started them; the packages stay. A node still\n" + + "lent is refused unless --force: take it back first, so its jobs end before the daemons do.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, _ []string) error { + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + if lent := docList(doc, []string{"slurm", "lent"}); len(lent) > 0 && !force { + return fmt.Errorf("%s still lent: `llmgw ctl slurm reclaim` them first, or --force", + strings.Join(lent, ", ")) + } + setList(doc, []string{"slurm", "lent"}, nil) //nolint:errcheck // an empty list + return doc.SetYAML([]string{"slurm", "enabled"}, "false") + }) + }, + } + c.Flags().BoolVar(&force, "force", false, "even with nodes still lent") + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c +} + +// newSlurmListCmd adds nodes to slurm.lent, or takes them out. +func newSlurmListCmd(client func() *ctlClient, use, short string, add bool) *cobra.Command { + var note string + c := &cobra.Command{ + Use: use + " NODE...", + Short: short, + Args: cobra.MinimumNArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + return changeSettings(cmd, client, note, func(doc *settings.Doc) error { + nodes := docList(doc, []string{"slurm", "nodes"}) + lent := docList(doc, []string{"slurm", "lent"}) + for _, n := range args { + switch { + case add && !slices.Contains(nodes, n): + return fmt.Errorf("%s does not run slurmd: add it with `llmgw ctl slurm setup --nodes`", n) + case add && !slices.Contains(lent, n): + lent = append(lent, n) + case !add: + lent = slices.DeleteFunc(lent, func(l string) bool { return l == n }) + } + } + return setList(doc, []string{"slurm", "lent"}, lent) + }) + }, + } + c.Flags().StringVar(¬e, "note", "", "why, kept with the settings' version") + return c +} + +// docList reads a list of strings from the settings, empty when unset. +func docList(doc *settings.Doc, path []string) []string { + n := doc.Node(path) + if n == nil { + return nil + } + var out []string + if err := n.Decode(&out); err != nil { + return nil + } + return out +} + +func setList(doc *settings.Doc, path []string, list []string) error { + if len(list) == 0 { + if doc.Node(path) != nil { + doc.Unset(path) + } + return nil + } + return doc.SetYAML(path, yamlList(list)) +} + +func yamlString(s string) string { + b, _ := json.Marshal(s) + return string(b) +} + +func yamlList(list []string) string { + b, _ := json.Marshal(list) + return string(b) +} diff --git a/internal/cli/ctl_slurm_test.go b/internal/cli/ctl_slurm_test.go new file mode 100644 index 0000000..0f41438 --- /dev/null +++ b/internal/cli/ctl_slurm_test.go @@ -0,0 +1,35 @@ +package cli + +import ( + "slices" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/settings" +) + +func TestSlurmDocLists(t *testing.T) { + doc, err := settings.FromBytes("settings", []byte("slurm:\n enabled: true\n nodes: [cuda01, site2]\n")) + if err != nil { + t.Fatal(err) + } + if got := docList(doc, []string{"slurm", "nodes"}); !slices.Equal(got, []string{"cuda01", "site2"}) { + t.Fatalf("nodes %v", got) + } + if got := docList(doc, []string{"slurm", "lent"}); got != nil { + t.Fatalf("lent %v", got) + } + if err := setList(doc, []string{"slurm", "lent"}, []string{"cuda01"}); err != nil { + t.Fatal(err) + } + if got := docList(doc, []string{"slurm", "lent"}); !slices.Equal(got, []string{"cuda01"}) { + t.Fatalf("lent %v", got) + } + if err := setList(doc, []string{"slurm", "lent"}, nil); err != nil { + t.Fatal(err) + } + b, _ := doc.Bytes() + if strings.Contains(string(b), "lent") { + t.Fatalf("an empty list left:\n%s", b) + } +} diff --git a/internal/cli/server.go b/internal/cli/server.go index 5e02e35..d53af8e 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -49,6 +49,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/fleetmetrics" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" @@ -59,6 +60,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/promq" "gitea.mynodes.xyz/ilker/llmgw/internal/router" "gitea.mynodes.xyz/ilker/llmgw/internal/secrets" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store" "gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf" "gitea.mynodes.xyz/ilker/llmgw/internal/version" @@ -261,6 +263,8 @@ func newServerCmd() *cobra.Command { // What the settings say of the firewall, for the mirror and the // store's members; nil without a control plane. var fwPub *firewallPublisher + // The Slurm cluster beside llmgw, when there is a store (BACKLOG P). + var slurmSvc *slurmsvc.Service updateMetrics := &modelserve.UpdateCollector{} // The store's cluster, when it is one; read by the metrics and // by `ctl status`. @@ -436,6 +440,19 @@ func newServerCmd() *cobra.Command { agentSvc.SetTrain(trainer) go trainer.Run(ctx) api.SetTrain(&adminapi.TrainAdmin{Store: st, Service: trainer}) + // Slurm on the nodes the settings name, its munge key + // made from a cluster secret. + slurmSvc = slurmsvc.New(func() config.SlurmSettings { return current.Load().Slurm }, + admin.Inventory(), agentSvc, func(ctx context.Context) ([]byte, error) { + secret, err := st.ClusterSecret(ctx, "slurm-munge") + if err != nil { + return nil, err + } + return slurm.MungeKey(secret) + }, log) + agentSvc.SetSlurm(slurmSvc) + go slurmSvc.Run(ctx) + api.SetSlurm(slurmSvc) } // The cluster's settings, each change checked against this // machine's own keys before it is kept. @@ -841,6 +858,9 @@ func newServerCmd() *cobra.Command { fwPub.Kick() } } + if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "slurm") }) && slurmSvc != nil { + slurmSvc.Kick() + } if slices.ContainsFunc(changed, func(c string) bool { return strings.HasPrefix(c, "on_demand") }) && admin != nil { admin.Reconciler().SetEvictionTimes(next.OnDemand.EvictAfterIdle, next.OnDemand.MinLifetime) } diff --git a/internal/control/adminapi/api.go b/internal/control/adminapi/api.go index 7f594b4..583d571 100644 --- a/internal/control/adminapi/api.go +++ b/internal/control/adminapi/api.go @@ -78,7 +78,9 @@ type API struct { // tenancy is organizations, keys and people; nil without a store. tenancy TenancyStore // train is adapter training; nil without a store and agents. - train *TrainAdmin + train *TrainAdmin + // slurm is the Slurm cluster beside llmgw; nil without a store and agents. + slurm SlurmService settings *Settings auth *AuthAdmin mcp MCPStore @@ -136,6 +138,7 @@ func (a *API) Register(mux Mux) { a.registerKB(mux) a.registerTenancy(mux) a.registerTrain(mux) + a.registerSlurm(mux) a.registerGuard(mux) a.registerSettings(mux) a.registerAuth(mux) diff --git a/internal/control/adminapi/routes.go b/internal/control/adminapi/routes.go index 14665f7..d003789 100644 --- a/internal/control/adminapi/routes.go +++ b/internal/control/adminapi/routes.go @@ -12,6 +12,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/federation" "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve" "gitea.mynodes.xyz/ilker/llmgw/internal/control/nvlink" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" "gitea.mynodes.xyz/ilker/llmgw/internal/notify" "gitea.mynodes.xyz/ilker/llmgw/internal/store" @@ -392,6 +393,12 @@ var Routes = []Route{ Description: "Their keys in projects they no longer reach are revoked.", Request: MembershipRequest{}, Response: UserView{}}, + {Method: "GET", Path: "/api/v1/slurm", Summary: "The Slurm cluster beside llmgw, and where each of its nodes is", + Description: "From the cluster's settings (slurm.*) and what each node's agent last said. A node's phase is llmgw " + + "(drained in Slurm, no job left), lending (its replicas moving off), lent, reclaiming (drained in Slurm, jobs " + + "still running) or unknown (its agent has not said).", + Response: slurmsvc.Status{}}, + {Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node", Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " + "the pairs are NDJSON of {\"q\", \"a\"}. An adapter learns how answers sound far better than what they say.", diff --git a/internal/control/adminapi/slurm.go b/internal/control/adminapi/slurm.go new file mode 100644 index 0000000..f899acc --- /dev/null +++ b/internal/control/adminapi/slurm.go @@ -0,0 +1,28 @@ +package adminapi + +import ( + "net/http" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/slurmsvc" +) + +// SlurmService is the Slurm cluster beside llmgw (BACKLOG P), as the leader +// keeps it. +type SlurmService interface { + Status() slurmsvc.Status +} + +// SetSlurm gives the API the Slurm service. +func (a *API) SetSlurm(s SlurmService) { a.slurm = s } + +func (a *API) registerSlurm(mux Mux) { + mux.HandleFunc("GET /api/v1/slurm", a.slurmStatus) +} + +func (a *API) slurmStatus(w http.ResponseWriter, _ *http.Request) { + if a.slurm == nil { + writeError(w, http.StatusNotImplemented, "Slurm needs the store and the nodes' agents, which this control plane runs without") + return + } + writeJSON(w, http.StatusOK, a.slurm.Status()) +} diff --git a/internal/control/agentsvc/service.go b/internal/control/agentsvc/service.go index 3658487..c026b95 100644 --- a/internal/control/agentsvc/service.go +++ b/internal/control/agentsvc/service.go @@ -84,6 +84,8 @@ type Service struct { agentSettings func(node string) string // train is told how training jobs go; see SetTrain. train TrainHandler + // slurm is told of the nodes' part in Slurm; see SetSlurm. + slurm SlurmHandler // logWaits are the log requests awaiting a reply, by request id. logMu sync.Mutex @@ -308,6 +310,10 @@ func (s *Service) Connect( return err } } + // Its part in Slurm, sent by the Slurm service's next pass. + if h := s.slurmHandler(); h != nil { + h.OnRegister(nodeID) + } // Recover replica state from what this node is already running, before any // reconciliation pass can conclude the cluster is empty and place @@ -515,6 +521,11 @@ func (s *Service) handle(nodeID string, msg *llmgwv1.AgentMessage) { h.OnTrainStatus(nodeID, m.TrainStatus) } + case *llmgwv1.AgentMessage_SlurmStatus: + if h := s.slurmHandler(); h != nil { + h.OnSlurmStatus(nodeID, m.SlurmStatus) + } + case *llmgwv1.AgentMessage_TrainArtifact: if h := s.trainHandler(); h != nil { h.OnTrainArtifact(nodeID, m.TrainArtifact) diff --git a/internal/control/agentsvc/slurm.go b/internal/control/agentsvc/slurm.go new file mode 100644 index 0000000..2747b9c --- /dev/null +++ b/internal/control/agentsvc/slurm.go @@ -0,0 +1,25 @@ +package agentsvc + +import ( + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" +) + +// SlurmHandler is told of the nodes' part in Slurm (BACKLOG P). +type SlurmHandler interface { + OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) + // OnRegister is a node's new stream, which is told again what it is. + OnRegister(node string) +} + +// SetSlurm gives the service what Slurm's messages go to. +func (s *Service) SetSlurm(h SlurmHandler) { + s.mu.Lock() + s.slurm = h + s.mu.Unlock() +} + +func (s *Service) slurmHandler() SlurmHandler { + s.mu.RLock() + defer s.mu.RUnlock() + return s.slurm +} diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go new file mode 100644 index 0000000..2245c76 --- /dev/null +++ b/internal/control/slurmsvc/service.go @@ -0,0 +1,417 @@ +// Package slurmsvc is the leader's side of the Slurm cluster run beside +// llmgw (BACKLOG P): it tells each node's agent what it is, writes the +// controller's files from what the nodes report, and hands nodes between +// llmgw and Slurm as the cluster's settings say. +// +// A node in slurm.lent is drained of llmgw's replicas, as `ctl drain` does, +// and lent to Slurm once none is left. A node taken out of slurm.lent is +// drained in Slurm, and is llmgw's again when its last job has ended. Both +// are kept in force every tick, from the settings and what the agents say, +// so a leader that takes over carries on where the last one was. +package slurmsvc + +import ( + "context" + "fmt" + "log/slog" + "maps" + "net" + "slices" + "strconv" + "strings" + "sync" + "time" + + "google.golang.org/protobuf/proto" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" +) + +// Agents is how the service reaches the nodes. +type Agents interface { + Send(node string, msg *llmgwv1.ControlMessage) error + ConnectedNodes() []string +} + +// Inventory is what the service reads and changes of llmgw's nodes. +type Inventory interface { + Get(id string) (inventory.Node, bool) + SetState(id string, st inventory.NodeState) bool + Allocations(nodeID string) []inventory.Allocation +} + +// Phase is where a node of slurm.nodes is between llmgw and Slurm. +type Phase string + +const ( + // Llmgw: drained in Slurm with no job left, llmgw's to place on. + Llmgw Phase = "llmgw" + // Lending: in slurm.lent, its replicas being moved off. + Lending Phase = "lending" + // Lent: Slurm's; llmgw places nothing there. + Lent Phase = "lent" + // Reclaiming: out of slurm.lent, drained in Slurm, jobs still running. + Reclaiming Phase = "reclaiming" + // Unknown: its agent has not said how it is in Slurm yet. + Unknown Phase = "unknown" +) + +// NodeStatus is one node as the service sees it. +type NodeStatus struct { + Node string `json:"node"` + Phase Phase `json:"phase"` + Replicas int `json:"replicas"` + Status *Reported `json:"slurm,omitempty"` + Connected bool `json:"connected"` +} + +// Reported is the agent's SlurmStatus, for the admin API. +type Reported struct { + Version string `json:"version,omitempty"` + Hardware string `json:"hardware,omitempty"` + SlurmdRunning bool `json:"slurmd_running"` + SlurmctldRunning bool `json:"slurmctld_running"` + State string `json:"state,omitempty"` + Reason string `json:"reason,omitempty"` + Jobs int `json:"jobs"` + Error string `json:"error,omitempty"` + FilesSHA256 string `json:"files_sha256,omitempty"` + At time.Time `json:"at"` +} + +// Status is the whole Slurm cluster, for `ctl slurm status`. +type Status struct { + Enabled bool `json:"enabled"` + Cluster string `json:"cluster"` + Controller string `json:"controller"` + Partition string `json:"partition"` + Ctld *Reported `json:"controller_status,omitempty"` + Nodes []NodeStatus `json:"nodes"` + // Waiting are nodes left out of slurm.conf until they report their + // hardware. + Waiting []string `json:"waiting,omitempty"` +} + +// Service keeps the Slurm cluster. +type Service struct { + settings func() config.SlurmSettings + inv Inventory + agents Agents + key func(ctx context.Context) ([]byte, error) + log *slog.Logger + + mu sync.Mutex + // status is each node's last SlurmStatus. + status map[string]*Reported + // drained are nodes this service put in Draining, which it alone makes + // ready again. + drained map[string]bool + // sent is what each node was last told, so it is told again only when + // that changes or its stream is new. + sent map[string]*llmgwv1.Slurm + mungeKy []byte + kick chan struct{} +} + +// New is a Service; key gives the munge key, made once from a cluster +// secret. +func New(settings func() config.SlurmSettings, inv Inventory, agents Agents, + key func(ctx context.Context) ([]byte, error), log *slog.Logger) *Service { + if log == nil { + log = slog.Default() + } + return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log, + status: map[string]*Reported{}, drained: map[string]bool{}, sent: map[string]*llmgwv1.Slurm{}, + kick: make(chan struct{}, 1)} +} + +// Every is how often the service looks again without being asked. +var Every = 10 * time.Second + +// Run keeps the cluster until ctx ends. +func (s *Service) Run(ctx context.Context) { + t := time.NewTicker(Every) + defer t.Stop() + for { + s.Tick(ctx) + select { + case <-ctx.Done(): + return + case <-t.C: + case <-s.kick: + } + } +} + +// Kick has the service look again now: the settings changed. +func (s *Service) Kick() { + select { + case s.kick <- struct{}{}: + default: + } +} + +// OnSlurmStatus records what a node's agent says. +func (s *Service) OnSlurmStatus(node string, st *llmgwv1.SlurmStatus) { + r := &Reported{Version: st.GetVersion(), Hardware: st.GetHardware(), SlurmdRunning: st.GetSlurmdRunning(), + SlurmctldRunning: st.GetSlurmctldRunning(), State: st.GetState(), Reason: st.GetReason(), + Jobs: int(st.GetJobs()), Error: st.GetError(), FilesSHA256: st.GetFilesSha256(), At: time.Now()} + s.mu.Lock() + prev := s.status[node] + s.status[node] = r + s.mu.Unlock() + if r.Error != "" && (prev == nil || prev.Error != r.Error) { + s.log.Warn("a node could not keep its part in Slurm", "node", node, "err", r.Error) + } + if prev == nil || prev.Hardware != r.Hardware || prev.State != r.State || prev.Jobs != r.Jobs { + s.Kick() + } +} + +// OnRegister is a node's new stream: it is told again what it is. +func (s *Service) OnRegister(node string) { + s.mu.Lock() + delete(s.sent, node) + s.mu.Unlock() + s.Kick() +} + +// Tick does one pass: the nodes handed over as far as they can be, and +// each told what it is. +func (s *Service) Tick(ctx context.Context) { + set := s.settings() + s.mu.Lock() + defer s.mu.Unlock() + phases := s.handOver(set) + var key []byte + if set.Enabled { + if s.mungeKy == nil { + k, err := s.key(ctx) + if err != nil { + s.log.Warn("no munge key for Slurm; nothing is sent to the nodes", "err", err) + return + } + s.mungeKy = k + } + key = s.mungeKy + } + files, _ := s.files(set) + for _, node := range s.agents.ConnectedNodes() { + m := s.message(set, node, phases[node] == Lent, key, files) + if old, ok := s.sent[node]; ok && proto.Equal(old, m) { + continue + } + if err := s.agents.Send(node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_Slurm{Slurm: m}}); err != nil { + s.log.Warn("could not tell a node its part in Slurm", "node", node, "err", err) + continue + } + s.sent[node] = m + } +} + +// handOver moves each node of slurm.nodes a step between llmgw and Slurm, +// answering where each is. Under s.mu. +func (s *Service) handOver(set config.SlurmSettings) map[string]Phase { + out := map[string]Phase{} + if !set.Enabled { + // Off: every node is llmgw's again. What Slurm still runs is left + // to finish where it is; turning Slurm off with work lent is the + // operator's choice. + for node := range s.drained { + s.ready(node) + } + return out + } + for _, node := range set.Nodes { + out[node] = s.phase(set, node) + } + // A node no longer in slurm.nodes is llmgw's. + for node := range s.drained { + if !slices.Contains(set.Nodes, node) { + s.ready(node) + } + } + return out +} + +func (s *Service) phase(set config.SlurmSettings, node string) Phase { + n, ok := s.inv.Get(node) + st := s.status[node] + if set.IsLent(node) { + if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) { + s.inv.SetState(node, inventory.Draining) + s.drained[node] = true + s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node) + } + if !ok || len(s.inv.Allocations(node)) > 0 { + return Lending + } + return Lent + } + // Not lent. Until the agent says Slurm has drained it with no job left, + // llmgw places nothing new on it: a new leader starts with every node + // ready. Cordoned, not drained: llmgw has nothing there to move. + if st == nil || slurmBusy(st) { + if ok && n.State == inventory.Ready { + s.inv.SetState(node, inventory.Cordoned) + s.drained[node] = true + if st != nil { + s.log.Info("taking a node back from Slurm: llmgw places nothing there until its jobs end", + "node", node, "jobs", st.Jobs) + } + } + if st == nil { + return Unknown + } + return Reclaiming + } + if s.drained[node] { + s.ready(node) + } + return Llmgw +} + +// slurmBusy is whether Slurm may still run work on a node: a job step left +// on it (or no telling), or a controller that would still give it one. A +// node the controller does not know of, or cannot be asked about, gets no +// job from it, so a Slurm that is broken never keeps a node from llmgw. +func slurmBusy(st *Reported) bool { + if st.Jobs != 0 { + return true + } + if st.State == "" { + return false + } + return !strings.Contains(st.State, "DRAIN") && !strings.Contains(st.State, "DOWN") +} + +func (s *Service) ready(node string) { + if n, ok := s.inv.Get(node); ok && (n.State == inventory.Draining || n.State == inventory.Cordoned) { + s.inv.SetState(node, inventory.Ready) + s.log.Info("a node is llmgw's again: Slurm has no job there", "node", node) + } + delete(s.drained, node) +} + +// files are the controller's files, and the nodes left out of them for not +// having reported their hardware. Under s.mu. +func (s *Service) files(set config.SlurmSettings) (map[string]string, []string) { + ctl, ok := s.inv.Get(set.Controller) + if !set.Enabled || !ok { + return nil, nil + } + c := slurm.Cluster{Name: set.Cluster(), Controller: set.Controller, ControllerAddr: ctl.MgmtAddr, + Partition: set.PartitionName()} + var waiting []string + for _, node := range set.Nodes { + n, ok := s.inv.Get(node) + st := s.status[node] + if !ok || st == nil || st.Hardware == "" { + waiting = append(waiting, node) + continue + } + hw, err := slurm.ParseHardware(st.Hardware) + if err != nil { + waiting = append(waiting, node) + continue + } + sn := slurm.Node{Name: node, Addr: n.MgmtAddr, Hardware: hw} + for _, g := range n.GPUs { + sn.GPUs = append(sn.GPUs, slurm.GPU{Minor: g.Index, Name: g.Name}) + } + c.Nodes = append(c.Nodes, sn) + } + return c.Files(), waiting +} + +// message is what a node is told. Under s.mu. +func (s *Service) message(set config.SlurmSettings, node string, lent bool, key []byte, + files map[string]string) *llmgwv1.Slurm { + m := &llmgwv1.Slurm{} + if !set.Enabled { + return m + } + isNode := slices.Contains(set.Nodes, node) + isCtl := set.Controller == node && files != nil + if !isNode && !isCtl { + return m + } + m.Name, m.MungeKey, m.Lent = node, key, lent + m.Node = isNode + if ctl, ok := s.inv.Get(set.Controller); ok { + m.ConfServer = net.JoinHostPort(ctl.MgmtAddr, strconv.Itoa(slurm.CtldPort)) + } + if isNode && m.ConfServer == "" { + // Without its controller a node cannot start; it is told when the + // controller registers. + m.Node = false + } + if isCtl { + m.Controller = true + m.Files = maps.Clone(files) + } + if !m.Node && !m.Controller { + return &llmgwv1.Slurm{} + } + return m +} + +// Status is the cluster as the service sees it now. +func (s *Service) Status() Status { + set := s.settings() + s.mu.Lock() + defer s.mu.Unlock() + out := Status{Enabled: set.Enabled, Cluster: set.Cluster(), Controller: set.Controller, + Partition: set.PartitionName(), Ctld: s.status[set.Controller]} + if !set.Enabled { + return out + } + _, out.Waiting = s.files(set) + conn := s.agents.ConnectedNodes() + for _, node := range set.Nodes { + ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)), + Connected: slices.Contains(conn, node)} + ns.Phase = s.peek(set, node) + out.Nodes = append(out.Nodes, ns) + } + return out +} + +// peek is phase without changing anything. Under s.mu. +func (s *Service) peek(set config.SlurmSettings, node string) Phase { + st := s.status[node] + if set.IsLent(node) { + if len(s.inv.Allocations(node)) > 0 { + return Lending + } + return Lent + } + switch { + case st == nil: + return Unknown + case slurmBusy(st): + return Reclaiming + } + return Llmgw +} + +// Describe is one line for a log or a screen. +func (n NodeStatus) Describe() string { + switch n.Phase { + case Lending: + return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas) + case Lent: + return "lent to Slurm" + case Reclaiming: + if n.Status != nil { + return fmt.Sprintf("being taken back: %d job(s) still running", n.Status.Jobs) + } + return "being taken back" + case Llmgw: + return "llmgw's" + } + return "not heard from yet" +} diff --git a/internal/control/slurmsvc/service_test.go b/internal/control/slurmsvc/service_test.go new file mode 100644 index 0000000..66d2753 --- /dev/null +++ b/internal/control/slurmsvc/service_test.go @@ -0,0 +1,243 @@ +package slurmsvc + +import ( + "context" + "strings" + "testing" + + llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/config" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +type fakeInv struct { + nodes map[string]*inventory.Node + allocs map[string]int +} + +func (f *fakeInv) Get(id string) (inventory.Node, bool) { + n, ok := f.nodes[id] + if !ok { + return inventory.Node{}, false + } + return *n, true +} + +func (f *fakeInv) SetState(id string, st inventory.NodeState) bool { + n, ok := f.nodes[id] + if ok { + n.State = st + } + return ok +} + +func (f *fakeInv) Allocations(id string) []inventory.Allocation { + return make([]inventory.Allocation, f.allocs[id]) +} + +type fakeAgents struct { + conn []string + got map[string][]*llmgwv1.Slurm +} + +func (f *fakeAgents) Send(node string, m *llmgwv1.ControlMessage) error { + f.got[node] = append(f.got[node], m.GetSlurm()) + return nil +} + +func (f *fakeAgents) ConnectedNodes() []string { return f.conn } + +func (f *fakeAgents) last(node string) *llmgwv1.Slurm { + ms := f.got[node] + if len(ms) == 0 { + return nil + } + return ms[len(ms)-1] +} + +type rig struct { + set config.SlurmSettings + inv *fakeInv + agents *fakeAgents + svc *Service +} + +func newRig() *rig { + r := &rig{ + set: config.SlurmSettings{Enabled: true, Controller: "node06", Nodes: []string{"cuda01"}}, + inv: &fakeInv{nodes: map[string]*inventory.Node{ + "node06": {ID: "node06", MgmtAddr: "10.0.0.146"}, + "cuda01": {ID: "cuda01", MgmtAddr: "10.0.0.147", GPUs: []discovery.GPU{{Index: 0, Name: "NVIDIA GeForce RTX 3090"}}}, + }, allocs: map[string]int{"cuda01": 2}}, + agents: &fakeAgents{conn: []string{"node06", "cuda01"}, got: map[string][]*llmgwv1.Slurm{}}, + } + r.svc = New(func() config.SlurmSettings { return r.set }, r.inv, r.agents, + func(context.Context) ([]byte, error) { return []byte(strings.Repeat("k", 1024)), nil }, nil) + return r +} + +func (r *rig) report(node string, st *llmgwv1.SlurmStatus) { r.svc.OnSlurmStatus(node, st) } + +func (r *rig) tick() { r.svc.Tick(context.Background()) } + +const hw = "NodeName=cuda01 CPUs=16 RealMemory=64000" + +func TestMessagesAndFiles(t *testing.T) { + r := newRig() + r.tick() + ctl, node := r.agents.last("node06"), r.agents.last("cuda01") + if !ctl.GetController() || ctl.GetNode() || len(ctl.GetMungeKey()) != 1024 { + t.Fatalf("controller told %v", ctl) + } + if !node.GetNode() || node.GetController() || node.GetConfServer() != "10.0.0.146:6817" || node.GetLent() || + node.GetName() != "cuda01" || len(node.GetFiles()) != 0 { + t.Fatalf("node told %v", node) + } + if strings.Contains(ctl.GetFiles()["slurm.conf"], "NodeName=cuda01") { + t.Fatal("a node with no hardware reported is in slurm.conf") + } + if w := r.svc.Status().Waiting; len(w) != 1 || w[0] != "cuda01" { + t.Fatalf("waiting %v", w) + } + // Same again: nobody is told twice. + r.tick() + if len(r.agents.got["node06"]) != 1 || len(r.agents.got["cuda01"]) != 1 { + t.Fatal("told again with nothing changed") + } + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: not lent yet"}) + r.tick() + conf := r.agents.last("node06").GetFiles()["slurm.conf"] + if !strings.Contains(conf, "NodeName=cuda01 NodeAddr=10.0.0.147 CPUs=16 RealMemory=62080 Gres=gpu:rtx3090:1") { + t.Fatalf("slurm.conf:\n%s", conf) + } + if !strings.Contains(r.agents.last("node06").GetFiles()["gres.conf"], "File=/dev/nvidia0") { + t.Fatal("gres.conf has no card") + } + if len(r.agents.got["cuda01"]) != 1 { + t.Fatal("a node was told again when only the controller's files changed") + } + // A new stream is told again. + r.svc.OnRegister("cuda01") + r.tick() + if len(r.agents.got["cuda01"]) != 2 { + t.Fatal("a new stream was not told") + } + if r.inv.nodes["cuda01"].State != inventory.Ready { + t.Fatal("a node drained in Slurm with no job was taken from llmgw") + } +} + +func TestLendAndReclaim(t *testing.T) { + r := newRig() + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: not lent yet"}) + r.set.Lent = []string{"cuda01"} + r.tick() + if r.inv.nodes["cuda01"].State != inventory.Draining { + t.Fatal("a lent node was not drained of its replicas") + } + if r.agents.last("cuda01").GetLent() { + t.Fatal("lent to Slurm with replicas still on it") + } + if p := r.svc.Status().Nodes[0].Phase; p != Lending { + t.Fatalf("phase %s", p) + } + r.inv.allocs["cuda01"] = 0 + r.tick() + if !r.agents.last("cuda01").GetLent() || r.svc.Status().Nodes[0].Phase != Lent { + t.Fatal("not lent once its replicas were gone") + } + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "ALLOCATED", Jobs: 3}) + + // Taken back: Slurm drains it; llmgw keeps off until its jobs end. + r.set.Lent = nil + r.tick() + if r.agents.last("cuda01").GetLent() { + t.Fatal("still lent") + } + if s := r.inv.nodes["cuda01"].State; s != inventory.Draining && s != inventory.Cordoned { + t.Fatalf("llmgw may place on a node with Slurm jobs: %v", s) + } + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "ALLOCATED+DRAIN", Jobs: 1}) + r.tick() + if r.svc.Status().Nodes[0].Phase != Reclaiming || r.inv.nodes["cuda01"].State == inventory.Ready { + t.Fatal("given back with a job left") + } + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN", Reason: "llmgw: serving models"}) + r.tick() + if r.inv.nodes["cuda01"].State != inventory.Ready || r.svc.Status().Nodes[0].Phase != Llmgw { + t.Fatal("not llmgw's when its last job ended") + } +} + +func TestNewLeaderKeepsOffUntilHeard(t *testing.T) { + r := newRig() + r.inv.allocs["cuda01"] = 0 + r.tick() + // Nothing heard yet: cordoned, nothing moved. + if r.inv.nodes["cuda01"].State != inventory.Cordoned { + t.Fatalf("state %v before the agent said anything", r.inv.nodes["cuda01"].State) + } + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "MIXED", Jobs: 2}) + r.tick() + if r.inv.nodes["cuda01"].State == inventory.Ready { + t.Fatal("ready with Slurm jobs on it") + } +} + +func TestBrokenSlurmLeavesTheNodeToLlmgw(t *testing.T) { + r := newRig() + r.report("cuda01", &llmgwv1.SlurmStatus{Error: "installing Slurm: no package", Jobs: 0}) + r.tick() + if r.inv.nodes["cuda01"].State != inventory.Ready { + t.Fatal("a Slurm that does not work took a node from llmgw") + } +} + +func TestOperatorsCordonIsLeft(t *testing.T) { + r := newRig() + r.inv.nodes["cuda01"].State = inventory.Cordoned + r.report("cuda01", &llmgwv1.SlurmStatus{Hardware: hw, State: "IDLE+DRAIN"}) + r.tick() + if r.inv.nodes["cuda01"].State != inventory.Cordoned { + t.Fatal("llmgw uncordoned a node an operator cordoned") + } +} + +func TestOffAndOut(t *testing.T) { + r := newRig() + r.inv.allocs["cuda01"] = 0 + r.set.Lent = []string{"cuda01"} + r.tick() + r.set.Enabled = false + r.tick() + if m := r.agents.last("cuda01"); m.GetNode() || m.GetController() { + t.Fatalf("told %v with Slurm off", m) + } + if r.inv.nodes["cuda01"].State != inventory.Ready { + t.Fatal("not llmgw's with Slurm off") + } + + r = newRig() + r.inv.allocs["cuda01"] = 0 + r.set.Lent = []string{"cuda01"} + r.tick() + r.set.Nodes, r.set.Lent = nil, nil + r.tick() + if r.inv.nodes["cuda01"].State != inventory.Ready || r.agents.last("cuda01").GetNode() { + t.Fatal("a node taken out of slurm.nodes is not llmgw's") + } + if !r.agents.last("node06").GetController() { + t.Fatal("the controller stopped with no nodes") + } +} + +func TestNoControllerYet(t *testing.T) { + r := newRig() + delete(r.inv.nodes, "node06") + r.agents.conn = []string{"cuda01"} + r.tick() + if m := r.agents.last("cuda01"); m.GetNode() { + t.Fatalf("a node was started with no controller to fetch its files from: %v", m) + } +}