diff --git a/BACKLOG.md b/BACKLOG.md index c154b5c..33af0bd 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2081,6 +2081,13 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t VMs: ayse without a job refused ("no active jobs on this node"), with one in job_N/step_extern (nproc 1), cut off when it ended; mehmet refused; admin's session unchanged. Left: sites whose users come from AD (sssd) are not held yet; a group of theirs would be the list. +- A GPU node, tried 2026-11-14 on the PoC (cuda01, Quadro RTX 6000, Slurm 23.11.4): gres.conf + `Type=rtx6000 File=/dev/nvidia0`, slurm.conf `Gres=gpu:rtx6000:1`. `srun --gres=gpu:1`: + CUDA_VISIBLE_DEVICES=0, the one card; torch found it and ran 14.1 TFLOPS (fp32 matmul). Without + --gres: "No devices found", torch none (ConstrainDevices holds). `--gres=gpu:2`: refused. sacct has + `gres/gpu=1` in AllocTRES. Found: lending a node whose GPU model has nowhere else to go waits out + the drain grace (5 min) before the replica stops, and `ctl slurm status` says only "1 replica(s) + still to move off"; it should say it waits for a replacement that cannot be placed, and until when. - P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm: each project an account (`sacctmgr add account`, parent its team and organization), each member a user of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json index 0386264..a6822d8 100644 --- a/docs/admin-api.openapi.json +++ b/docs/admin-api.openapi.json @@ -12336,6 +12336,11 @@ "replicas": { "type": "integer" }, + "stops_by": { + "type": "string", + "format": "date-time", + "description": "StopsBy is, while it is being lent, when a replica still on it is stopped even with nowhere else to go: the drain's grace after the lend began. Zero when not known (a leader that took over meanwhile)." + }, "slurm": { "$ref": "#/components/schemas/slurmsvc.Reported" }, diff --git a/internal/control/slurmsvc/service.go b/internal/control/slurmsvc/service.go index 8f29747..3a10c89 100644 --- a/internal/control/slurmsvc/service.go +++ b/internal/control/slurmsvc/service.go @@ -27,6 +27,7 @@ import ( llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/config" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/slurm" ) @@ -62,9 +63,13 @@ const ( // NodeStatus is one node as the service sees it. type NodeStatus struct { - Node string `json:"node"` - Phase Phase `json:"phase"` - Replicas int `json:"replicas"` + Node string `json:"node"` + Phase Phase `json:"phase"` + Replicas int `json:"replicas"` + // StopsBy is, while it is being lent, when a replica still on it is + // stopped even with nowhere else to go: the drain's grace after the + // lend began. Zero when not known (a leader that took over meanwhile). + StopsBy time.Time `json:"stops_by,omitempty"` Status *Reported `json:"slurm,omitempty"` Connected bool `json:"connected"` } @@ -129,6 +134,8 @@ type Service struct { // drained are nodes this service put in Draining, which it alone makes // ready again. drained map[string]bool + // lendingAt is when this leader began moving a node's replicas off. + lendingAt map[string]time.Time // sent is what each node was last told, so it is told again only when // that changes or its stream is new. sent map[string]*llmgwv1.Slurm @@ -202,7 +209,7 @@ func New(settings func() config.SlurmSettings, inv Inventory, agents Agents, log = slog.Default() } return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log, - status: map[string]*Reported{}, drained: map[string]bool{}, sent: map[string]*llmgwv1.Slurm{}, + status: map[string]*Reported{}, drained: map[string]bool{}, lendingAt: map[string]time.Time{}, sent: map[string]*llmgwv1.Slurm{}, kick: make(chan struct{}, 1)} } @@ -406,11 +413,14 @@ func (s *Service) phase(set config.SlurmSettings, node string) Phase { if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) { s.inv.SetState(node, inventory.Draining) s.drained[node] = true - s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node) + s.lendingAt[node] = time.Now() + s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node, + "stopped_by", time.Now().Add(drainGrace).Format(time.TimeOnly)) } if !ok || len(s.inv.Allocations(node)) > 0 { return Lending } + delete(s.lendingAt, node) return Lent } // Not lent. Until the agent says Slurm has drained it with no job left, @@ -430,12 +440,17 @@ func (s *Service) phase(set config.SlurmSettings, node string) Phase { } return Reclaiming } + delete(s.lendingAt, node) if s.drained[node] { s.ready(node) } return Llmgw } +// drainGrace is how long a drained replica waits for its replacement +// before it is stopped anyway: the reconciler's, which is not a setting. +var drainGrace = reconciler.DefaultConfig().DrainGrace + // slurmBusy is whether Slurm may still run work on a node: a job step left // on it (or no telling), or a controller that would still give it one. A // node the controller does not know of, or cannot be asked about, gets no @@ -643,6 +658,9 @@ func (s *Service) Status() Status { ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)), Connected: slices.Contains(conn, node)} ns.Phase = s.peek(set, node) + if at, ok := s.lendingAt[node]; ok && ns.Phase == Lending { + ns.StopsBy = at.Add(drainGrace) + } out.Nodes = append(out.Nodes, ns) } return out @@ -670,6 +688,12 @@ func (s *Service) peek(set config.SlurmSettings, node string) Phase { func (n NodeStatus) Describe() string { switch n.Phase { case Lending: + if !n.StopsBy.IsZero() { + // A replica moves once its replacement serves; one with nowhere + // else to go waits out the grace, and that wait is not a hang. + return fmt.Sprintf("being lent: %d replica(s) still to move off; one with nowhere else to go is stopped at %s", + n.Replicas, n.StopsBy.Local().Format(time.TimeOnly)) + } return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas) case Lent: return "lent to Slurm" diff --git a/internal/control/slurmsvc/service_test.go b/internal/control/slurmsvc/service_test.go index e802004..d25b5e9 100644 --- a/internal/control/slurmsvc/service_test.go +++ b/internal/control/slurmsvc/service_test.go @@ -5,6 +5,7 @@ import ( "slices" "strings" "testing" + "time" llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" @@ -272,3 +273,17 @@ func TestAllNodes(t *testing.T) { t.Fatalf("cuda02 not lent: %+v", p) } } + +func TestLendingSaysWhenAReplicaWithNowhereToGoStops(t *testing.T) { + // Seen on the PoC: a GPU model with no other card waited out the drain's + // grace, and five minutes of "1 replica(s) still to move off" read as a hang. + at := time.Date(2026, 11, 14, 19, 38, 34, 0, time.Local) + n := NodeStatus{Node: "cuda01", Phase: Lending, Replicas: 1, StopsBy: at} + if d := n.Describe(); !strings.Contains(d, "nowhere else to go is stopped at 19:38:34") { + t.Errorf("%q", d) + } + n.StopsBy = time.Time{} + if d := n.Describe(); d != "being lent: 1 replica(s) still to move off" { + t.Errorf("%q", d) + } +}