A node being lent says when a replica with nowhere to go is stopped

Tried on the PoC: lending cuda01, the one GPU node, its model had nowhere
else to go and waited out the drain's five-minute grace, while ctl slurm
status said only "1 replica(s) still to move off". It now says the time
that replica is stopped at. Also recorded: Slurm on a GPU node, tried on
cuda01 (gres, ConstrainDevices, accounting of gres/gpu).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

BACKLOG.md

@@ -2081,6 +2081,13 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t
   VMs: ayse without a job refused ("no active jobs on this node"), with one in
   job_N/step_extern (nproc 1), cut off when it ended; mehmet refused; admin's session unchanged.
   Left: sites whose users come from AD (sssd) are not held yet; a group of theirs would be the list.
+- A GPU node, tried 2026-11-14 on the PoC (cuda01, Quadro RTX 6000, Slurm 23.11.4): gres.conf
+  `Type=rtx6000 File=/dev/nvidia0`, slurm.conf `Gres=gpu:rtx6000:1`. `srun --gres=gpu:1`:
+  CUDA_VISIBLE_DEVICES=0, the one card; torch found it and ran 14.1 TFLOPS (fp32 matmul). Without
+  --gres: "No devices found", torch none (ConstrainDevices holds). `--gres=gpu:2`: refused. sacct has
+  `gres/gpu=1` in AllocTRES. Found: lending a node whose GPU model has nowhere else to go waits out
+  the drain grace (5 min) before the replica stops, and `ctl slurm status` says only "1 replica(s)
+  still to move off"; it should say it waits for a replacement that cannot be placed, and until when.
 - P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm:
   each project an account (`sacctmgr add account`, parent its team and organization), each member a user
   of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads

docs/admin-api.openapi.json

@@ -12336,6 +12336,11 @@
           "replicas": {
             "type": "integer"
           },
+          "stops_by": {
+            "type": "string",
+            "format": "date-time",
+            "description": "StopsBy is, while it is being lent, when a replica still on it is stopped even with nowhere else to go: the drain's grace after the lend began. Zero when not known (a leader that took over meanwhile)."
+          },
           "slurm": {
             "$ref": "#/components/schemas/slurmsvc.Reported"
           },

internal/control/slurmsvc/service.go

@@ -27,6 +27,7 @@ import (
 	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/config"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
+	"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
 )
 
@@ -62,9 +63,13 @@ const (
 
 // NodeStatus is one node as the service sees it.
 type NodeStatus struct {
-	Node      string    `json:"node"`
-	Phase     Phase     `json:"phase"`
-	Replicas  int       `json:"replicas"`
+	Node     string `json:"node"`
+	Phase    Phase  `json:"phase"`
+	Replicas int    `json:"replicas"`
+	// StopsBy is, while it is being lent, when a replica still on it is
+	// stopped even with nowhere else to go: the drain's grace after the
+	// lend began. Zero when not known (a leader that took over meanwhile).
+	StopsBy   time.Time `json:"stops_by,omitempty"`
 	Status    *Reported `json:"slurm,omitempty"`
 	Connected bool      `json:"connected"`
 }
@@ -129,6 +134,8 @@ type Service struct {
 	// drained are nodes this service put in Draining, which it alone makes
 	// ready again.
 	drained map[string]bool
+	// lendingAt is when this leader began moving a node's replicas off.
+	lendingAt map[string]time.Time
 	// sent is what each node was last told, so it is told again only when
 	// that changes or its stream is new.
 	sent    map[string]*llmgwv1.Slurm
@@ -202,7 +209,7 @@ func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
 		log = slog.Default()
 	}
 	return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log,
-		status: map[string]*Reported{}, drained: map[string]bool{}, sent: map[string]*llmgwv1.Slurm{},
+		status: map[string]*Reported{}, drained: map[string]bool{}, lendingAt: map[string]time.Time{}, sent: map[string]*llmgwv1.Slurm{},
 		kick: make(chan struct{}, 1)}
 }
 
@@ -406,11 +413,14 @@ func (s *Service) phase(set config.SlurmSettings, node string) Phase {
 		if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
 			s.inv.SetState(node, inventory.Draining)
 			s.drained[node] = true
-			s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node)
+			s.lendingAt[node] = time.Now()
+			s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node,
+				"stopped_by", time.Now().Add(drainGrace).Format(time.TimeOnly))
 		}
 		if !ok || len(s.inv.Allocations(node)) > 0 {
 			return Lending
 		}
+		delete(s.lendingAt, node)
 		return Lent
 	}
 	// Not lent. Until the agent says Slurm has drained it with no job left,
@@ -430,12 +440,17 @@ func (s *Service) phase(set config.SlurmSettings, node string) Phase {
 		}
 		return Reclaiming
 	}
+	delete(s.lendingAt, node)
 	if s.drained[node] {
 		s.ready(node)
 	}
 	return Llmgw
 }
 
+// drainGrace is how long a drained replica waits for its replacement
+// before it is stopped anyway: the reconciler's, which is not a setting.
+var drainGrace = reconciler.DefaultConfig().DrainGrace
+
 // slurmBusy is whether Slurm may still run work on a node: a job step left
 // on it (or no telling), or a controller that would still give it one. A
 // node the controller does not know of, or cannot be asked about, gets no
@@ -643,6 +658,9 @@ func (s *Service) Status() Status {
 		ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)),
 			Connected: slices.Contains(conn, node)}
 		ns.Phase = s.peek(set, node)
+		if at, ok := s.lendingAt[node]; ok && ns.Phase == Lending {
+			ns.StopsBy = at.Add(drainGrace)
+		}
 		out.Nodes = append(out.Nodes, ns)
 	}
 	return out
@@ -670,6 +688,12 @@ func (s *Service) peek(set config.SlurmSettings, node string) Phase {
 func (n NodeStatus) Describe() string {
 	switch n.Phase {
 	case Lending:
+		if !n.StopsBy.IsZero() {
+			// A replica moves once its replacement serves; one with nowhere
+			// else to go waits out the grace, and that wait is not a hang.
+			return fmt.Sprintf("being lent: %d replica(s) still to move off; one with nowhere else to go is stopped at %s",
+				n.Replicas, n.StopsBy.Local().Format(time.TimeOnly))
+		}
 		return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas)
 	case Lent:
 		return "lent to Slurm"

internal/control/slurmsvc/service_test.go

@@ -5,6 +5,7 @@ import (
 	"slices"
 	"strings"
 	"testing"
+	"time"
 
 	llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
 	"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
@@ -272,3 +273,17 @@ func TestAllNodes(t *testing.T) {
 		t.Fatalf("cuda02 not lent: %+v", p)
 	}
 }
+
+func TestLendingSaysWhenAReplicaWithNowhereToGoStops(t *testing.T) {
+	// Seen on the PoC: a GPU model with no other card waited out the drain's
+	// grace, and five minutes of "1 replica(s) still to move off" read as a hang.
+	at := time.Date(2026, 11, 14, 19, 38, 34, 0, time.Local)
+	n := NodeStatus{Node: "cuda01", Phase: Lending, Replicas: 1, StopsBy: at}
+	if d := n.Describe(); !strings.Contains(d, "nowhere else to go is stopped at 19:38:34") {
+		t.Errorf("%q", d)
+	}
+	n.StopsBy = time.Time{}
+	if d := n.Describe(); d != "being lent: 1 replica(s) still to move off" {
+		t.Errorf("%q", d)
+	}
+}