A node being lent says when a replica with nowhere to go is stopped
CI / test (push) Successful in 9m37s
CI / test (push) Successful in 9m37s
Tried on the PoC: lending cuda01, the one GPU node, its model had nowhere else to go and waited out the drain's five-minute grace, while ctl slurm status said only "1 replica(s) still to move off". It now says the time that replica is stopped at. Also recorded: Slurm on a GPU node, tried on cuda01 (gres, ConstrainDevices, accounting of gres/gpu). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
4b4306bb6a
commit
c3a75dcf3f
4 files changed
+56
-5
No files matched your search
@@ -2081,6 +2081,13 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t
|
|||||||
VMs: ayse without a job refused ("no active jobs on this node"), with one in
|
VMs: ayse without a job refused ("no active jobs on this node"), with one in
|
||||||
job_N/step_extern (nproc 1), cut off when it ended; mehmet refused; admin's session unchanged.
|
job_N/step_extern (nproc 1), cut off when it ended; mehmet refused; admin's session unchanged.
|
||||||
Left: sites whose users come from AD (sssd) are not held yet; a group of theirs would be the list.
|
Left: sites whose users come from AD (sssd) are not held yet; a group of theirs would be the list.
|
||||||
|
- A GPU node, tried 2026-11-14 on the PoC (cuda01, Quadro RTX 6000, Slurm 23.11.4): gres.conf
|
||||||
|
`Type=rtx6000 File=/dev/nvidia0`, slurm.conf `Gres=gpu:rtx6000:1`. `srun --gres=gpu:1`:
|
||||||
|
CUDA_VISIBLE_DEVICES=0, the one card; torch found it and ran 14.1 TFLOPS (fp32 matmul). Without
|
||||||
|
--gres: "No devices found", torch none (ConstrainDevices holds). `--gres=gpu:2`: refused. sacct has
|
||||||
|
`gres/gpu=1` in AllocTRES. Found: lending a node whose GPU model has nowhere else to go waits out
|
||||||
|
the drain grace (5 min) before the replica stops, and `ctl slurm status` says only "1 replica(s)
|
||||||
|
still to move off"; it should say it waits for a replacement that cannot be placed, and until when.
|
||||||
- P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm:
|
- P6, Slurm's jobs in llmgw's accounting (written 2026-11-12, not started). The tenancy tree to Slurm:
|
||||||
each project an account (`sacctmgr add account`, parent its team and organization), each member a user
|
each project an account (`sacctmgr add account`, parent its team and organization), each member a user
|
||||||
of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads
|
of it, kept by the leader from the store; `sbatch --account=PROJECT`. The records back: the leader reads
|
||||||
|
|||||||
@@ -12336,6 +12336,11 @@
|
|||||||
"replicas": {
|
"replicas": {
|
||||||
"type": "integer"
|
"type": "integer"
|
||||||
},
|
},
|
||||||
|
"stops_by": {
|
||||||
|
"type": "string",
|
||||||
|
"format": "date-time",
|
||||||
|
"description": "StopsBy is, while it is being lent, when a replica still on it is stopped even with nowhere else to go: the drain's grace after the lend began. Zero when not known (a leader that took over meanwhile)."
|
||||||
|
},
|
||||||
"slurm": {
|
"slurm": {
|
||||||
"$ref": "#/components/schemas/slurmsvc.Reported"
|
"$ref": "#/components/schemas/slurmsvc.Reported"
|
||||||
},
|
},
|
||||||
|
|||||||
@@ -27,6 +27,7 @@ import (
|
|||||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||||
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/slurm"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -62,9 +63,13 @@ const (
|
|||||||
|
|
||||||
// NodeStatus is one node as the service sees it.
|
// NodeStatus is one node as the service sees it.
|
||||||
type NodeStatus struct {
|
type NodeStatus struct {
|
||||||
Node string `json:"node"`
|
Node string `json:"node"`
|
||||||
Phase Phase `json:"phase"`
|
Phase Phase `json:"phase"`
|
||||||
Replicas int `json:"replicas"`
|
Replicas int `json:"replicas"`
|
||||||
|
// StopsBy is, while it is being lent, when a replica still on it is
|
||||||
|
// stopped even with nowhere else to go: the drain's grace after the
|
||||||
|
// lend began. Zero when not known (a leader that took over meanwhile).
|
||||||
|
StopsBy time.Time `json:"stops_by,omitempty"`
|
||||||
Status *Reported `json:"slurm,omitempty"`
|
Status *Reported `json:"slurm,omitempty"`
|
||||||
Connected bool `json:"connected"`
|
Connected bool `json:"connected"`
|
||||||
}
|
}
|
||||||
@@ -129,6 +134,8 @@ type Service struct {
|
|||||||
// drained are nodes this service put in Draining, which it alone makes
|
// drained are nodes this service put in Draining, which it alone makes
|
||||||
// ready again.
|
// ready again.
|
||||||
drained map[string]bool
|
drained map[string]bool
|
||||||
|
// lendingAt is when this leader began moving a node's replicas off.
|
||||||
|
lendingAt map[string]time.Time
|
||||||
// sent is what each node was last told, so it is told again only when
|
// sent is what each node was last told, so it is told again only when
|
||||||
// that changes or its stream is new.
|
// that changes or its stream is new.
|
||||||
sent map[string]*llmgwv1.Slurm
|
sent map[string]*llmgwv1.Slurm
|
||||||
@@ -202,7 +209,7 @@ func New(settings func() config.SlurmSettings, inv Inventory, agents Agents,
|
|||||||
log = slog.Default()
|
log = slog.Default()
|
||||||
}
|
}
|
||||||
return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log,
|
return &Service{settings: settings, inv: inv, agents: agents, key: key, log: log,
|
||||||
status: map[string]*Reported{}, drained: map[string]bool{}, sent: map[string]*llmgwv1.Slurm{},
|
status: map[string]*Reported{}, drained: map[string]bool{}, lendingAt: map[string]time.Time{}, sent: map[string]*llmgwv1.Slurm{},
|
||||||
kick: make(chan struct{}, 1)}
|
kick: make(chan struct{}, 1)}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -406,11 +413,14 @@ func (s *Service) phase(set config.SlurmSettings, node string) Phase {
|
|||||||
if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
|
if ok && (n.State == inventory.Ready || n.State == inventory.Cordoned) {
|
||||||
s.inv.SetState(node, inventory.Draining)
|
s.inv.SetState(node, inventory.Draining)
|
||||||
s.drained[node] = true
|
s.drained[node] = true
|
||||||
s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node)
|
s.lendingAt[node] = time.Now()
|
||||||
|
s.log.Info("lending a node to Slurm: its replicas are moved off first", "node", node,
|
||||||
|
"stopped_by", time.Now().Add(drainGrace).Format(time.TimeOnly))
|
||||||
}
|
}
|
||||||
if !ok || len(s.inv.Allocations(node)) > 0 {
|
if !ok || len(s.inv.Allocations(node)) > 0 {
|
||||||
return Lending
|
return Lending
|
||||||
}
|
}
|
||||||
|
delete(s.lendingAt, node)
|
||||||
return Lent
|
return Lent
|
||||||
}
|
}
|
||||||
// Not lent. Until the agent says Slurm has drained it with no job left,
|
// Not lent. Until the agent says Slurm has drained it with no job left,
|
||||||
@@ -430,12 +440,17 @@ func (s *Service) phase(set config.SlurmSettings, node string) Phase {
|
|||||||
}
|
}
|
||||||
return Reclaiming
|
return Reclaiming
|
||||||
}
|
}
|
||||||
|
delete(s.lendingAt, node)
|
||||||
if s.drained[node] {
|
if s.drained[node] {
|
||||||
s.ready(node)
|
s.ready(node)
|
||||||
}
|
}
|
||||||
return Llmgw
|
return Llmgw
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// drainGrace is how long a drained replica waits for its replacement
|
||||||
|
// before it is stopped anyway: the reconciler's, which is not a setting.
|
||||||
|
var drainGrace = reconciler.DefaultConfig().DrainGrace
|
||||||
|
|
||||||
// slurmBusy is whether Slurm may still run work on a node: a job step left
|
// slurmBusy is whether Slurm may still run work on a node: a job step left
|
||||||
// on it (or no telling), or a controller that would still give it one. A
|
// on it (or no telling), or a controller that would still give it one. A
|
||||||
// node the controller does not know of, or cannot be asked about, gets no
|
// node the controller does not know of, or cannot be asked about, gets no
|
||||||
@@ -643,6 +658,9 @@ func (s *Service) Status() Status {
|
|||||||
ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)),
|
ns := NodeStatus{Node: node, Status: s.status[node], Replicas: len(s.inv.Allocations(node)),
|
||||||
Connected: slices.Contains(conn, node)}
|
Connected: slices.Contains(conn, node)}
|
||||||
ns.Phase = s.peek(set, node)
|
ns.Phase = s.peek(set, node)
|
||||||
|
if at, ok := s.lendingAt[node]; ok && ns.Phase == Lending {
|
||||||
|
ns.StopsBy = at.Add(drainGrace)
|
||||||
|
}
|
||||||
out.Nodes = append(out.Nodes, ns)
|
out.Nodes = append(out.Nodes, ns)
|
||||||
}
|
}
|
||||||
return out
|
return out
|
||||||
@@ -670,6 +688,12 @@ func (s *Service) peek(set config.SlurmSettings, node string) Phase {
|
|||||||
func (n NodeStatus) Describe() string {
|
func (n NodeStatus) Describe() string {
|
||||||
switch n.Phase {
|
switch n.Phase {
|
||||||
case Lending:
|
case Lending:
|
||||||
|
if !n.StopsBy.IsZero() {
|
||||||
|
// A replica moves once its replacement serves; one with nowhere
|
||||||
|
// else to go waits out the grace, and that wait is not a hang.
|
||||||
|
return fmt.Sprintf("being lent: %d replica(s) still to move off; one with nowhere else to go is stopped at %s",
|
||||||
|
n.Replicas, n.StopsBy.Local().Format(time.TimeOnly))
|
||||||
|
}
|
||||||
return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas)
|
return fmt.Sprintf("being lent: %d replica(s) still to move off", n.Replicas)
|
||||||
case Lent:
|
case Lent:
|
||||||
return "lent to Slurm"
|
return "lent to Slurm"
|
||||||
|
|||||||
@@ -5,6 +5,7 @@ import (
|
|||||||
"slices"
|
"slices"
|
||||||
"strings"
|
"strings"
|
||||||
"testing"
|
"testing"
|
||||||
|
"time"
|
||||||
|
|
||||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||||
@@ -272,3 +273,17 @@ func TestAllNodes(t *testing.T) {
|
|||||||
t.Fatalf("cuda02 not lent: %+v", p)
|
t.Fatalf("cuda02 not lent: %+v", p)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestLendingSaysWhenAReplicaWithNowhereToGoStops(t *testing.T) {
|
||||||
|
// Seen on the PoC: a GPU model with no other card waited out the drain's
|
||||||
|
// grace, and five minutes of "1 replica(s) still to move off" read as a hang.
|
||||||
|
at := time.Date(2026, 11, 14, 19, 38, 34, 0, time.Local)
|
||||||
|
n := NodeStatus{Node: "cuda01", Phase: Lending, Replicas: 1, StopsBy: at}
|
||||||
|
if d := n.Describe(); !strings.Contains(d, "nowhere else to go is stopped at 19:38:34") {
|
||||||
|
t.Errorf("%q", d)
|
||||||
|
}
|
||||||
|
n.StopsBy = time.Time{}
|
||||||
|
if d := n.Describe(); d != "being lent: 1 replica(s) still to move off" {
|
||||||
|
t.Errorf("%q", d)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user