Track which replicas are actually being used, and summarise the fleet
Two things a cluster with many nodes and many GPUs needs that were missing, and they are the same thing seen from two distances. Idle tracking. The control plane knew what was placed and whether it was healthy, never whether anyone wanted it — so on a fleet with more models than cards there was no way to answer the first question anyone asks, which is what can I stop. The router is the only thing that knows, so it now reports it: a timestamp written in Acquire, where a request already meets an endpoint under a lock already held, and collected once a tick as a map. That shape is the point. Per-request notification would be a firehose carrying one bit — the newest timestamp wins — and this costs the same on four replicas as on four hundred. It travels through an interface in the direction opposite EndpointSink, for the same reason that one exists: the control plane decides what should run, the router knows whether anyone wants it, and neither should need to be in the other's process to say so. Unknown is kept distinct from idle throughout. A router that has just restarted knows nothing, and a zero timestamp would be the oldest idle replica in every comparison — so one restart would nominate an entire fleet for eviction. Absent from the map, "-" in the table, negative in the JSON, and IdleFor returns a second value rather than a duration anything could act on by mistake. ctl status. A per-node listing is right for four nodes and wrong for two hundred: how full am I, what is broken, what is holding cards nobody is using are aggregates, and reading them off a long table is a chore, which means it gets skipped. Capacity per GPU class with usage bars, free whole nodes per fabric — the number that decides whether a multi-node replica can land, since free cards on the wrong fabric are not capacity for a model that spans nodes — replica and model counts, models that cannot be placed, and what has gone unasked-for. Derived from the same state the tables show, so it cannot disagree with them. Unhealthy GPUs count as neither used nor free, because counting them free is how a cluster looks like it has room it does not. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1431426db0
commit
9d8cd5e284
15 files changed
+928
-20
No files matched your search
@@ -299,6 +299,47 @@ func (r *Reconciler) fleetHasSettled() bool {
|
||||
return settled
|
||||
}
|
||||
|
||||
// RecordUsage stamps replicas with when each was last sent a request.
|
||||
//
|
||||
// Replicas absent from the map are left alone rather than zeroed: a router that
|
||||
// has just restarted knows nothing about what happened before it, and treating
|
||||
// that as "never used" would make every replica look abandoned at once —
|
||||
// exactly when a fleet is least able to afford acting on that.
|
||||
func (r *Reconciler) RecordUsage(lastUsed map[string]time.Time) {
|
||||
if len(lastUsed) == 0 {
|
||||
return
|
||||
}
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
for id, at := range lastUsed {
|
||||
rep, ok := r.replicas[id]
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
// Only forwards. Two routers reporting, or a stale snapshot arriving
|
||||
// late, must not move a replica's last use backwards.
|
||||
if at.After(rep.LastUsedAt) {
|
||||
rep.LastUsedAt = at
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// IdleFor reports how long a replica has gone without a request, and whether
|
||||
// that is known at all.
|
||||
//
|
||||
// The second return is not a detail. "Idle for an hour" and "nobody has told me
|
||||
// yet" are different states, and anything that acts on idleness — eviction,
|
||||
// most obviously — has to refuse to act on the second.
|
||||
func (r *Reconciler) IdleFor(id string) (time.Duration, bool) {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
rep, ok := r.replicas[id]
|
||||
if !ok || rep.LastUsedAt.IsZero() {
|
||||
return 0, false
|
||||
}
|
||||
return r.now().Sub(rep.LastUsedAt), true
|
||||
}
|
||||
|
||||
// expireStuckReplicas fails replicas that never finished coming up.
|
||||
//
|
||||
// Every other path out of Starting depends on the agent reporting something. A
|
||||
|
||||
@@ -1153,3 +1153,79 @@ func TestSettlingOutlastsAnAgentReconnectBackoff(t *testing.T) {
|
||||
t.Fatal("placement never resumed")
|
||||
}
|
||||
}
|
||||
|
||||
// --- idle tracking ---
|
||||
|
||||
func TestUsageIsRecordedAgainstReplicas(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("expected a replica, got %d", len(reps))
|
||||
}
|
||||
id := reps[0].ID
|
||||
|
||||
if _, known := h.r.IdleFor(id); known {
|
||||
t.Error("a replica nothing has used reported a known idleness")
|
||||
}
|
||||
|
||||
used := h.clock.now()
|
||||
h.r.RecordUsage(map[string]time.Time{id: used})
|
||||
h.advance(5 * time.Minute)
|
||||
|
||||
idle, known := h.r.IdleFor(id)
|
||||
if !known {
|
||||
t.Fatal("usage was recorded but idleness is still unknown")
|
||||
}
|
||||
if idle != 5*time.Minute {
|
||||
t.Errorf("idle = %s, want 5m", idle)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsageNeverMovesBackwards(t *testing.T) {
|
||||
// Two routers reporting, or a snapshot arriving late, must not make a busy
|
||||
// replica look abandoned.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
id := h.r.Replicas()[0].ID
|
||||
|
||||
recent := h.clock.now()
|
||||
h.r.RecordUsage(map[string]time.Time{id: recent})
|
||||
h.r.RecordUsage(map[string]time.Time{id: recent.Add(-time.Hour)})
|
||||
|
||||
h.advance(time.Minute)
|
||||
if idle, _ := h.r.IdleFor(id); idle != time.Minute {
|
||||
t.Errorf("idle = %s; a stale report moved last-use backwards", idle)
|
||||
}
|
||||
}
|
||||
|
||||
func TestUsageForReplicasWeDoNotHaveIsIgnored(t *testing.T) {
|
||||
// The router's list and the reconciler's drift constantly on a live fleet:
|
||||
// a replica stops between the snapshot and the read. That is ordinary, not
|
||||
// an error, and must not panic or create phantom state.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
before := len(h.r.Replicas())
|
||||
|
||||
h.r.RecordUsage(map[string]time.Time{"a-replica-that-left": h.clock.now()})
|
||||
if got := len(h.r.Replicas()); got != before {
|
||||
t.Errorf("replica count changed from %d to %d", before, got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnEmptyUsageReportLeavesEverythingAlone(t *testing.T) {
|
||||
// A router that has just started knows nothing, and treating that as "never
|
||||
// used" would make every replica look abandoned at once — exactly when a
|
||||
// fleet is least able to afford acting on it.
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
id := h.r.Replicas()[0].ID
|
||||
|
||||
h.r.RecordUsage(map[string]time.Time{id: h.clock.now()})
|
||||
h.advance(time.Minute)
|
||||
h.r.RecordUsage(nil)
|
||||
|
||||
if idle, known := h.r.IdleFor(id); !known || idle != time.Minute {
|
||||
t.Errorf("idle=%s known=%v; an empty report erased what was known", idle, known)
|
||||
}
|
||||
}
|
||||
@@ -104,6 +104,14 @@ type Replica struct {
|
||||
|
||||
CreatedAt time.Time
|
||||
ReadyAt time.Time
|
||||
// LastUsedAt is when a request was last routed to this replica, zero if
|
||||
// none has been since the router started.
|
||||
//
|
||||
// The difference between a replica that is holding eight cards for work and
|
||||
// one that is holding them out of habit. Nothing recorded it, so on a fleet
|
||||
// with more models than GPUs there was no way to answer the first question
|
||||
// anyone asks: what can I stop?
|
||||
LastUsedAt time.Time
|
||||
// DrainingAt is when the replica stopped taking new work, so the wait for a
|
||||
// replacement can be bounded.
|
||||
DrainingAt time.Time
|
||||
|
||||
Reference in new issue
Block a user