A job waiting for the card says what holds it and about how long
CI / test (push) Failing after 6m16s

A 3D job waiting for the picture model to give up the one card had an empty
progress; its client could not tell waiting from stuck. The control plane
says which model holds the cards and when it lets go, plus the waiting
model's last start; the 503 carries it as headers, and a job keeps it as
progress {state: waiting_gpu, held_by, eta_seconds}.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-10 22:25:27 +02:00
1 parent cf2edfd995
commit b1490dbf30
12 files changed
+252 -8

No files matched your search

+79
View File
@@ -169,6 +169,85 @@ func (r *Reconciler) Unservable(model string) (string, bool) {
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
}
// WaitFor says which model holds the cards an on-demand model is waiting
// for, and about how long until it serves: until the holder lets go — evicted
// once idle and old enough, or stopped at its own idle timeout — and then its
// own start, as long as its last one took. ok is false when it is not waiting
// on another model; eta is zero when when the holder lets go is not known.
func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok bool) {
cat := r.cat.Load()
m, found := cat.Resolve(model)
if !found || !m.Scaling.OnDemand() {
return "", 0, false
}
type holder struct {
v victim
release time.Time
}
now := r.now()
var holders []holder
r.mu.Lock()
for _, rep := range r.replicas {
if !rep.State.Counts() || len(nodesOf(rep)) != 1 {
continue
}
if rep.Model == m.Name {
r.mu.Unlock()
return "", 0, false // its own replica is on its way
}
other, found := cat.Resolve(rep.Model)
if !found || !other.Scaling.OnDemand() {
continue
}
h := holder{v: victim{ID: rep.ID, Model: rep.Model}}
if !rep.LastUsedAt.IsZero() {
evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace)
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) {
evict = rep.ReadyAt.Add(r.cfg.MinLifetime)
}
h.release = evict
if idle := rep.LastUsedAt.Add(r.idleTimeout(other)); idle.Before(evict) {
h.release = idle
}
}
holders = append(holders, h)
}
cold := r.coldStart[m.Name]
r.mu.Unlock()
// The one whose going alone makes room, letting go soonest; failing
// that, all of them, and the last to let go.
var best *holder
for i := range holders {
h := &holders[i]
if !r.fitsWithout(m, nil, []victim{h.v}) {
continue
}
if best == nil || (!h.release.IsZero() && (best.release.IsZero() || h.release.Before(best.release))) {
best = h
}
}
if best == nil {
all := make([]victim, len(holders))
for i, h := range holders {
all[i] = h.v
}
if len(all) == 0 || !r.fitsWithout(m, nil, all) {
return "", 0, false
}
for i := range holders {
h := &holders[i]
if best == nil || h.release.IsZero() || (!best.release.IsZero() && h.release.After(best.release)) {
best = h
}
}
}
if best.release.IsZero() {
return best.v.Model, 0, true
}
return best.v.Model, max(best.release.Sub(now), 0) + cold, true
}
func firstLine(s string) string {
s = strings.TrimSpace(s)
if i := strings.IndexByte(s, '\n'); i >= 0 {
@@ -264,6 +264,9 @@ type Reconciler struct {
// because it is state about this moment and not about what the model is.
// A model nobody has wanted has no entry and is simply not running.
wanted map[string]time.Time
// coldStart is how long each model's last replica took from placed to
// ready, for telling a request waiting on it about how long it waits.
coldStart map[string]time.Duration
// restartQueue holds, per model, the replicas an operator asked to be
// restarted and not yet begun. One is replaced at a time: replacing them
// all at once would need the memory of every replica twice over.
@@ -330,6 +333,7 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l
attempts: map[string]int{},
reservedPorts: map[int]struct{}{},
wanted: map[string]time.Time{},
coldStart: map[string]time.Duration{},
restartQueue: map[string][]string{},
restartInPlace: map[string]bool{},
restartPace: map[string]RestartPace{},
@@ -1371,6 +1375,9 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e
// counting from scratch rather than from an old streak.
r.attempts[rep.Model] = 0
delete(r.nextAttempt, rep.Model)
if !rep.CreatedAt.IsZero() {
r.coldStart[rep.Model] = r.now().Sub(rep.CreatedAt)
}
}
}
@@ -1745,6 +1745,33 @@ func TestAnEvictedModelDoesNotTakeTheCardBack(t *testing.T) {
}
}
func TestAModelWaitingOnAnotherIsToldWhichAndForAboutHowLong(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 4))
h.r.cat.Store(twoOnDemandModels(t))
h.r.Want("llama")
h.r.Reconcile(t.Context())
h.markReady()
reps := h.r.Replicas()
if len(reps) != 1 {
t.Fatalf("setup: %d replicas", len(reps))
}
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
h.advance(time.Minute)
h.r.Want("second")
h.r.Reconcile(t.Context())
// Evicted once it has been up MinLifetime, a minute of which has gone;
// second has never started, so there is no start of its to add.
by, eta, ok := h.r.WaitFor("second")
if !ok || by != "llama" || eta != DefaultConfig().MinLifetime-time.Minute {
t.Errorf("WaitFor = %q %s %v", by, eta, ok)
}
// The model holding the cards waits on nobody.
if _, _, ok := h.r.WaitFor("llama"); ok {
t.Error("a model with its own replica was said to be waiting")
}
}
func TestAModelWaitingForAnotherOnDemandOneToStepAsideIsNotUnservable(t *testing.T) {
// A 3D model asked for straight after a picture, on a cluster with one
// card: the picture model holds it, too freshly started to evict. That