A job waiting for the card says what holds it and about how long
CI / test (push) Failing after 6m16s
CI / test (push) Failing after 6m16s
A 3D job waiting for the picture model to give up the one card had an empty
progress; its client could not tell waiting from stuck. The control plane
says which model holds the cards and when it lets go, plus the waiting
model's last start; the 503 carries it as headers, and a job keeps it as
progress {state: waiting_gpu, held_by, eta_seconds}.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
cf2edfd995
commit
b1490dbf30
12 files changed
+252
-8
No files matched your search
@@ -169,6 +169,85 @@ func (r *Reconciler) Unservable(model string) (string, bool) {
|
||||
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
|
||||
}
|
||||
|
||||
// WaitFor says which model holds the cards an on-demand model is waiting
|
||||
// for, and about how long until it serves: until the holder lets go — evicted
|
||||
// once idle and old enough, or stopped at its own idle timeout — and then its
|
||||
// own start, as long as its last one took. ok is false when it is not waiting
|
||||
// on another model; eta is zero when when the holder lets go is not known.
|
||||
func (r *Reconciler) WaitFor(model string) (heldBy string, eta time.Duration, ok bool) {
|
||||
cat := r.cat.Load()
|
||||
m, found := cat.Resolve(model)
|
||||
if !found || !m.Scaling.OnDemand() {
|
||||
return "", 0, false
|
||||
}
|
||||
type holder struct {
|
||||
v victim
|
||||
release time.Time
|
||||
}
|
||||
now := r.now()
|
||||
var holders []holder
|
||||
r.mu.Lock()
|
||||
for _, rep := range r.replicas {
|
||||
if !rep.State.Counts() || len(nodesOf(rep)) != 1 {
|
||||
continue
|
||||
}
|
||||
if rep.Model == m.Name {
|
||||
r.mu.Unlock()
|
||||
return "", 0, false // its own replica is on its way
|
||||
}
|
||||
other, found := cat.Resolve(rep.Model)
|
||||
if !found || !other.Scaling.OnDemand() {
|
||||
continue
|
||||
}
|
||||
h := holder{v: victim{ID: rep.ID, Model: rep.Model}}
|
||||
if !rep.LastUsedAt.IsZero() {
|
||||
evict := rep.LastUsedAt.Add(r.cfg.EvictionGrace)
|
||||
if !rep.ReadyAt.IsZero() && rep.ReadyAt.Add(r.cfg.MinLifetime).After(evict) {
|
||||
evict = rep.ReadyAt.Add(r.cfg.MinLifetime)
|
||||
}
|
||||
h.release = evict
|
||||
if idle := rep.LastUsedAt.Add(r.idleTimeout(other)); idle.Before(evict) {
|
||||
h.release = idle
|
||||
}
|
||||
}
|
||||
holders = append(holders, h)
|
||||
}
|
||||
cold := r.coldStart[m.Name]
|
||||
r.mu.Unlock()
|
||||
|
||||
// The one whose going alone makes room, letting go soonest; failing
|
||||
// that, all of them, and the last to let go.
|
||||
var best *holder
|
||||
for i := range holders {
|
||||
h := &holders[i]
|
||||
if !r.fitsWithout(m, nil, []victim{h.v}) {
|
||||
continue
|
||||
}
|
||||
if best == nil || (!h.release.IsZero() && (best.release.IsZero() || h.release.Before(best.release))) {
|
||||
best = h
|
||||
}
|
||||
}
|
||||
if best == nil {
|
||||
all := make([]victim, len(holders))
|
||||
for i, h := range holders {
|
||||
all[i] = h.v
|
||||
}
|
||||
if len(all) == 0 || !r.fitsWithout(m, nil, all) {
|
||||
return "", 0, false
|
||||
}
|
||||
for i := range holders {
|
||||
h := &holders[i]
|
||||
if best == nil || h.release.IsZero() || (!best.release.IsZero() && h.release.After(best.release)) {
|
||||
best = h
|
||||
}
|
||||
}
|
||||
}
|
||||
if best.release.IsZero() {
|
||||
return best.v.Model, 0, true
|
||||
}
|
||||
return best.v.Model, max(best.release.Sub(now), 0) + cold, true
|
||||
}
|
||||
|
||||
func firstLine(s string) string {
|
||||
s = strings.TrimSpace(s)
|
||||
if i := strings.IndexByte(s, '\n'); i >= 0 {
|
||||
|
||||
@@ -264,6 +264,9 @@ type Reconciler struct {
|
||||
// because it is state about this moment and not about what the model is.
|
||||
// A model nobody has wanted has no entry and is simply not running.
|
||||
wanted map[string]time.Time
|
||||
// coldStart is how long each model's last replica took from placed to
|
||||
// ready, for telling a request waiting on it about how long it waits.
|
||||
coldStart map[string]time.Duration
|
||||
// restartQueue holds, per model, the replicas an operator asked to be
|
||||
// restarted and not yet begun. One is replaced at a time: replacing them
|
||||
// all at once would need the memory of every replica twice over.
|
||||
@@ -330,6 +333,7 @@ func New(cfg Config, cat *catalog.Live, inv *inventory.Store, disp Dispatcher, l
|
||||
attempts: map[string]int{},
|
||||
reservedPorts: map[int]struct{}{},
|
||||
wanted: map[string]time.Time{},
|
||||
coldStart: map[string]time.Duration{},
|
||||
restartQueue: map[string][]string{},
|
||||
restartInPlace: map[string]bool{},
|
||||
restartPace: map[string]RestartPace{},
|
||||
@@ -1371,6 +1375,9 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e
|
||||
// counting from scratch rather than from an old streak.
|
||||
r.attempts[rep.Model] = 0
|
||||
delete(r.nextAttempt, rep.Model)
|
||||
if !rep.CreatedAt.IsZero() {
|
||||
r.coldStart[rep.Model] = r.now().Sub(rep.CreatedAt)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1745,6 +1745,33 @@ func TestAnEvictedModelDoesNotTakeTheCardBack(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelWaitingOnAnotherIsToldWhichAndForAboutHowLong(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
||||
h.r.cat.Store(twoOnDemandModels(t))
|
||||
h.r.Want("llama")
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
reps := h.r.Replicas()
|
||||
if len(reps) != 1 {
|
||||
t.Fatalf("setup: %d replicas", len(reps))
|
||||
}
|
||||
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
||||
h.advance(time.Minute)
|
||||
|
||||
h.r.Want("second")
|
||||
h.r.Reconcile(t.Context())
|
||||
// Evicted once it has been up MinLifetime, a minute of which has gone;
|
||||
// second has never started, so there is no start of its to add.
|
||||
by, eta, ok := h.r.WaitFor("second")
|
||||
if !ok || by != "llama" || eta != DefaultConfig().MinLifetime-time.Minute {
|
||||
t.Errorf("WaitFor = %q %s %v", by, eta, ok)
|
||||
}
|
||||
// The model holding the cards waits on nobody.
|
||||
if _, _, ok := h.r.WaitFor("llama"); ok {
|
||||
t.Error("a model with its own replica was said to be waiting")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAModelWaitingForAnotherOnDemandOneToStepAsideIsNotUnservable(t *testing.T) {
|
||||
// A 3D model asked for straight after a picture, on a cluster with one
|
||||
// card: the picture model holds it, too freshly started to evict. That
|
||||
|
||||
Reference in new issue
Block a user