A model waiting for an on-demand one to step aside is waiting, not unplaceable
A 3D job asked for straight after a picture, on a cluster with one card, failed at once with "cannot be placed": flux held the card, too freshly started to evict. The cards coming back when an on-demand model goes idle makes it a wait, so the request is told to retry and the job waits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
b743b11c54
commit
d0f7b1fc4f
2 files changed
+62
-5
No files matched your search
@@ -134,21 +134,39 @@ func (r *Reconciler) Unservable(model string) (string, bool) {
|
|||||||
if !ok {
|
if !ok {
|
||||||
return "", false
|
return "", false
|
||||||
}
|
}
|
||||||
|
cat := r.cat.Load()
|
||||||
r.mu.Lock()
|
r.mu.Lock()
|
||||||
defer r.mu.Unlock()
|
|
||||||
if p, ok := r.parked[m.Name]; ok {
|
if p, ok := r.parked[m.Name]; ok {
|
||||||
|
r.mu.Unlock()
|
||||||
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
|
return fmt.Sprintf("%s failed to start %d times in a row and is not being retried: %s",
|
||||||
m.Name, p.Failures, p.Reason), true
|
m.Name, p.Failures, p.Reason), true
|
||||||
}
|
}
|
||||||
|
var asleepLater []victim
|
||||||
for _, rep := range r.replicas {
|
for _, rep := range r.replicas {
|
||||||
if rep.Model == m.Name && rep.State.Counts() {
|
if !rep.State.Counts() {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if rep.Model == m.Name {
|
||||||
|
r.mu.Unlock()
|
||||||
return "", false // something is on its way
|
return "", false // something is on its way
|
||||||
}
|
}
|
||||||
|
if other, ok := cat.Resolve(rep.Model); ok && other.Scaling.OnDemand() && len(nodesOf(rep)) == 1 {
|
||||||
|
asleepLater = append(asleepLater, victim{ID: rep.ID, Model: rep.Model})
|
||||||
|
}
|
||||||
}
|
}
|
||||||
if res, ok := r.lastExplain[m.Name]; ok && res.Placement == nil {
|
res, ok := r.lastExplain[m.Name]
|
||||||
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
|
r.mu.Unlock()
|
||||||
|
if !ok || res.Placement != nil {
|
||||||
|
return "", false
|
||||||
}
|
}
|
||||||
return "", false
|
// The cards are held by models that let go of them when idle, which
|
||||||
|
// makeRoomFor hastens once they have been up a while. That is a wait, not
|
||||||
|
// a never: a 3D model asked for straight after a picture, on a cluster
|
||||||
|
// with one card, waits for the picture model to step aside.
|
||||||
|
if m.Scaling.OnDemand() && len(asleepLater) > 0 && r.fitsWithout(m, nil, asleepLater) {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
return m.Name + " cannot be placed on any node: " + firstLine(res.Explain()), true
|
||||||
}
|
}
|
||||||
|
|
||||||
func firstLine(s string) string {
|
func firstLine(s string) string {
|
||||||
|
|||||||
@@ -1708,6 +1708,45 @@ func TestAFreshlyStartedReplicaIsNotEvicted(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestAModelWaitingForAnotherOnDemandOneToStepAsideIsNotUnservable(t *testing.T) {
|
||||||
|
// A 3D model asked for straight after a picture, on a cluster with one
|
||||||
|
// card: the picture model holds it, too freshly started to evict. That
|
||||||
|
// is a wait — told "cannot be placed", a job failed at once.
|
||||||
|
h := newHarness(t, testModel(1), gpuNode("1", 4))
|
||||||
|
h.r.cat.Store(twoOnDemandModels(t))
|
||||||
|
h.r.Want("llama")
|
||||||
|
h.r.Reconcile(t.Context())
|
||||||
|
h.markReady()
|
||||||
|
reps := h.r.Replicas()
|
||||||
|
if len(reps) != 1 {
|
||||||
|
t.Fatalf("setup: %d replicas", len(reps))
|
||||||
|
}
|
||||||
|
h.r.RecordUsage(map[string]time.Time{reps[0].ID: h.clock.now()})
|
||||||
|
|
||||||
|
h.r.Want("second")
|
||||||
|
h.r.Reconcile(t.Context())
|
||||||
|
if res, ok := h.r.lastExplain["second"]; !ok || res.Placement != nil {
|
||||||
|
t.Fatalf("setup: second was not left unplaced: %+v", res)
|
||||||
|
}
|
||||||
|
if why, no := h.r.Unservable("second"); no {
|
||||||
|
t.Errorf("a model waiting for an on-demand one to go idle was called unservable: %q", why)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Held by a model that stays up, the card is not coming back.
|
||||||
|
always := testModel(1)
|
||||||
|
always.Name = "llama"
|
||||||
|
always.Scaling = catalog.Scaling{Min: 1, Max: 1}
|
||||||
|
second := mustResolve(t, h, "second")
|
||||||
|
built, err := catalog.New([]catalog.Model{always, second})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
h.r.cat.Store(built)
|
||||||
|
if _, no := h.r.Unservable("second"); !no {
|
||||||
|
t.Error("a model behind an always-on one was not called unservable")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestAStartedShardIsToldWhichProfileVersionItIs(t *testing.T) {
|
func TestAStartedShardIsToldWhichProfileVersionItIs(t *testing.T) {
|
||||||
// The fingerprint has to leave the control plane, or it cannot come back
|
// The fingerprint has to leave the control plane, or it cannot come back
|
||||||
// after a restart — and a restart is exactly when the configuration is
|
// after a restart — and a restart is exactly when the configuration is
|
||||||
|
|||||||
Reference in new issue
Block a user