The startup timeout runs from when a node's download last moved, so a model of hundreds of gigabytes on a slow mirror is not failed while it arrives. ctl model serve --env sets engine variables (NCCL_DEBUG=INFO) on every node, and a layout given by hand keeps expert parallelism for a model of experts. docs/large-models.md: how a model is split, what each way needs of a card, and a checklist for bringing one up on a fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4fe9b46451
commit
8ffde2b80c
12 files changed
+259
-4
No files matched your search
@@ -582,7 +582,11 @@ func (r *Reconciler) expireStuckReplicas() {
|
||||
if rep.State != Starting && rep.State != Placing {
|
||||
continue
|
||||
}
|
||||
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
|
||||
since := rep.CreatedAt
|
||||
if rep.fetchedAt.After(since) {
|
||||
since = rep.fetchedAt
|
||||
}
|
||||
if now.Sub(since) < r.startupTimeout(rep) {
|
||||
continue
|
||||
}
|
||||
rep.State = Failed
|
||||
@@ -1328,6 +1332,9 @@ func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, to
|
||||
if rep.Shards[i].ID != shardID {
|
||||
continue
|
||||
}
|
||||
if prev := rep.Shards[i].Fetch; prev == nil || done > prev.BytesDone {
|
||||
rep.fetchedAt = r.now()
|
||||
}
|
||||
rep.Shards[i].Fetch = &FetchProgress{
|
||||
Source: source, BytesDone: done, BytesTotal: total,
|
||||
}
|
||||
|
||||
@@ -160,6 +160,11 @@ type Replica struct {
|
||||
AdoptedAt time.Time
|
||||
|
||||
CreatedAt time.Time
|
||||
// fetchedAt is when a shard's download last moved: the startup timeout
|
||||
// runs from then, so a model of hundreds of gigabytes on a slow mirror
|
||||
// is not failed while it is still arriving, and one whose download
|
||||
// stalls is failed a timeout after it stopped.
|
||||
fetchedAt time.Time
|
||||
ReadyAt time.Time
|
||||
// LastUsedAt is when a request was last routed to this replica, zero if
|
||||
// none has been since the router started.
|
||||
|
||||
@@ -406,3 +406,34 @@ func TestAMultiNodeReplicaKnowsHowManyShardsItWants(t *testing.T) {
|
||||
t.Errorf("WantShards = %d, want 2", reps[0].WantShards)
|
||||
}
|
||||
}
|
||||
|
||||
// A model of hundreds of gigabytes on a slow mirror takes longer to arrive
|
||||
// than to load: the startup timeout runs from when its download last moved,
|
||||
// and a download that stops is failed a timeout after it stopped.
|
||||
func TestAReplicaStillDownloadingIsNotFailed(t *testing.T) {
|
||||
m := testModel(1, catalog.Profile{
|
||||
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 20 * time.Minute,
|
||||
})
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
h.r.Reconcile(context.Background())
|
||||
rep := h.r.Replicas()[0]
|
||||
shard := rep.Shards[0].ID
|
||||
|
||||
// Forty minutes of download, moving all the while.
|
||||
for i := int64(1); i <= 8; i++ {
|
||||
h.advance(5 * time.Minute)
|
||||
h.r.OnFetchProgress(rep.ID, shard, "mirror", i<<35, 400<<30)
|
||||
h.r.expireStuckReplicas()
|
||||
}
|
||||
if got := h.r.Replicas()[0].State; got == Failed {
|
||||
t.Fatal("a replica whose download kept moving was failed")
|
||||
}
|
||||
// The same figure again is not progress: stalled, it fails a timeout on.
|
||||
h.advance(10 * time.Minute)
|
||||
h.r.OnFetchProgress(rep.ID, shard, "mirror", 8<<35, 400<<30)
|
||||
h.advance(11 * time.Minute)
|
||||
h.r.expireStuckReplicas()
|
||||
if got := h.r.Replicas()[0].State; got != Failed {
|
||||
t.Errorf("a stalled download was never failed: %s", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user