A big model's download no longer runs out its startup timeout
CI / test (push) Successful in 5m18s

The startup timeout runs from when a node's download last moved, so a
model of hundreds of gigabytes on a slow mirror is not failed while it
arrives. ctl model serve --env sets engine variables (NCCL_DEBUG=INFO)
on every node, and a layout given by hand keeps expert parallelism for a
model of experts. docs/large-models.md: how a model is split, what each
way needs of a card, and a checklist for bringing one up on a fabric.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-20 09:45:31 +02:00
1 parent 4fe9b46451
commit 8ffde2b80c
12 files changed
+259 -4

No files matched your search

+8 -1
View File
@@ -582,7 +582,11 @@ func (r *Reconciler) expireStuckReplicas() {
if rep.State != Starting && rep.State != Placing {
continue
}
if now.Sub(rep.CreatedAt) < r.startupTimeout(rep) {
since := rep.CreatedAt
if rep.fetchedAt.After(since) {
since = rep.fetchedAt
}
if now.Sub(since) < r.startupTimeout(rep) {
continue
}
rep.State = Failed
@@ -1328,6 +1332,9 @@ func (r *Reconciler) OnFetchProgress(replicaID, shardID, source string, done, to
if rep.Shards[i].ID != shardID {
continue
}
if prev := rep.Shards[i].Fetch; prev == nil || done > prev.BytesDone {
rep.fetchedAt = r.now()
}
rep.Shards[i].Fetch = &FetchProgress{
Source: source, BytesDone: done, BytesTotal: total,
}
+5
View File
@@ -160,6 +160,11 @@ type Replica struct {
AdoptedAt time.Time
CreatedAt time.Time
// fetchedAt is when a shard's download last moved: the startup timeout
// runs from then, so a model of hundreds of gigabytes on a slow mirror
// is not failed while it is still arriving, and one whose download
// stalls is failed a timeout after it stopped.
fetchedAt time.Time
ReadyAt time.Time
// LastUsedAt is when a request was last routed to this replica, zero if
// none has been since the router started.
+31
View File
@@ -406,3 +406,34 @@ func TestAMultiNodeReplicaKnowsHowManyShardsItWants(t *testing.T) {
t.Errorf("WantShards = %d, want 2", reps[0].WantShards)
}
}
// A model of hundreds of gigabytes on a slow mirror takes longer to arrive
// than to load: the startup timeout runs from when its download last moved,
// and a download that stops is failed a timeout after it stopped.
func TestAReplicaStillDownloadingIsNotFailed(t *testing.T) {
m := testModel(1, catalog.Profile{
Name: "tp4", GPUClass: "h100-80g", TP: 4, StartupTimeout: 20 * time.Minute,
})
h := newHarness(t, m, gpuNode("1", 8))
h.r.Reconcile(context.Background())
rep := h.r.Replicas()[0]
shard := rep.Shards[0].ID
// Forty minutes of download, moving all the while.
for i := int64(1); i <= 8; i++ {
h.advance(5 * time.Minute)
h.r.OnFetchProgress(rep.ID, shard, "mirror", i<<35, 400<<30)
h.r.expireStuckReplicas()
}
if got := h.r.Replicas()[0].State; got == Failed {
t.Fatal("a replica whose download kept moving was failed")
}
// The same figure again is not progress: stalled, it fails a timeout on.
h.advance(10 * time.Minute)
h.r.OnFetchProgress(rep.ID, shard, "mirror", 8<<35, 400<<30)
h.advance(11 * time.Minute)
h.r.expireStuckReplicas()
if got := h.r.Replicas()[0].State; got != Failed {
t.Errorf("a stalled download was never failed: %s", got)
}
}