A big model's download no longer runs out its startup timeout
The startup timeout runs from when a node's download last moved, so a model of hundreds of gigabytes on a slow mirror is not failed while it arrives. ctl model serve --env sets engine variables (NCCL_DEBUG=INFO) on every node, and a layout given by hand keeps expert parallelism for a model of experts. docs/large-models.md: how a model is split, what each way needs of a card, and a checklist for bringing one up on a fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4fe9b46451
commit
8ffde2b80c
12 files changed
+259
-4
No files matched your search
@@ -475,3 +475,26 @@ func TestAJobAddingOnlyIfAbsentGivesWayToAModelAddedMeanwhile(t *testing.T) {
|
||||
t.Errorf("job %s, added %d: %+v", j.State, len(taken.added), j.Steps)
|
||||
}
|
||||
}
|
||||
|
||||
// A model of experts given its cards by hand spreads them still, and the
|
||||
// environment asked for reaches every profile.
|
||||
func TestAHandGivenLayoutOfExpertsKeepsExpertParallelAndTheEnvironment(t *testing.T) {
|
||||
plan := Plan{Repo: "Qwen/Qwen3-235B-A22B", Experts: 128}
|
||||
req := Request{Format: "safetensors", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, ContextLength: 8192,
|
||||
Env: map[string]string{"NCCL_DEBUG": "INFO"}}
|
||||
layout, err := chooseLayout(req, plan, true, hf.FileSet{Name: "original"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
profs := profilesFor(layout, req, 0, 0)
|
||||
p := profs[0]
|
||||
if p.TP != 8 || p.PP != 2 || p.Nodes != 2 || !slices.Contains(p.Args, "--enable-expert-parallel") || p.Env["NCCL_DEBUG"] != "INFO" {
|
||||
t.Errorf("profile %+v", p)
|
||||
}
|
||||
// A dense model given the same cards does not.
|
||||
plan.Experts = 0
|
||||
layout, _ = chooseLayout(req, plan, true, hf.FileSet{Name: "original"})
|
||||
if slices.Contains(profilesFor(layout, req, 0, 0)[0].Args, "--enable-expert-parallel") {
|
||||
t.Error("a dense model was given expert parallelism")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user