A big model's download no longer runs out its startup timeout
CI / test (push) Successful in 5m18s

The startup timeout runs from when a node's download last moved, so a
model of hundreds of gigabytes on a slow mirror is not failed while it
arrives. ctl model serve --env sets engine variables (NCCL_DEBUG=INFO)
on every node, and a layout given by hand keeps expert parallelism for a
model of experts. docs/large-models.md: how a model is split, what each
way needs of a card, and a checklist for bringing one up on a fabric.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-20 09:45:31 +02:00
1 parent 4fe9b46451
commit 8ffde2b80c
12 files changed
+259 -4

No files matched your search

+7 -1
View File
@@ -163,6 +163,10 @@ type Request struct {
// CPUCores is what a replica computes on; zero takes half the smallest
// node it fits on.
CPUCores int `json:"cpu_cores,omitempty"`
// Env is set in the engine's environment, on every node of a replica:
// NCCL_DEBUG=INFO to see how its cards talk, NCCL_IB_* to steer them.
// Only the variables engines are tuned with; the catalog refuses others.
Env map[string]string `json:"env,omitempty"`
// IfAbsent adds the model only if nobody has meanwhile: the leader's own
// serving of a judge a policy names gives way to the one init or an
// operator added while it fetched, rather than replacing it.
@@ -580,6 +584,8 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
o = p
}
}
// A way the plan did not offer is still a model of experts.
o.ExpertParallel = engine == "vllm" && plan.Experts > 1 && o.GPUs() > 1
out = append(out, o)
case len(opts) > 0:
out = append(out, opts[0])
@@ -614,7 +620,7 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile {
var out []catalog.Profile
for i, o := range layout {
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i}
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i, Env: req.Env}
switch {
case o.CPU:
p.Name = "cpu"
@@ -475,3 +475,26 @@ func TestAJobAddingOnlyIfAbsentGivesWayToAModelAddedMeanwhile(t *testing.T) {
t.Errorf("job %s, added %d: %+v", j.State, len(taken.added), j.Steps)
}
}
// A model of experts given its cards by hand spreads them still, and the
// environment asked for reaches every profile.
func TestAHandGivenLayoutOfExpertsKeepsExpertParallelAndTheEnvironment(t *testing.T) {
plan := Plan{Repo: "Qwen/Qwen3-235B-A22B", Experts: 128}
req := Request{Format: "safetensors", GPUClass: "h100-80g", GPUsPerNode: 8, Nodes: 2, ContextLength: 8192,
Env: map[string]string{"NCCL_DEBUG": "INFO"}}
layout, err := chooseLayout(req, plan, true, hf.FileSet{Name: "original"})
if err != nil {
t.Fatal(err)
}
profs := profilesFor(layout, req, 0, 0)
p := profs[0]
if p.TP != 8 || p.PP != 2 || p.Nodes != 2 || !slices.Contains(p.Args, "--enable-expert-parallel") || p.Env["NCCL_DEBUG"] != "INFO" {
t.Errorf("profile %+v", p)
}
// A dense model given the same cards does not.
plan.Experts = 0
layout, _ = chooseLayout(req, plan, true, hf.FileSet{Name: "original"})
if slices.Contains(profilesFor(layout, req, 0, 0)[0].Args, "--enable-expert-parallel") {
t.Error("a dense model was given expert parallelism")
}
}
+4 -2
View File
@@ -45,7 +45,9 @@ type Plan struct {
// KVBytesPerToken is one token of context, the whole model: what an
// option needs for another context is worked out from it. 0 is unknown.
KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"`
Gated bool `json:"gated,omitempty"`
// Experts is a mixture-of-experts model's routed experts; 0 for dense.
Experts int `json:"experts,omitempty"`
Gated bool `json:"gated,omitempty"`
// Servable is false when the nodes' engine cannot run the model at all,
// and Reason says why.
Servable bool `json:"servable"`
@@ -161,7 +163,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
p.Note = "the model's shape was not readable, so the context's memory is not counted"
}
shape.Params = p.Params
p.KVBytesPerToken = shape.KVBytesPerToken
p.KVBytesPerToken, p.Experts = shape.KVBytesPerToken, shape.Experts
p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context)
if !p.Servable {
p.Options = nil