A big model's download no longer runs out its startup timeout
The startup timeout runs from when a node's download last moved, so a model of hundreds of gigabytes on a slow mirror is not failed while it arrives. ctl model serve --env sets engine variables (NCCL_DEBUG=INFO) on every node, and a layout given by hand keeps expert parallelism for a model of experts. docs/large-models.md: how a model is split, what each way needs of a card, and a checklist for bringing one up on a fabric. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4fe9b46451
commit
8ffde2b80c
12 files changed
+259
-4
No files matched your search
@@ -163,6 +163,10 @@ type Request struct {
|
||||
// CPUCores is what a replica computes on; zero takes half the smallest
|
||||
// node it fits on.
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
// Env is set in the engine's environment, on every node of a replica:
|
||||
// NCCL_DEBUG=INFO to see how its cards talk, NCCL_IB_* to steer them.
|
||||
// Only the variables engines are tuned with; the catalog refuses others.
|
||||
Env map[string]string `json:"env,omitempty"`
|
||||
// IfAbsent adds the model only if nobody has meanwhile: the leader's own
|
||||
// serving of a judge a policy names gives way to the one init or an
|
||||
// operator added while it fetched, rather than replacing it.
|
||||
@@ -580,6 +584,8 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
|
||||
o = p
|
||||
}
|
||||
}
|
||||
// A way the plan did not offer is still a model of experts.
|
||||
o.ExpertParallel = engine == "vllm" && plan.Experts > 1 && o.GPUs() > 1
|
||||
out = append(out, o)
|
||||
case len(opts) > 0:
|
||||
out = append(out, opts[0])
|
||||
@@ -614,7 +620,7 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
|
||||
func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile {
|
||||
var out []catalog.Profile
|
||||
for i, o := range layout {
|
||||
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i}
|
||||
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i, Env: req.Env}
|
||||
switch {
|
||||
case o.CPU:
|
||||
p.Name = "cpu"
|
||||
|
||||
Reference in new issue
Block a user