A plan for a big model: experts spread, any number of nodes, memory said
A mixture-of-experts model on more than one card is served with vLLM's --enable-expert-parallel; config.json's experts are read by any of their names, and a model that also reads pictures is sized by its text_config. Tensor parallelism past the KV heads is allowed when it is a multiple of them (Qwen3-235B, four KV heads, runs on eight cards), which the plan had refused. Several nodes are offered in any number, not only powers of two. Each way says what it needs of a card, part by part, and of all its cards; the serve form checks a context longer than planned still fits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
314cc5e6a3
commit
4fe9b46451
12 files changed
+257
-34
No files matched your search
@@ -461,8 +461,17 @@ func (s *Server) run(j *Job) {
|
||||
for _, o := range layout {
|
||||
where = append(where, o.Label())
|
||||
}
|
||||
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s",
|
||||
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else ")))
|
||||
// What the first way needs of each card, for the context asked for.
|
||||
need := ""
|
||||
if first := layout[0]; first.NeedGB > 0 {
|
||||
tokens := req.ContextLength
|
||||
if req.Format != "safetensors" && req.CtxPool > 0 {
|
||||
tokens = req.CtxPool
|
||||
}
|
||||
need = "; needs " + first.WithContext(plan.KVBytesPerToken, tokens).MemoryText()
|
||||
}
|
||||
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s%s",
|
||||
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else "), need))
|
||||
|
||||
// 2. The mirror: fetch, and check what arrived against what was published.
|
||||
source, err := s.mirror(ctx, j, repo, set, name)
|
||||
@@ -622,6 +631,11 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
|
||||
p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode
|
||||
}
|
||||
p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)}
|
||||
if o.ExpertParallel {
|
||||
// The experts whole on each card rather than each sliced
|
||||
// across them: less traffic between the cards for every token.
|
||||
p.Args = append(p.Args, "--enable-expert-parallel")
|
||||
}
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
|
||||
@@ -42,7 +42,10 @@ type Plan struct {
|
||||
License string `json:"license,omitempty"`
|
||||
Params int64 `json:"params,omitempty"`
|
||||
Context int `json:"context"`
|
||||
Gated bool `json:"gated,omitempty"`
|
||||
// KVBytesPerToken is one token of context, the whole model: what an
|
||||
// option needs for another context is worked out from it. 0 is unknown.
|
||||
KVBytesPerToken int64 `json:"kv_bytes_per_token,omitempty"`
|
||||
Gated bool `json:"gated,omitempty"`
|
||||
// Servable is false when the nodes' engine cannot run the model at all,
|
||||
// and Reason says why.
|
||||
Servable bool `json:"servable"`
|
||||
@@ -144,6 +147,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
|
||||
if err == nil {
|
||||
shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16)
|
||||
shape.KVHeads = m.KVHeads
|
||||
shape.Experts = m.Experts
|
||||
if p.Params == 0 {
|
||||
p.Params = m.Params
|
||||
}
|
||||
@@ -157,6 +161,7 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
|
||||
p.Note = "the model's shape was not readable, so the context's memory is not counted"
|
||||
}
|
||||
shape.Params = p.Params
|
||||
p.KVBytesPerToken = shape.KVBytesPerToken
|
||||
p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context)
|
||||
if !p.Servable {
|
||||
p.Options = nil
|
||||
|
||||
Reference in new issue
Block a user