A plan for a big model: experts spread, any number of nodes, memory said
A mixture-of-experts model on more than one card is served with vLLM's --enable-expert-parallel; config.json's experts are read by any of their names, and a model that also reads pictures is sized by its text_config. Tensor parallelism past the KV heads is allowed when it is a multiple of them (Qwen3-235B, four KV heads, runs on eight cards), which the plan had refused. Several nodes are offered in any number, not only powers of two. Each way says what it needs of a card, part by part, and of all its cards; the serve form checks a context longer than planned still fits. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
314cc5e6a3
commit
4fe9b46451
12 files changed
+257
-34
No files matched your search
@@ -291,14 +291,14 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
w := tabTo(out)
|
||||
fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tHOLDS\tOF\tNOW")
|
||||
fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tA CARD\tOF\tALL CARDS\tNOW")
|
||||
for _, a := range p.Artifacts {
|
||||
name := a.Name
|
||||
if a.Shards > 1 && a.Format == "gguf" {
|
||||
name += fmt.Sprintf(" (%d parts)", a.Shards)
|
||||
}
|
||||
if a.Best == nil {
|
||||
fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\n", name, bytesShort(a.Bytes))
|
||||
fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\t\n", name, bytesShort(a.Bytes))
|
||||
continue
|
||||
}
|
||||
b := a.Best
|
||||
@@ -306,11 +306,11 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
|
||||
if b.FreeNow {
|
||||
now = "free"
|
||||
}
|
||||
holds := fmt.Sprintf("%.1f GB a card", b.NeedGB)
|
||||
holds, all := fmt.Sprintf("%.1f GB", b.NeedGB), fmt.Sprintf("%.0f GB", b.TotalGB())
|
||||
if b.CPU {
|
||||
holds = fmt.Sprintf("%.1f GB of RAM", b.NeedGB)
|
||||
holds, all = fmt.Sprintf("%.1f GB of RAM", b.NeedGB), "-"
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, now)
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, all, now)
|
||||
}
|
||||
_ = w.Flush()
|
||||
rec, ok := p.Recommended()
|
||||
@@ -318,7 +318,11 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
|
||||
fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.")
|
||||
return errDoesNotFit
|
||||
}
|
||||
fmt.Fprintf(out, "\nBest: %s on %s.\n", rec.Artifact, rec.Label())
|
||||
fmt.Fprintf(out, "\nBest: %s on %s: %s.\n", rec.Artifact, rec.Label(), rec.MemoryText())
|
||||
if p.KVBytesPerToken > 0 {
|
||||
fmt.Fprintf(out, "The context costs %.2f GB per 1000 tokens, the whole model (--context to plan another).\n",
|
||||
modelplan.GB(p.KVBytesPerToken*1000))
|
||||
}
|
||||
if len(p.Options) > 1 {
|
||||
fmt.Fprintln(out, "\nEvery way, best first:")
|
||||
w = tabTo(out)
|
||||
@@ -327,7 +331,7 @@ func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string
|
||||
if o.FreeNow {
|
||||
now = "free"
|
||||
}
|
||||
fmt.Fprintf(w, " %s\t%s\t%.1f of %.0f GB\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.NeedGB, o.HaveGB, o.Hosts, now)
|
||||
fmt.Fprintf(w, " %s\t%s\t%s\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.MemoryText(), o.Hosts, now)
|
||||
}
|
||||
_ = w.Flush()
|
||||
}
|
||||
|
||||
+14
-7
@@ -107,11 +107,7 @@ func optionLabel(o modelplan.Option) string {
|
||||
if o.FreeNow {
|
||||
now = sOK.Render("free")
|
||||
}
|
||||
need := fmt.Sprintf("%.1f of %.0f GB a card", o.NeedGB, o.HaveGB)
|
||||
if o.CPU {
|
||||
need = fmt.Sprintf("%.1f GB of RAM", o.NeedGB)
|
||||
}
|
||||
return fmt.Sprintf("%s · %s · %s", o.Label(), need, now)
|
||||
return fmt.Sprintf("%s · %s · %s", o.Label(), o.MemoryText(), now)
|
||||
}
|
||||
|
||||
// repoMenu offers a repository's files, each with the best way it runs here;
|
||||
@@ -197,8 +193,15 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
|
||||
if p.Note != "" {
|
||||
note += "\n" + sWarn.Render("! ") + p.Note
|
||||
}
|
||||
note += fmt.Sprintf("\nPlanned for a %d-token context. The mirror fetches it and checks it, it joins the\n"+
|
||||
"catalog, and a replica is started and asked to say hello.", p.Context)
|
||||
note += fmt.Sprintf("\nMemory for a %d-token context, each way it runs:", p.Context)
|
||||
for _, o := range opts {
|
||||
note += "\n " + o.Label() + ": " + o.MemoryText()
|
||||
}
|
||||
if p.KVBytesPerToken > 0 {
|
||||
note += fmt.Sprintf("\nThe context costs %.2f GB per 1000 tokens, the whole model; a longer one is checked\n"+
|
||||
"before it is served.", modelplan.GB(p.KVBytesPerToken*1000))
|
||||
}
|
||||
note += "\nThe mirror fetches it and checks it, it joins the catalog, and a replica is started\nand asked to say hello."
|
||||
ctxDefault := "4096"
|
||||
if a.Format == "safetensors" {
|
||||
ctxDefault = strconv.Itoa(p.Context)
|
||||
@@ -234,6 +237,10 @@ func (m *consoleModel) serveForm(p modelserve.Plan, a modelserve.PlannedArtifact
|
||||
req.File = a.Name
|
||||
}
|
||||
o := opts[slices.Index(where, v[4])]
|
||||
// Asked for a context other than planned: does it still fit?
|
||||
if sized := o.WithContext(p.KVBytesPerToken, ctxLen); !sized.Fits() {
|
||||
return nil, fmt.Errorf("with a %d-token context it needs %s: a shorter context, or another way", ctxLen, sized.MemoryText())
|
||||
}
|
||||
if o.CPU {
|
||||
req.CPU = true
|
||||
} else {
|
||||
|
||||
Reference in new issue
Block a user