Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
630d2788c0
commit
54db7aecb7
15 files changed
+1224
-158
No files matched your search
+102
-5
@@ -2,11 +2,15 @@ package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
@@ -26,13 +30,20 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
|
||||
context int
|
||||
params int64
|
||||
precs []string
|
||||
format string
|
||||
)
|
||||
c := &cobra.Command{
|
||||
Use: "plan <huggingface-repo>",
|
||||
Short: "Work out whether a model fits this cluster, and how it should be split",
|
||||
Long: "Reads the model's architecture from its repository and the hardware from\n" +
|
||||
"the running fleet, then prints every way it could run — fewest GPUs\n" +
|
||||
"first — and a catalog entry for the best one.\n\n" +
|
||||
Short: "Work out which of a model's files suits this cluster, and how it should be split",
|
||||
Long: "Lists every file the repository has — each GGUF quantisation, or with\n" +
|
||||
"--format safetensors its original weights — with the best way each could\n" +
|
||||
"run here: on one card, on several of a node's, over several nodes on a\n" +
|
||||
"fabric, or on a CPU node; what each card would hold and whether the cards\n" +
|
||||
"are free now. The model's shape is read from one file's header, nothing is\n" +
|
||||
"downloaded. `llmgw ctl model serve` without --quant takes the first.\n\n" +
|
||||
"With --precision it plans in the abstract instead, from the model's\n" +
|
||||
"config: every way it could run at bf16, fp8, q8_0 or q4_k_m, fewest GPUs\n" +
|
||||
"first, and a catalog entry for the best one.\n\n" +
|
||||
"Every candidate is listed rather than one answer, because the choice is\n" +
|
||||
"not the planner's to make: with cards to spare you want the\n" +
|
||||
"full-precision fit, and with one card free you want to know a quantised\n" +
|
||||
@@ -40,6 +51,11 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
repo := args[0]
|
||||
// The files the repository has, against the fleet as it is —
|
||||
// unless a precision or a size is given to plan in the abstract.
|
||||
if len(precs) == 0 && params == 0 {
|
||||
return planFiles(cmd, client, repo, format, context)
|
||||
}
|
||||
m, err := modelplan.FetchModel(cmd.Context(), repo)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -112,7 +128,9 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
|
||||
}
|
||||
c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)")
|
||||
c.Flags().Int64Var(¶ms, "params", 0, "parameter count, when the repository does not publish one")
|
||||
c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)")
|
||||
c.Flags().StringSliceVar(&precs, "precision", nil, "plan in the abstract, for these precisions (bf16, fp8, q8_0, q4_k_m), "+
|
||||
"from the model's config rather than its files")
|
||||
c.Flags().StringVar(&format, "format", "gguf", "gguf files, for llama.cpp, or safetensors, the original weights for vLLM")
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -238,3 +256,82 @@ func humanBytes(n int64) string {
|
||||
return fmt.Sprintf("%.0f KiB", float64(n)/1024)
|
||||
}
|
||||
}
|
||||
|
||||
// planFiles prints the control plane's plan of a repository's files.
|
||||
func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string, context int) error {
|
||||
q := url.Values{"repo": {repo}, "format": {format}}
|
||||
if context > 0 {
|
||||
q.Set("context", strconv.Itoa(context))
|
||||
}
|
||||
var p modelserve.Plan
|
||||
if err := client().get(cmd.Context(), "/api/v1/plan?"+q.Encode(), &p); err != nil {
|
||||
return err
|
||||
}
|
||||
out := cmd.OutOrStdout()
|
||||
fmt.Fprintf(out, "%s at %s", p.Repo, catalog.ShortRevision(p.Revision))
|
||||
if p.Arch != "" {
|
||||
fmt.Fprintf(out, ", %s", p.Arch)
|
||||
}
|
||||
if p.Params > 0 {
|
||||
fmt.Fprintf(out, ", %.1fB parameters", float64(p.Params)/1e9)
|
||||
}
|
||||
fmt.Fprintf(out, "; planned for a %d-token context\n", p.Context)
|
||||
if p.Gated {
|
||||
fmt.Fprintln(out, "gated: the mirror cannot fetch it without an account's approval")
|
||||
}
|
||||
if !p.Servable {
|
||||
fmt.Fprintf(out, "the nodes' engine cannot run it: %s\n", p.Reason)
|
||||
return errDoesNotFit
|
||||
}
|
||||
if p.Note != "" {
|
||||
fmt.Fprintln(out, "note: "+p.Note)
|
||||
}
|
||||
fmt.Fprintln(out)
|
||||
w := tabTo(out)
|
||||
fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tHOLDS\tOF\tNOW")
|
||||
for _, a := range p.Artifacts {
|
||||
name := a.Name
|
||||
if a.Shards > 1 && a.Format == "gguf" {
|
||||
name += fmt.Sprintf(" (%d parts)", a.Shards)
|
||||
}
|
||||
if a.Best == nil {
|
||||
fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\n", name, bytesShort(a.Bytes))
|
||||
continue
|
||||
}
|
||||
b := a.Best
|
||||
now := "busy"
|
||||
if b.FreeNow {
|
||||
now = "free"
|
||||
}
|
||||
holds := fmt.Sprintf("%.1f GB a card", b.NeedGB)
|
||||
if b.CPU {
|
||||
holds = fmt.Sprintf("%.1f GB of RAM", b.NeedGB)
|
||||
}
|
||||
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, now)
|
||||
}
|
||||
_ = w.Flush()
|
||||
rec, ok := p.Recommended()
|
||||
if !ok {
|
||||
fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.")
|
||||
return errDoesNotFit
|
||||
}
|
||||
fmt.Fprintf(out, "\nBest: %s on %s.\n", rec.Artifact, rec.Label())
|
||||
if len(p.Options) > 1 {
|
||||
fmt.Fprintln(out, "\nEvery way, best first:")
|
||||
w = tabTo(out)
|
||||
for _, o := range p.Options {
|
||||
now := "busy"
|
||||
if o.FreeNow {
|
||||
now = "free"
|
||||
}
|
||||
fmt.Fprintf(w, " %s\t%s\t%.1f of %.0f GB\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.NeedGB, o.HaveGB, o.Hosts, now)
|
||||
}
|
||||
_ = w.Flush()
|
||||
}
|
||||
fmt.Fprintf(out, "\nServe it: llmgw ctl model serve %s", p.Repo)
|
||||
if p.Format == "safetensors" {
|
||||
fmt.Fprint(out, " --format safetensors")
|
||||
}
|
||||
fmt.Fprintln(out, " --fallbacks")
|
||||
return nil
|
||||
}
|
||||
Reference in new issue
Block a user