Plan a repository's files against the fleet, and serve the one that suits it the way it fits

The control plane reads the model's shape from one GGUF header or its
config.json and lists, for every file, the best way it runs here: one card,
several of a node's (by layer for llama.cpp, tensor-parallel for vLLM),
several nodes' over one fabric (vLLM, pipeline-parallel between them), or a
CPU node; what each card holds and whether it is free now. ctl model plan
prints it; model serve takes its file and way unless told, gives a GGUF on
GPUs a GPU profile, and with --fallbacks adds the next best ways. The
console shows it per file and asks for the way and the fallback.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:03:48 +02:00
1 parent 630d2788c0
commit 54db7aecb7
15 files changed
+1224 -158

No files matched your search

+102 -5
View File
@@ -2,11 +2,15 @@ package cli
import (
"fmt"
"net/url"
"strconv"
"strings"
"github.com/spf13/cobra"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
@@ -26,13 +30,20 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
context int
params int64
precs []string
format string
)
c := &cobra.Command{
Use: "plan <huggingface-repo>",
Short: "Work out whether a model fits this cluster, and how it should be split",
Long: "Reads the model's architecture from its repository and the hardware from\n" +
"the running fleet, then prints every way it could run — fewest GPUs\n" +
"first — and a catalog entry for the best one.\n\n" +
Short: "Work out which of a model's files suits this cluster, and how it should be split",
Long: "Lists every file the repository has — each GGUF quantisation, or with\n" +
"--format safetensors its original weights — with the best way each could\n" +
"run here: on one card, on several of a node's, over several nodes on a\n" +
"fabric, or on a CPU node; what each card would hold and whether the cards\n" +
"are free now. The model's shape is read from one file's header, nothing is\n" +
"downloaded. `llmgw ctl model serve` without --quant takes the first.\n\n" +
"With --precision it plans in the abstract instead, from the model's\n" +
"config: every way it could run at bf16, fp8, q8_0 or q4_k_m, fewest GPUs\n" +
"first, and a catalog entry for the best one.\n\n" +
"Every candidate is listed rather than one answer, because the choice is\n" +
"not the planner's to make: with cards to spare you want the\n" +
"full-precision fit, and with one card free you want to know a quantised\n" +
@@ -40,6 +51,11 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
repo := args[0]
// The files the repository has, against the fleet as it is —
// unless a precision or a size is given to plan in the abstract.
if len(precs) == 0 && params == 0 {
return planFiles(cmd, client, repo, format, context)
}
m, err := modelplan.FetchModel(cmd.Context(), repo)
if err != nil {
return err
@@ -112,7 +128,9 @@ func newModelPlanCmd(client func() *ctlClient) *cobra.Command {
}
c.Flags().IntVar(&context, "context", 0, "token pool to plan for (default: the model's trained context)")
c.Flags().Int64Var(&params, "params", 0, "parameter count, when the repository does not publish one")
c.Flags().StringSliceVar(&precs, "precision", nil, "precisions to consider (default: bf16, fp8, q8_0, q4_k_m)")
c.Flags().StringSliceVar(&precs, "precision", nil, "plan in the abstract, for these precisions (bf16, fp8, q8_0, q4_k_m), "+
"from the model's config rather than its files")
c.Flags().StringVar(&format, "format", "gguf", "gguf files, for llama.cpp, or safetensors, the original weights for vLLM")
return c
}
@@ -238,3 +256,82 @@ func humanBytes(n int64) string {
return fmt.Sprintf("%.0f KiB", float64(n)/1024)
}
}
// planFiles prints the control plane's plan of a repository's files.
func planFiles(cmd *cobra.Command, client func() *ctlClient, repo, format string, context int) error {
q := url.Values{"repo": {repo}, "format": {format}}
if context > 0 {
q.Set("context", strconv.Itoa(context))
}
var p modelserve.Plan
if err := client().get(cmd.Context(), "/api/v1/plan?"+q.Encode(), &p); err != nil {
return err
}
out := cmd.OutOrStdout()
fmt.Fprintf(out, "%s at %s", p.Repo, catalog.ShortRevision(p.Revision))
if p.Arch != "" {
fmt.Fprintf(out, ", %s", p.Arch)
}
if p.Params > 0 {
fmt.Fprintf(out, ", %.1fB parameters", float64(p.Params)/1e9)
}
fmt.Fprintf(out, "; planned for a %d-token context\n", p.Context)
if p.Gated {
fmt.Fprintln(out, "gated: the mirror cannot fetch it without an account's approval")
}
if !p.Servable {
fmt.Fprintf(out, "the nodes' engine cannot run it: %s\n", p.Reason)
return errDoesNotFit
}
if p.Note != "" {
fmt.Fprintln(out, "note: "+p.Note)
}
fmt.Fprintln(out)
w := tabTo(out)
fmt.Fprintln(w, "FILE\tSIZE\tBEST WAY\tHOLDS\tOF\tNOW")
for _, a := range p.Artifacts {
name := a.Name
if a.Shards > 1 && a.Format == "gguf" {
name += fmt.Sprintf(" (%d parts)", a.Shards)
}
if a.Best == nil {
fmt.Fprintf(w, "%s\t%s\tfits nowhere\t\t\t\n", name, bytesShort(a.Bytes))
continue
}
b := a.Best
now := "busy"
if b.FreeNow {
now = "free"
}
holds := fmt.Sprintf("%.1f GB a card", b.NeedGB)
if b.CPU {
holds = fmt.Sprintf("%.1f GB of RAM", b.NeedGB)
}
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%.0f GB\t%s\n", name, bytesShort(a.Bytes), b.Label(), holds, b.HaveGB, now)
}
_ = w.Flush()
rec, ok := p.Recommended()
if !ok {
fmt.Fprintln(out, "\nNo file fits this cluster as it is: a smaller context (--context), or more memory.")
return errDoesNotFit
}
fmt.Fprintf(out, "\nBest: %s on %s.\n", rec.Artifact, rec.Label())
if len(p.Options) > 1 {
fmt.Fprintln(out, "\nEvery way, best first:")
w = tabTo(out)
for _, o := range p.Options {
now := "busy"
if o.FreeNow {
now = "free"
}
fmt.Fprintf(w, " %s\t%s\t%.1f of %.0f GB\t%d host(s)\t%s\n", o.Artifact, o.Label(), o.NeedGB, o.HaveGB, o.Hosts, now)
}
_ = w.Flush()
}
fmt.Fprintf(out, "\nServe it: llmgw ctl model serve %s", p.Repo)
if p.Format == "safetensors" {
fmt.Fprint(out, " --format safetensors")
}
fmt.Fprintln(out, " --fallbacks")
return nil
}