Plan a repository's files against the fleet, and serve the one that suits it the way it fits

The control plane reads the model's shape from one GGUF header or its
config.json and lists, for every file, the best way it runs here: one card,
several of a node's (by layer for llama.cpp, tensor-parallel for vLLM),
several nodes' over one fabric (vLLM, pipeline-parallel between them), or a
CPU node; what each card holds and whether it is free now. ctl model plan
prints it; model serve takes its file and way unless told, gives a GGUF on
GPUs a GPU profile, and with --fallbacks adds the next best ways. The
console shows it per file and asks for the way and the fallback.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:03:48 +02:00
1 parent 630d2788c0
commit 54db7aecb7
15 files changed
+1224 -158

No files matched your search

+142 -25
View File
@@ -30,6 +30,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
// Registry reads a repository.
@@ -77,6 +78,10 @@ type Deps struct {
Current func() []catalog.Model
Models Models
Restart func(ids ...string) error
// Pool is the fleet's ready nodes as a plan sees them: what decides
// which file suits the hardware, and how it is spread.
Pool func() []modelplan.PoolNode
}
// Request is what to serve.
@@ -93,9 +98,18 @@ type Request struct {
// Format is "gguf", run by llama.cpp, or "safetensors": the original
// weights with their config and tokenizer, run by vLLM on GPUs of
// GPUClass, across TP of them.
Format string `json:"format,omitempty"`
GPUClass string `json:"gpu_class,omitempty"`
TP int `json:"tp,omitempty"`
Format string `json:"format,omitempty"`
// Where it runs: on the CPU, or on GPUsPerNode cards of GPUClass on
// each of Nodes nodes. Left empty, the plan decides: the best way the
// cluster can run the chosen file. TP is GPUsPerNode's older name.
CPU bool `json:"cpu,omitempty"`
GPUClass string `json:"gpu_class,omitempty"`
GPUsPerNode int `json:"gpus_per_node,omitempty"`
Nodes int `json:"nodes,omitempty"`
TP int `json:"tp,omitempty"`
// Fallbacks adds the next best ways, lower in priority, so a model whose
// first choice is busy runs on another rather than waiting.
Fallbacks bool `json:"fallbacks,omitempty"`
// Name is the catalog name; the repository suggests one otherwise.
Name string `json:"name,omitempty"`
Aliases []string `json:"aliases,omitempty"`
@@ -192,19 +206,15 @@ func (s *Server) Start(req Request) (Job, error) {
case "", "gguf":
req.Format = "gguf"
case "safetensors":
if req.GPUClass == "" {
return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it")
}
if req.Projector != "" {
return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files")
}
req.TP = max(req.TP, 1)
if req.CPU {
return Job{}, errors.New("original weights run on vLLM, on GPUs; a GGUF file runs on the CPU")
}
default:
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
}
if req.Quant == "" && req.File == "" {
req.Quant = "Q4_K_M"
}
if req.CtxPool <= 0 {
req.CtxPool = 16384
}
@@ -274,6 +284,30 @@ func (s *Server) run(j *Job) {
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo))
return
}
if req.TP > 0 && req.GPUsPerNode == 0 {
req.GPUsPerNode = req.TP
}
// The plan: which file suits the hardware, and how it spreads over it.
planCtx := req.CtxPool
if req.Format == "safetensors" {
planCtx = req.ContextLength
}
var plan Plan
planned := false
if s.deps.Pool != nil {
if plan, err = s.plan(ctx, repo, PlanRequest{Repo: repo.ID, Format: req.Format, Context: planCtx}); err == nil {
planned = true
} else {
s.deps.Log.Warn("planning a model", "repo", repo.ID, "err", err)
}
}
if req.Format == "gguf" && req.Quant == "" && req.File == "" {
if rec, ok := plan.Recommended(); planned && ok {
req.File = rec.Artifact
} else {
req.Quant = "Q4_K_M"
}
}
pickSet := pick
if req.Format == "safetensors" {
pickSet = pickSafetensors
@@ -283,6 +317,11 @@ func (s *Server) run(j *Job) {
s.fail(j, "registry", err)
return
}
layout, err := chooseLayout(req, plan, planned, set)
if err != nil {
s.fail(j, "registry", err)
return
}
if req.Projector != "" {
i := slices.IndexFunc(repo.Files, func(f hf.File) bool {
return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) &&
@@ -303,8 +342,12 @@ func (s *Server) run(j *Job) {
s.mu.Lock()
j.Model = name
s.mu.Unlock()
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s",
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision()))
where := make([]string, 0, len(layout))
for _, o := range layout {
where = append(where, o.Label())
}
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s",
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else ")))
// 2. The mirror: fetch, and check what arrived against what was published.
source, err := s.mirror(ctx, j, repo, set, name)
@@ -313,15 +356,12 @@ func (s *Server) run(j *Job) {
return
}
// 3. Whether the nodes' llama.cpp loads it, and how much memory it needs.
// 3. Whether the nodes' engine loads it, and how much memory it needs.
if req.Format == "safetensors" {
// vLLM reads the model's shape from its config.json, which came
// with the weights; the GPUs it needs are the class and width asked.
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass))
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100,
Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)},
})
// with the weights.
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM", gb(set.Bytes)))
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0))
return
}
s.step(j, "check", Running, "reading its header from the mirror")
@@ -351,22 +391,99 @@ func (s *Server) run(j *Job) {
cores = catalog.SuggestCores(nc.NodeCPUs(memGB))
}
}
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, memGB, cores))
}
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
})
// chooseLayout is where a model runs: what was asked, else the plan's best
// for the chosen file; with Fallbacks, the next best of other shapes after.
func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]modelplan.Option, error) {
artifact := set.Name
engine := "llamacpp"
if req.Format == "safetensors" {
artifact, engine = "original", "vllm"
}
var opts []modelplan.Option
for _, o := range plan.Options {
if o.Artifact == artifact {
opts = append(opts, o)
}
}
var out []modelplan.Option
switch {
case req.CPU || req.GPUClass != "":
o := modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: req.CPU}
if !req.CPU {
o.Class, o.GPUsPerNode, o.Nodes = req.GPUClass, max(req.GPUsPerNode, 1), max(req.Nodes, 1)
}
if o.Nodes > 1 && engine == "llamacpp" {
return nil, errors.New("llama.cpp does not span nodes; serve the original weights with vLLM for that")
}
out = append(out, o)
case len(opts) > 0:
out = append(out, opts[0])
case planned:
return nil, fmt.Errorf("%s fits nowhere in this cluster with a %d-token context: no card holds it, "+
"nor does any node's memory; `llmgw ctl model plan %s` shows what each file needs", set.Name, plan.Context, plan.Repo)
case engine == "vllm":
return nil, errors.New("original weights run on vLLM, on GPUs: name the GPU class")
default:
out = append(out, modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: true})
}
if req.Fallbacks {
shape := func(o modelplan.Option) string {
return fmt.Sprintf("%v/%s/%d/%d", o.CPU, o.Class, o.GPUsPerNode, o.Nodes)
}
seen := map[string]bool{shape(out[0]): true}
for _, o := range opts {
if len(out) == 3 {
break
}
if !seen[shape(o)] {
seen[shape(o)] = true
out = append(out, o)
}
}
}
return out, nil
}
// profilesFor turns the ways chosen into catalog profiles, the first
// preferred.
func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile {
var out []catalog.Profile
for i, o := range layout {
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i}
switch {
case o.CPU:
p.Name = "cpu"
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
p.MinMemoryGB, p.CPUCores = memGB, cores
case o.Engine == "llamacpp":
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class)
p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
default:
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class)
p.GPUClass, p.TP = o.Class, o.GPUsPerNode
if o.Nodes > 1 {
p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode
}
p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)}
}
out = append(out, p)
}
return out
}
// addAndStart puts the model in the catalog with its profile, waits for a
// replica and asks it to answer.
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) {
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, profs []catalog.Profile) {
req := j.Request
// 4. The catalog.
m := catalog.Model{
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
Profiles: []catalog.Profile{prof},
Profiles: profs,
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},