Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
630d2788c0
commit
54db7aecb7
15 files changed
+1224
-158
No files matched your search
@@ -30,6 +30,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
// Registry reads a repository.
|
||||
@@ -77,6 +78,10 @@ type Deps struct {
|
||||
Current func() []catalog.Model
|
||||
Models Models
|
||||
Restart func(ids ...string) error
|
||||
|
||||
// Pool is the fleet's ready nodes as a plan sees them: what decides
|
||||
// which file suits the hardware, and how it is spread.
|
||||
Pool func() []modelplan.PoolNode
|
||||
}
|
||||
|
||||
// Request is what to serve.
|
||||
@@ -93,9 +98,18 @@ type Request struct {
|
||||
// Format is "gguf", run by llama.cpp, or "safetensors": the original
|
||||
// weights with their config and tokenizer, run by vLLM on GPUs of
|
||||
// GPUClass, across TP of them.
|
||||
Format string `json:"format,omitempty"`
|
||||
GPUClass string `json:"gpu_class,omitempty"`
|
||||
TP int `json:"tp,omitempty"`
|
||||
Format string `json:"format,omitempty"`
|
||||
// Where it runs: on the CPU, or on GPUsPerNode cards of GPUClass on
|
||||
// each of Nodes nodes. Left empty, the plan decides: the best way the
|
||||
// cluster can run the chosen file. TP is GPUsPerNode's older name.
|
||||
CPU bool `json:"cpu,omitempty"`
|
||||
GPUClass string `json:"gpu_class,omitempty"`
|
||||
GPUsPerNode int `json:"gpus_per_node,omitempty"`
|
||||
Nodes int `json:"nodes,omitempty"`
|
||||
TP int `json:"tp,omitempty"`
|
||||
// Fallbacks adds the next best ways, lower in priority, so a model whose
|
||||
// first choice is busy runs on another rather than waiting.
|
||||
Fallbacks bool `json:"fallbacks,omitempty"`
|
||||
// Name is the catalog name; the repository suggests one otherwise.
|
||||
Name string `json:"name,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
@@ -192,19 +206,15 @@ func (s *Server) Start(req Request) (Job, error) {
|
||||
case "", "gguf":
|
||||
req.Format = "gguf"
|
||||
case "safetensors":
|
||||
if req.GPUClass == "" {
|
||||
return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it")
|
||||
}
|
||||
if req.Projector != "" {
|
||||
return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files")
|
||||
}
|
||||
req.TP = max(req.TP, 1)
|
||||
if req.CPU {
|
||||
return Job{}, errors.New("original weights run on vLLM, on GPUs; a GGUF file runs on the CPU")
|
||||
}
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
|
||||
}
|
||||
if req.Quant == "" && req.File == "" {
|
||||
req.Quant = "Q4_K_M"
|
||||
}
|
||||
if req.CtxPool <= 0 {
|
||||
req.CtxPool = 16384
|
||||
}
|
||||
@@ -274,6 +284,30 @@ func (s *Server) run(j *Job) {
|
||||
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo))
|
||||
return
|
||||
}
|
||||
if req.TP > 0 && req.GPUsPerNode == 0 {
|
||||
req.GPUsPerNode = req.TP
|
||||
}
|
||||
// The plan: which file suits the hardware, and how it spreads over it.
|
||||
planCtx := req.CtxPool
|
||||
if req.Format == "safetensors" {
|
||||
planCtx = req.ContextLength
|
||||
}
|
||||
var plan Plan
|
||||
planned := false
|
||||
if s.deps.Pool != nil {
|
||||
if plan, err = s.plan(ctx, repo, PlanRequest{Repo: repo.ID, Format: req.Format, Context: planCtx}); err == nil {
|
||||
planned = true
|
||||
} else {
|
||||
s.deps.Log.Warn("planning a model", "repo", repo.ID, "err", err)
|
||||
}
|
||||
}
|
||||
if req.Format == "gguf" && req.Quant == "" && req.File == "" {
|
||||
if rec, ok := plan.Recommended(); planned && ok {
|
||||
req.File = rec.Artifact
|
||||
} else {
|
||||
req.Quant = "Q4_K_M"
|
||||
}
|
||||
}
|
||||
pickSet := pick
|
||||
if req.Format == "safetensors" {
|
||||
pickSet = pickSafetensors
|
||||
@@ -283,6 +317,11 @@ func (s *Server) run(j *Job) {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
layout, err := chooseLayout(req, plan, planned, set)
|
||||
if err != nil {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
if req.Projector != "" {
|
||||
i := slices.IndexFunc(repo.Files, func(f hf.File) bool {
|
||||
return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) &&
|
||||
@@ -303,8 +342,12 @@ func (s *Server) run(j *Job) {
|
||||
s.mu.Lock()
|
||||
j.Model = name
|
||||
s.mu.Unlock()
|
||||
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s",
|
||||
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision()))
|
||||
where := make([]string, 0, len(layout))
|
||||
for _, o := range layout {
|
||||
where = append(where, o.Label())
|
||||
}
|
||||
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s",
|
||||
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else ")))
|
||||
|
||||
// 2. The mirror: fetch, and check what arrived against what was published.
|
||||
source, err := s.mirror(ctx, j, repo, set, name)
|
||||
@@ -313,15 +356,12 @@ func (s *Server) run(j *Job) {
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Whether the nodes' llama.cpp loads it, and how much memory it needs.
|
||||
// 3. Whether the nodes' engine loads it, and how much memory it needs.
|
||||
if req.Format == "safetensors" {
|
||||
// vLLM reads the model's shape from its config.json, which came
|
||||
// with the weights; the GPUs it needs are the class and width asked.
|
||||
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass))
|
||||
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
|
||||
Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100,
|
||||
Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)},
|
||||
})
|
||||
// with the weights.
|
||||
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM", gb(set.Bytes)))
|
||||
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0))
|
||||
return
|
||||
}
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
@@ -351,22 +391,99 @@ func (s *Server) run(j *Job) {
|
||||
cores = catalog.SuggestCores(nc.NodeCPUs(memGB))
|
||||
}
|
||||
}
|
||||
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, memGB, cores))
|
||||
}
|
||||
|
||||
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
|
||||
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
|
||||
})
|
||||
// chooseLayout is where a model runs: what was asked, else the plan's best
|
||||
// for the chosen file; with Fallbacks, the next best of other shapes after.
|
||||
func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]modelplan.Option, error) {
|
||||
artifact := set.Name
|
||||
engine := "llamacpp"
|
||||
if req.Format == "safetensors" {
|
||||
artifact, engine = "original", "vllm"
|
||||
}
|
||||
var opts []modelplan.Option
|
||||
for _, o := range plan.Options {
|
||||
if o.Artifact == artifact {
|
||||
opts = append(opts, o)
|
||||
}
|
||||
}
|
||||
var out []modelplan.Option
|
||||
switch {
|
||||
case req.CPU || req.GPUClass != "":
|
||||
o := modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: req.CPU}
|
||||
if !req.CPU {
|
||||
o.Class, o.GPUsPerNode, o.Nodes = req.GPUClass, max(req.GPUsPerNode, 1), max(req.Nodes, 1)
|
||||
}
|
||||
if o.Nodes > 1 && engine == "llamacpp" {
|
||||
return nil, errors.New("llama.cpp does not span nodes; serve the original weights with vLLM for that")
|
||||
}
|
||||
out = append(out, o)
|
||||
case len(opts) > 0:
|
||||
out = append(out, opts[0])
|
||||
case planned:
|
||||
return nil, fmt.Errorf("%s fits nowhere in this cluster with a %d-token context: no card holds it, "+
|
||||
"nor does any node's memory; `llmgw ctl model plan %s` shows what each file needs", set.Name, plan.Context, plan.Repo)
|
||||
case engine == "vllm":
|
||||
return nil, errors.New("original weights run on vLLM, on GPUs: name the GPU class")
|
||||
default:
|
||||
out = append(out, modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: true})
|
||||
}
|
||||
if req.Fallbacks {
|
||||
shape := func(o modelplan.Option) string {
|
||||
return fmt.Sprintf("%v/%s/%d/%d", o.CPU, o.Class, o.GPUsPerNode, o.Nodes)
|
||||
}
|
||||
seen := map[string]bool{shape(out[0]): true}
|
||||
for _, o := range opts {
|
||||
if len(out) == 3 {
|
||||
break
|
||||
}
|
||||
if !seen[shape(o)] {
|
||||
seen[shape(o)] = true
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// profilesFor turns the ways chosen into catalog profiles, the first
|
||||
// preferred.
|
||||
func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile {
|
||||
var out []catalog.Profile
|
||||
for i, o := range layout {
|
||||
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i}
|
||||
switch {
|
||||
case o.CPU:
|
||||
p.Name = "cpu"
|
||||
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
|
||||
p.MinMemoryGB, p.CPUCores = memGB, cores
|
||||
case o.Engine == "llamacpp":
|
||||
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class)
|
||||
p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode
|
||||
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
|
||||
default:
|
||||
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class)
|
||||
p.GPUClass, p.TP = o.Class, o.GPUsPerNode
|
||||
if o.Nodes > 1 {
|
||||
p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode
|
||||
}
|
||||
p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)}
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// addAndStart puts the model in the catalog with its profile, waits for a
|
||||
// replica and asks it to answer.
|
||||
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) {
|
||||
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, profs []catalog.Profile) {
|
||||
req := j.Request
|
||||
// 4. The catalog.
|
||||
m := catalog.Model{
|
||||
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
|
||||
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
|
||||
Profiles: []catalog.Profile{prof},
|
||||
Profiles: profs,
|
||||
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
|
||||
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
|
||||
|
||||
Reference in new issue
Block a user