Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
630d2788c0
commit
54db7aecb7
15 files changed
+1224
-158
No files matched your search
@@ -30,6 +30,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
// Registry reads a repository.
|
||||
@@ -77,6 +78,10 @@ type Deps struct {
|
||||
Current func() []catalog.Model
|
||||
Models Models
|
||||
Restart func(ids ...string) error
|
||||
|
||||
// Pool is the fleet's ready nodes as a plan sees them: what decides
|
||||
// which file suits the hardware, and how it is spread.
|
||||
Pool func() []modelplan.PoolNode
|
||||
}
|
||||
|
||||
// Request is what to serve.
|
||||
@@ -93,9 +98,18 @@ type Request struct {
|
||||
// Format is "gguf", run by llama.cpp, or "safetensors": the original
|
||||
// weights with their config and tokenizer, run by vLLM on GPUs of
|
||||
// GPUClass, across TP of them.
|
||||
Format string `json:"format,omitempty"`
|
||||
GPUClass string `json:"gpu_class,omitempty"`
|
||||
TP int `json:"tp,omitempty"`
|
||||
Format string `json:"format,omitempty"`
|
||||
// Where it runs: on the CPU, or on GPUsPerNode cards of GPUClass on
|
||||
// each of Nodes nodes. Left empty, the plan decides: the best way the
|
||||
// cluster can run the chosen file. TP is GPUsPerNode's older name.
|
||||
CPU bool `json:"cpu,omitempty"`
|
||||
GPUClass string `json:"gpu_class,omitempty"`
|
||||
GPUsPerNode int `json:"gpus_per_node,omitempty"`
|
||||
Nodes int `json:"nodes,omitempty"`
|
||||
TP int `json:"tp,omitempty"`
|
||||
// Fallbacks adds the next best ways, lower in priority, so a model whose
|
||||
// first choice is busy runs on another rather than waiting.
|
||||
Fallbacks bool `json:"fallbacks,omitempty"`
|
||||
// Name is the catalog name; the repository suggests one otherwise.
|
||||
Name string `json:"name,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
@@ -192,19 +206,15 @@ func (s *Server) Start(req Request) (Job, error) {
|
||||
case "", "gguf":
|
||||
req.Format = "gguf"
|
||||
case "safetensors":
|
||||
if req.GPUClass == "" {
|
||||
return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it")
|
||||
}
|
||||
if req.Projector != "" {
|
||||
return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files")
|
||||
}
|
||||
req.TP = max(req.TP, 1)
|
||||
if req.CPU {
|
||||
return Job{}, errors.New("original weights run on vLLM, on GPUs; a GGUF file runs on the CPU")
|
||||
}
|
||||
default:
|
||||
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
|
||||
}
|
||||
if req.Quant == "" && req.File == "" {
|
||||
req.Quant = "Q4_K_M"
|
||||
}
|
||||
if req.CtxPool <= 0 {
|
||||
req.CtxPool = 16384
|
||||
}
|
||||
@@ -274,6 +284,30 @@ func (s *Server) run(j *Job) {
|
||||
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo))
|
||||
return
|
||||
}
|
||||
if req.TP > 0 && req.GPUsPerNode == 0 {
|
||||
req.GPUsPerNode = req.TP
|
||||
}
|
||||
// The plan: which file suits the hardware, and how it spreads over it.
|
||||
planCtx := req.CtxPool
|
||||
if req.Format == "safetensors" {
|
||||
planCtx = req.ContextLength
|
||||
}
|
||||
var plan Plan
|
||||
planned := false
|
||||
if s.deps.Pool != nil {
|
||||
if plan, err = s.plan(ctx, repo, PlanRequest{Repo: repo.ID, Format: req.Format, Context: planCtx}); err == nil {
|
||||
planned = true
|
||||
} else {
|
||||
s.deps.Log.Warn("planning a model", "repo", repo.ID, "err", err)
|
||||
}
|
||||
}
|
||||
if req.Format == "gguf" && req.Quant == "" && req.File == "" {
|
||||
if rec, ok := plan.Recommended(); planned && ok {
|
||||
req.File = rec.Artifact
|
||||
} else {
|
||||
req.Quant = "Q4_K_M"
|
||||
}
|
||||
}
|
||||
pickSet := pick
|
||||
if req.Format == "safetensors" {
|
||||
pickSet = pickSafetensors
|
||||
@@ -283,6 +317,11 @@ func (s *Server) run(j *Job) {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
layout, err := chooseLayout(req, plan, planned, set)
|
||||
if err != nil {
|
||||
s.fail(j, "registry", err)
|
||||
return
|
||||
}
|
||||
if req.Projector != "" {
|
||||
i := slices.IndexFunc(repo.Files, func(f hf.File) bool {
|
||||
return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) &&
|
||||
@@ -303,8 +342,12 @@ func (s *Server) run(j *Job) {
|
||||
s.mu.Lock()
|
||||
j.Model = name
|
||||
s.mu.Unlock()
|
||||
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s",
|
||||
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision()))
|
||||
where := make([]string, 0, len(layout))
|
||||
for _, o := range layout {
|
||||
where = append(where, o.Label())
|
||||
}
|
||||
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s",
|
||||
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else ")))
|
||||
|
||||
// 2. The mirror: fetch, and check what arrived against what was published.
|
||||
source, err := s.mirror(ctx, j, repo, set, name)
|
||||
@@ -313,15 +356,12 @@ func (s *Server) run(j *Job) {
|
||||
return
|
||||
}
|
||||
|
||||
// 3. Whether the nodes' llama.cpp loads it, and how much memory it needs.
|
||||
// 3. Whether the nodes' engine loads it, and how much memory it needs.
|
||||
if req.Format == "safetensors" {
|
||||
// vLLM reads the model's shape from its config.json, which came
|
||||
// with the weights; the GPUs it needs are the class and width asked.
|
||||
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass))
|
||||
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
|
||||
Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100,
|
||||
Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)},
|
||||
})
|
||||
// with the weights.
|
||||
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM", gb(set.Bytes)))
|
||||
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0))
|
||||
return
|
||||
}
|
||||
s.step(j, "check", Running, "reading its header from the mirror")
|
||||
@@ -351,22 +391,99 @@ func (s *Server) run(j *Job) {
|
||||
cores = catalog.SuggestCores(nc.NodeCPUs(memGB))
|
||||
}
|
||||
}
|
||||
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, memGB, cores))
|
||||
}
|
||||
|
||||
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
|
||||
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
|
||||
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
|
||||
})
|
||||
// chooseLayout is where a model runs: what was asked, else the plan's best
|
||||
// for the chosen file; with Fallbacks, the next best of other shapes after.
|
||||
func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]modelplan.Option, error) {
|
||||
artifact := set.Name
|
||||
engine := "llamacpp"
|
||||
if req.Format == "safetensors" {
|
||||
artifact, engine = "original", "vllm"
|
||||
}
|
||||
var opts []modelplan.Option
|
||||
for _, o := range plan.Options {
|
||||
if o.Artifact == artifact {
|
||||
opts = append(opts, o)
|
||||
}
|
||||
}
|
||||
var out []modelplan.Option
|
||||
switch {
|
||||
case req.CPU || req.GPUClass != "":
|
||||
o := modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: req.CPU}
|
||||
if !req.CPU {
|
||||
o.Class, o.GPUsPerNode, o.Nodes = req.GPUClass, max(req.GPUsPerNode, 1), max(req.Nodes, 1)
|
||||
}
|
||||
if o.Nodes > 1 && engine == "llamacpp" {
|
||||
return nil, errors.New("llama.cpp does not span nodes; serve the original weights with vLLM for that")
|
||||
}
|
||||
out = append(out, o)
|
||||
case len(opts) > 0:
|
||||
out = append(out, opts[0])
|
||||
case planned:
|
||||
return nil, fmt.Errorf("%s fits nowhere in this cluster with a %d-token context: no card holds it, "+
|
||||
"nor does any node's memory; `llmgw ctl model plan %s` shows what each file needs", set.Name, plan.Context, plan.Repo)
|
||||
case engine == "vllm":
|
||||
return nil, errors.New("original weights run on vLLM, on GPUs: name the GPU class")
|
||||
default:
|
||||
out = append(out, modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: true})
|
||||
}
|
||||
if req.Fallbacks {
|
||||
shape := func(o modelplan.Option) string {
|
||||
return fmt.Sprintf("%v/%s/%d/%d", o.CPU, o.Class, o.GPUsPerNode, o.Nodes)
|
||||
}
|
||||
seen := map[string]bool{shape(out[0]): true}
|
||||
for _, o := range opts {
|
||||
if len(out) == 3 {
|
||||
break
|
||||
}
|
||||
if !seen[shape(o)] {
|
||||
seen[shape(o)] = true
|
||||
out = append(out, o)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// profilesFor turns the ways chosen into catalog profiles, the first
|
||||
// preferred.
|
||||
func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile {
|
||||
var out []catalog.Profile
|
||||
for i, o := range layout {
|
||||
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i}
|
||||
switch {
|
||||
case o.CPU:
|
||||
p.Name = "cpu"
|
||||
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
|
||||
p.MinMemoryGB, p.CPUCores = memGB, cores
|
||||
case o.Engine == "llamacpp":
|
||||
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class)
|
||||
p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode
|
||||
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
|
||||
default:
|
||||
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class)
|
||||
p.GPUClass, p.TP = o.Class, o.GPUsPerNode
|
||||
if o.Nodes > 1 {
|
||||
p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode
|
||||
}
|
||||
p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)}
|
||||
}
|
||||
out = append(out, p)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// addAndStart puts the model in the catalog with its profile, waits for a
|
||||
// replica and asks it to answer.
|
||||
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) {
|
||||
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, profs []catalog.Profile) {
|
||||
req := j.Request
|
||||
// 4. The catalog.
|
||||
m := catalog.Model{
|
||||
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
|
||||
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
|
||||
Profiles: []catalog.Profile{prof},
|
||||
Profiles: profs,
|
||||
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
|
||||
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
|
||||
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
|
||||
|
||||
@@ -16,6 +16,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
// ggufHeader is enough of a GGUF file for its architecture and shape.
|
||||
@@ -38,7 +39,12 @@ func ggufHeader() []byte {
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
type fakeRegistry struct{ repo hf.Repo }
|
||||
type fakeRegistry struct {
|
||||
repo hf.Repo
|
||||
base string
|
||||
}
|
||||
|
||||
func (f fakeRegistry) Base() string { return f.base }
|
||||
|
||||
func (f fakeRegistry) Repo(context.Context, string) (hf.Repo, error) { return f.repo, nil }
|
||||
|
||||
@@ -265,8 +271,10 @@ func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T)
|
||||
{Path: "original/consolidated.safetensors", Bytes: 7 << 30},
|
||||
}
|
||||
s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"})
|
||||
if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil {
|
||||
t.Error("safetensors with no GPU class was taken")
|
||||
// With no fleet to plan against, the GPUs must be named.
|
||||
job0, _ := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", Name: "x"})
|
||||
if j := wait(t, s, job0.ID); j.State != Failed || !strings.Contains(j.Error, "GPU class") {
|
||||
t.Errorf("safetensors with no GPU class: %+v", j)
|
||||
}
|
||||
job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1})
|
||||
if err != nil {
|
||||
@@ -289,3 +297,48 @@ func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T)
|
||||
t.Errorf("%s %+v", mod.Source, p)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTheFileThatSuitsTheCardsIsChosenAndRunsOnThem(t *testing.T) {
|
||||
files := []hf.File{
|
||||
{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "a"},
|
||||
{Path: "m-Q8_0.gguf", Bytes: 2 << 30, SHA256: "b"},
|
||||
{Path: "m-F16.gguf", Bytes: 40 << 30, SHA256: "c"},
|
||||
}
|
||||
s, m, c, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "a", "m-Q8_0.gguf": "b"})
|
||||
s.deps.Registry = fakeRegistry{repo: hf.Repo{ID: "o/m-GGUF", Revision: "abcdef123456", Files: files}, base: m.srv.URL}
|
||||
s.deps.Pool = func() []modelplan.PoolNode {
|
||||
return []modelplan.PoolNode{
|
||||
{ID: "g", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 64, HostFreeGB: 60},
|
||||
{ID: "c", HostMemGB: 16, HostFreeGB: 14},
|
||||
}
|
||||
}
|
||||
p, err := s.Plan(context.Background(), PlanRequest{Repo: "o/m-GGUF"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
rec, _ := p.Recommended()
|
||||
if rec.Artifact != "m-Q8_0.gguf" || rec.CPU || rec.Class != "6000-24g" || p.Artifacts[2].Best == nil || !p.Artifacts[2].Best.CPU {
|
||||
t.Fatalf("plan: %+v; f16: %+v", rec, p.Artifacts[2].Best)
|
||||
}
|
||||
job, err := s.Start(Request{Repo: "o/m-GGUF", Replicas: 1, Fallbacks: true})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("%s %+v", j.Error, j.Steps)
|
||||
}
|
||||
if m.got[0].File != "m-Q8_0.gguf" {
|
||||
t.Errorf("fetched %s", m.got[0].File)
|
||||
}
|
||||
profs := c.added[0].Profiles
|
||||
if len(profs) != 2 || profs[0].GPUClass != "6000-24g" || profs[0].GPUsPerNode != 1 || profs[0].Engine != "llamacpp" ||
|
||||
profs[1].Name != "cpu" || profs[1].Priority >= profs[0].Priority || profs[1].MinMemoryGB == 0 {
|
||||
t.Errorf("profiles: %+v", profs)
|
||||
}
|
||||
// A file too big for any card or any node's memory: refused, saying so.
|
||||
s.deps.Pool = func() []modelplan.PoolNode { return []modelplan.PoolNode{{ID: "c", HostMemGB: 16, HostFreeGB: 14}} }
|
||||
job, _ = s.Start(Request{Repo: "o/m-GGUF", File: "m-F16.gguf", Name: "big"})
|
||||
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "fits nowhere") {
|
||||
t.Errorf("%+v", j)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,175 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
// Planning: which of a repository's files suits the cluster's hardware, and
|
||||
// how — one card, several of a node's, several nodes' over a fabric, or a
|
||||
// CPU node — worked out from the files themselves before any is fetched.
|
||||
// The model's shape is read from one GGUF file's header, a few megabytes
|
||||
// from the registry, or from its config.json; the hardware is the fleet as
|
||||
// it is now.
|
||||
|
||||
// PlanRequest is what to plan.
|
||||
type PlanRequest struct {
|
||||
Repo string `json:"repo"`
|
||||
Format string `json:"format,omitempty"` // gguf (default) or safetensors
|
||||
// Context is the token pool: llama.cpp's --ctx-size, or vLLM's
|
||||
// --max-model-len. 0 takes 16384 and 8192.
|
||||
Context int `json:"context,omitempty"`
|
||||
}
|
||||
|
||||
// PlannedArtifact is one file a repository offers, and the best way to run it.
|
||||
type PlannedArtifact struct {
|
||||
modelplan.Artifact
|
||||
Best *modelplan.Option `json:"best,omitempty"`
|
||||
}
|
||||
|
||||
// Plan is every way a repository's files could run here, best first.
|
||||
type Plan struct {
|
||||
Repo string `json:"repo"`
|
||||
Revision string `json:"revision"`
|
||||
Format string `json:"format"`
|
||||
Arch string `json:"architecture,omitempty"`
|
||||
License string `json:"license,omitempty"`
|
||||
Params int64 `json:"params,omitempty"`
|
||||
Context int `json:"context"`
|
||||
Gated bool `json:"gated,omitempty"`
|
||||
// Servable is false when the nodes' engine cannot run the model at all,
|
||||
// and Reason says why.
|
||||
Servable bool `json:"servable"`
|
||||
Reason string `json:"reason,omitempty"`
|
||||
// Note is anything the plan could not know, said once.
|
||||
Note string `json:"note,omitempty"`
|
||||
Projectors []string `json:"projectors,omitempty"`
|
||||
Artifacts []PlannedArtifact `json:"artifacts"`
|
||||
Options []modelplan.Option `json:"options"`
|
||||
}
|
||||
|
||||
// Recommended is the best option of all, if any runs.
|
||||
func (p Plan) Recommended() (modelplan.Option, bool) {
|
||||
if len(p.Options) == 0 {
|
||||
return modelplan.Option{}, false
|
||||
}
|
||||
return p.Options[0], true
|
||||
}
|
||||
|
||||
// ggufSets are a repository's GGUF weights: models, not projectors or
|
||||
// importance matrices.
|
||||
func ggufSets(repo hf.Repo) []hf.FileSet {
|
||||
var weights []hf.File
|
||||
for _, f := range repo.Files {
|
||||
lower := strings.ToLower(f.Path)
|
||||
if strings.HasSuffix(lower, ".gguf") && !strings.Contains(lower, "mmproj") && !strings.Contains(lower, "imatrix") {
|
||||
weights = append(weights, f)
|
||||
}
|
||||
}
|
||||
return hf.Group(weights)
|
||||
}
|
||||
|
||||
// Plan works out where a repository's files could run.
|
||||
func (s *Server) Plan(ctx context.Context, req PlanRequest) (Plan, error) {
|
||||
if s.deps.Pool == nil {
|
||||
return Plan{}, errors.New("this control plane cannot see its fleet here")
|
||||
}
|
||||
req.Repo = strings.Trim(strings.TrimSpace(req.Repo), "/")
|
||||
if !repoPattern.MatchString(req.Repo) {
|
||||
return Plan{}, fmt.Errorf("%q is not a repository; it looks like owner/name", req.Repo)
|
||||
}
|
||||
repo, err := s.deps.Registry.Repo(ctx, req.Repo)
|
||||
if err != nil {
|
||||
return Plan{}, err
|
||||
}
|
||||
return s.plan(ctx, repo, req)
|
||||
}
|
||||
|
||||
func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, error) {
|
||||
p := Plan{Repo: repo.ID, Revision: repo.Revision, Format: req.Format, Arch: repo.Architecture,
|
||||
License: repo.License, Params: repo.Params, Context: req.Context, Gated: repo.Gated.Is(), Servable: true,
|
||||
Projectors: Projectors(repo)}
|
||||
var arts []modelplan.Artifact
|
||||
var shape modelplan.Shape
|
||||
switch req.Format {
|
||||
case "", "gguf":
|
||||
p.Format = "gguf"
|
||||
if p.Context <= 0 {
|
||||
p.Context = 16384
|
||||
}
|
||||
sets := ggufSets(repo)
|
||||
if len(sets) == 0 {
|
||||
return p, fmt.Errorf("%s has no GGUF files", repo.ID)
|
||||
}
|
||||
small := sets[0]
|
||||
for _, st := range sets {
|
||||
arts = append(arts, modelplan.Artifact{Name: st.Name, Format: "gguf", Bytes: st.Bytes,
|
||||
Shards: st.Shards, Quant: modelplan.QuantOf(st.Name)})
|
||||
if st.Bytes < small.Bytes {
|
||||
small = st
|
||||
}
|
||||
}
|
||||
// Every quantisation of a model has its shape: the smallest file's
|
||||
// header says it, read from the registry without fetching the file.
|
||||
base := ""
|
||||
if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
|
||||
base = b.Base()
|
||||
}
|
||||
v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion)
|
||||
if v.Checked && !v.Servable {
|
||||
p.Servable, p.Reason = false, v.Reason
|
||||
}
|
||||
shape.KVBytesPerToken = v.Header.KVBytesPerToken()
|
||||
shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count
|
||||
if p.Arch == "" {
|
||||
p.Arch = v.Header.Architecture
|
||||
}
|
||||
case "safetensors":
|
||||
if p.Context <= 0 {
|
||||
p.Context = 8192
|
||||
}
|
||||
set, err := pickSafetensors(repo, Request{})
|
||||
if err != nil {
|
||||
return p, err
|
||||
}
|
||||
arts = append(arts, modelplan.Artifact{Name: "original", Format: "safetensors", Bytes: set.Bytes,
|
||||
Shards: len(set.Files), Quant: "bf16"})
|
||||
m, err := modelplan.FetchModel(ctx, repo.ID)
|
||||
if err == nil {
|
||||
shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16)
|
||||
shape.KVHeads = m.KVHeads
|
||||
if p.Params == 0 {
|
||||
p.Params = m.Params
|
||||
}
|
||||
} else {
|
||||
p.Note = "its config.json could not be read: " + err.Error()
|
||||
}
|
||||
default:
|
||||
return p, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
|
||||
}
|
||||
if shape.KVBytesPerToken == 0 && p.Note == "" {
|
||||
p.Note = "the model's shape was not readable, so the context's memory is not counted"
|
||||
}
|
||||
shape.Params = p.Params
|
||||
p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context)
|
||||
if !p.Servable {
|
||||
p.Options = nil
|
||||
}
|
||||
for _, a := range arts {
|
||||
pa := PlannedArtifact{Artifact: a}
|
||||
for i := range p.Options {
|
||||
if p.Options[i].Artifact == a.Name {
|
||||
pa.Best = &p.Options[i]
|
||||
break
|
||||
}
|
||||
}
|
||||
p.Artifacts = append(p.Artifacts, pa)
|
||||
}
|
||||
return p, nil
|
||||
}
|
||||
Reference in new issue
Block a user