Plan a repository's files against the fleet, and serve the one that suits it the way it fits

The control plane reads the model's shape from one GGUF header or its
config.json and lists, for every file, the best way it runs here: one card,
several of a node's (by layer for llama.cpp, tensor-parallel for vLLM),
several nodes' over one fabric (vLLM, pipeline-parallel between them), or a
CPU node; what each card holds and whether it is free now. ctl model plan
prints it; model serve takes its file and way unless told, gives a GGUF on
GPUs a GPU profile, and with --fallbacks adds the next best ways. The
console shows it per file and asks for the way and the fallback.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-04 01:03:48 +02:00
1 parent 630d2788c0
commit 54db7aecb7
15 files changed
+1224 -158

No files matched your search

+52
View File
@@ -22,6 +22,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/release"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
@@ -126,6 +127,7 @@ func (a *API) Register(mux *http.ServeMux) {
mux.HandleFunc("POST /api/v1/models/{name}/update", a.updateModel)
mux.HandleFunc("POST /api/v1/models/{name}/rollback", a.rollbackModel)
mux.HandleFunc("POST /api/v1/models/{name}/origin", a.setOrigin)
mux.HandleFunc("GET /api/v1/plan", a.planModel)
mux.HandleFunc("GET /api/v1/requests/{id}", a.request)
mux.HandleFunc("POST /api/v1/models/{name}/retry", a.retryModel)
mux.HandleFunc("GET /api/v1/models/{name}/headroom", func(w http.ResponseWriter, r *http.Request) {
@@ -1544,6 +1546,56 @@ func (a *API) setOrigin(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, OriginView{Repo: o.Repo, Revision: o.Revision})
}
// PoolNodes is the fleet's ready nodes as a plan sees them: their cards,
// which of them are free, the fabric they share, and their memory.
func (a *API) PoolNodes() []modelplan.PoolNode {
var out []modelplan.PoolNode
for _, n := range a.Nodes() {
if n.State != "ready" {
continue
}
p := modelplan.PoolNode{ID: n.ID, GPUs: n.GPUTotal, FreeGPUs: n.GPUFree, HostMemGB: n.MemoryGB,
HostFreeGB: n.MemoryGB}
if len(n.Classes) > 0 {
p.Class = n.Classes[0]
p.GPUMemGB = modelplan.MemFromClass(p.Class)
}
if len(n.Fabrics) > 0 {
p.Fabric = n.Fabrics[0]
}
if l := n.Load; l != nil && l.MemTotal > 0 {
p.HostFreeGB = int((l.MemTotal - l.MemUsed) >> 30)
}
out = append(out, p)
}
return out
}
func (a *API) planModel(w http.ResponseWriter, r *http.Request) {
if a.serve == nil {
writeError(w, http.StatusNotImplemented, "planning needs a mirror and its registry, and this control plane lacks one")
return
}
q := r.URL.Query()
req := modelserve.PlanRequest{Repo: q.Get("repo"), Format: q.Get("format")}
if c := q.Get("context"); c != "" {
n, err := strconv.Atoi(c)
if err != nil || n < 0 {
writeError(w, http.StatusBadRequest, "context is a whole number of tokens")
return
}
req.Context = n
}
ctx, cancel := context.WithTimeout(r.Context(), time.Minute)
defer cancel()
p, err := a.serve.Plan(ctx, req)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, p)
}
// actorOf is who a change is recorded as made by: what the caller says, since
// the admin API answers only this machine and trusts whoever reaches it. Kept
// to a short printable name, so a claim cannot pass for more than one — no
+142 -25
View File
@@ -30,6 +30,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
// Registry reads a repository.
@@ -77,6 +78,10 @@ type Deps struct {
Current func() []catalog.Model
Models Models
Restart func(ids ...string) error
// Pool is the fleet's ready nodes as a plan sees them: what decides
// which file suits the hardware, and how it is spread.
Pool func() []modelplan.PoolNode
}
// Request is what to serve.
@@ -93,9 +98,18 @@ type Request struct {
// Format is "gguf", run by llama.cpp, or "safetensors": the original
// weights with their config and tokenizer, run by vLLM on GPUs of
// GPUClass, across TP of them.
Format string `json:"format,omitempty"`
GPUClass string `json:"gpu_class,omitempty"`
TP int `json:"tp,omitempty"`
Format string `json:"format,omitempty"`
// Where it runs: on the CPU, or on GPUsPerNode cards of GPUClass on
// each of Nodes nodes. Left empty, the plan decides: the best way the
// cluster can run the chosen file. TP is GPUsPerNode's older name.
CPU bool `json:"cpu,omitempty"`
GPUClass string `json:"gpu_class,omitempty"`
GPUsPerNode int `json:"gpus_per_node,omitempty"`
Nodes int `json:"nodes,omitempty"`
TP int `json:"tp,omitempty"`
// Fallbacks adds the next best ways, lower in priority, so a model whose
// first choice is busy runs on another rather than waiting.
Fallbacks bool `json:"fallbacks,omitempty"`
// Name is the catalog name; the repository suggests one otherwise.
Name string `json:"name,omitempty"`
Aliases []string `json:"aliases,omitempty"`
@@ -192,19 +206,15 @@ func (s *Server) Start(req Request) (Job, error) {
case "", "gguf":
req.Format = "gguf"
case "safetensors":
if req.GPUClass == "" {
return Job{}, errors.New("safetensors weights run on vLLM, on GPUs: name the GPU class, as `llmgw ctl status` lists it")
}
if req.Projector != "" {
return Job{}, errors.New("a projector is for a GGUF model; vLLM reads a vision model's own files")
}
req.TP = max(req.TP, 1)
if req.CPU {
return Job{}, errors.New("original weights run on vLLM, on GPUs; a GGUF file runs on the CPU")
}
default:
return Job{}, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
}
if req.Quant == "" && req.File == "" {
req.Quant = "Q4_K_M"
}
if req.CtxPool <= 0 {
req.CtxPool = 16384
}
@@ -274,6 +284,30 @@ func (s *Server) run(j *Job) {
s.fail(j, "registry", fmt.Errorf("%s is gated: it needs an account's approval the mirror does not have", req.Repo))
return
}
if req.TP > 0 && req.GPUsPerNode == 0 {
req.GPUsPerNode = req.TP
}
// The plan: which file suits the hardware, and how it spreads over it.
planCtx := req.CtxPool
if req.Format == "safetensors" {
planCtx = req.ContextLength
}
var plan Plan
planned := false
if s.deps.Pool != nil {
if plan, err = s.plan(ctx, repo, PlanRequest{Repo: repo.ID, Format: req.Format, Context: planCtx}); err == nil {
planned = true
} else {
s.deps.Log.Warn("planning a model", "repo", repo.ID, "err", err)
}
}
if req.Format == "gguf" && req.Quant == "" && req.File == "" {
if rec, ok := plan.Recommended(); planned && ok {
req.File = rec.Artifact
} else {
req.Quant = "Q4_K_M"
}
}
pickSet := pick
if req.Format == "safetensors" {
pickSet = pickSafetensors
@@ -283,6 +317,11 @@ func (s *Server) run(j *Job) {
s.fail(j, "registry", err)
return
}
layout, err := chooseLayout(req, plan, planned, set)
if err != nil {
s.fail(j, "registry", err)
return
}
if req.Projector != "" {
i := slices.IndexFunc(repo.Files, func(f hf.File) bool {
return (f.Path == req.Projector || mirror.LastPathElement(f.Path) == req.Projector) &&
@@ -303,8 +342,12 @@ func (s *Server) run(j *Job) {
s.mu.Lock()
j.Model = name
s.mu.Unlock()
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s",
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision()))
where := make([]string, 0, len(layout))
for _, o := range layout {
where = append(where, o.Label())
}
s.step(j, "registry", Done, fmt.Sprintf("%s, %.1f GB in %d file(s), revision %s; runs on %s",
set.Name, gb(set.Bytes), len(set.Files), repo.ShortRevision(), strings.Join(where, ", else ")))
// 2. The mirror: fetch, and check what arrived against what was published.
source, err := s.mirror(ctx, j, repo, set, name)
@@ -313,15 +356,12 @@ func (s *Server) run(j *Job) {
return
}
// 3. Whether the nodes' llama.cpp loads it, and how much memory it needs.
// 3. Whether the nodes' engine loads it, and how much memory it needs.
if req.Format == "safetensors" {
// vLLM reads the model's shape from its config.json, which came
// with the weights; the GPUs it needs are the class and width asked.
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM on %d × %s", gb(set.Bytes), req.TP, req.GPUClass))
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
Name: "gpu", Engine: "vllm", GPUClass: req.GPUClass, TP: req.TP, Weight: 1, Priority: 100,
Args: []string{"--max-model-len", strconv.Itoa(req.ContextLength)},
})
// with the weights.
s.step(j, "check", Done, fmt.Sprintf("%.1f GB of weights for vLLM", gb(set.Bytes)))
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0))
return
}
s.step(j, "check", Running, "reading its header from the mirror")
@@ -351,22 +391,99 @@ func (s *Server) run(j *Job) {
cores = catalog.SuggestCores(nc.NodeCPUs(memGB))
}
}
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, memGB, cores))
}
s.addAndStart(ctx, j, repo, set, name, source, catalog.Profile{
Name: "cpu", Engine: "llamacpp", Weight: 1, Priority: 100,
Args: []string{"--ctx-size", strconv.Itoa(req.CtxPool)}, MinMemoryGB: memGB, CPUCores: cores,
})
// chooseLayout is where a model runs: what was asked, else the plan's best
// for the chosen file; with Fallbacks, the next best of other shapes after.
func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]modelplan.Option, error) {
artifact := set.Name
engine := "llamacpp"
if req.Format == "safetensors" {
artifact, engine = "original", "vllm"
}
var opts []modelplan.Option
for _, o := range plan.Options {
if o.Artifact == artifact {
opts = append(opts, o)
}
}
var out []modelplan.Option
switch {
case req.CPU || req.GPUClass != "":
o := modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: req.CPU}
if !req.CPU {
o.Class, o.GPUsPerNode, o.Nodes = req.GPUClass, max(req.GPUsPerNode, 1), max(req.Nodes, 1)
}
if o.Nodes > 1 && engine == "llamacpp" {
return nil, errors.New("llama.cpp does not span nodes; serve the original weights with vLLM for that")
}
out = append(out, o)
case len(opts) > 0:
out = append(out, opts[0])
case planned:
return nil, fmt.Errorf("%s fits nowhere in this cluster with a %d-token context: no card holds it, "+
"nor does any node's memory; `llmgw ctl model plan %s` shows what each file needs", set.Name, plan.Context, plan.Repo)
case engine == "vllm":
return nil, errors.New("original weights run on vLLM, on GPUs: name the GPU class")
default:
out = append(out, modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: true})
}
if req.Fallbacks {
shape := func(o modelplan.Option) string {
return fmt.Sprintf("%v/%s/%d/%d", o.CPU, o.Class, o.GPUsPerNode, o.Nodes)
}
seen := map[string]bool{shape(out[0]): true}
for _, o := range opts {
if len(out) == 3 {
break
}
if !seen[shape(o)] {
seen[shape(o)] = true
out = append(out, o)
}
}
}
return out, nil
}
// profilesFor turns the ways chosen into catalog profiles, the first
// preferred.
func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []catalog.Profile {
var out []catalog.Profile
for i, o := range layout {
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i}
switch {
case o.CPU:
p.Name = "cpu"
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
p.MinMemoryGB, p.CPUCores = memGB, cores
case o.Engine == "llamacpp":
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUsPerNode, o.Class)
p.GPUClass, p.GPUsPerNode = o.Class, o.GPUsPerNode
p.Args = []string{"--ctx-size", strconv.Itoa(req.CtxPool)}
default:
p.Name = fmt.Sprintf("gpu-%dx%s", o.GPUs(), o.Class)
p.GPUClass, p.TP = o.Class, o.GPUsPerNode
if o.Nodes > 1 {
p.PP, p.Nodes, p.GPUsPerNode = o.Nodes, o.Nodes, o.GPUsPerNode
}
p.Args = []string{"--max-model-len", strconv.Itoa(req.ContextLength)}
}
out = append(out, p)
}
return out
}
// addAndStart puts the model in the catalog with its profile, waits for a
// replica and asks it to answer.
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, prof catalog.Profile) {
func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.FileSet, name, source string, profs []catalog.Profile) {
req := j.Request
// 4. The catalog.
m := catalog.Model{
Name: name, Aliases: req.Aliases, Description: req.Description, Source: source,
RoutingProfile: "chat", ContextLength: req.ContextLength, MaxOutput: 1024,
Profiles: []catalog.Profile{prof},
Profiles: profs,
Scaling: catalog.Scaling{Min: req.Replicas, Max: max(req.Replicas, 1)},
Capabilities: catalog.Capabilities{Vision: req.Projector != ""},
Origin: &catalog.Origin{Repo: repo.ID, Revision: repo.Revision, Files: originFiles(set)},
+56 -3
View File
@@ -16,6 +16,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
// ggufHeader is enough of a GGUF file for its architecture and shape.
@@ -38,7 +39,12 @@ func ggufHeader() []byte {
return b.Bytes()
}
type fakeRegistry struct{ repo hf.Repo }
type fakeRegistry struct {
repo hf.Repo
base string
}
func (f fakeRegistry) Base() string { return f.base }
func (f fakeRegistry) Repo(context.Context, string) (hf.Repo, error) { return f.repo, nil }
@@ -265,8 +271,10 @@ func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T)
{Path: "original/consolidated.safetensors", Bytes: 7 << 30},
}
s, m, c, _ := setup(t, files, map[string]string{"model-00001-of-00002.safetensors": "a1", "model-00002-of-00002.safetensors": "a2"})
if _, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors"}); err == nil {
t.Error("safetensors with no GPU class was taken")
// With no fleet to plan against, the GPUs must be named.
job0, _ := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", Name: "x"})
if j := wait(t, s, job0.ID); j.State != Failed || !strings.Contains(j.Error, "GPU class") {
t.Errorf("safetensors with no GPU class: %+v", j)
}
job, err := s.Start(Request{Repo: "Qwen/Qwen2.5-7B-Instruct", Format: "safetensors", GPUClass: "l40s-48g", Replicas: 1})
if err != nil {
@@ -289,3 +297,48 @@ func TestOriginalWeightsAreServedOnVLLMWithTheirConfigAndTokenizer(t *testing.T)
t.Errorf("%s %+v", mod.Source, p)
}
}
func TestTheFileThatSuitsTheCardsIsChosenAndRunsOnThem(t *testing.T) {
files := []hf.File{
{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "a"},
{Path: "m-Q8_0.gguf", Bytes: 2 << 30, SHA256: "b"},
{Path: "m-F16.gguf", Bytes: 40 << 30, SHA256: "c"},
}
s, m, c, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "a", "m-Q8_0.gguf": "b"})
s.deps.Registry = fakeRegistry{repo: hf.Repo{ID: "o/m-GGUF", Revision: "abcdef123456", Files: files}, base: m.srv.URL}
s.deps.Pool = func() []modelplan.PoolNode {
return []modelplan.PoolNode{
{ID: "g", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 64, HostFreeGB: 60},
{ID: "c", HostMemGB: 16, HostFreeGB: 14},
}
}
p, err := s.Plan(context.Background(), PlanRequest{Repo: "o/m-GGUF"})
if err != nil {
t.Fatal(err)
}
rec, _ := p.Recommended()
if rec.Artifact != "m-Q8_0.gguf" || rec.CPU || rec.Class != "6000-24g" || p.Artifacts[2].Best == nil || !p.Artifacts[2].Best.CPU {
t.Fatalf("plan: %+v; f16: %+v", rec, p.Artifacts[2].Best)
}
job, err := s.Start(Request{Repo: "o/m-GGUF", Replicas: 1, Fallbacks: true})
if err != nil {
t.Fatal(err)
}
if j := wait(t, s, job.ID); j.State != Done {
t.Fatalf("%s %+v", j.Error, j.Steps)
}
if m.got[0].File != "m-Q8_0.gguf" {
t.Errorf("fetched %s", m.got[0].File)
}
profs := c.added[0].Profiles
if len(profs) != 2 || profs[0].GPUClass != "6000-24g" || profs[0].GPUsPerNode != 1 || profs[0].Engine != "llamacpp" ||
profs[1].Name != "cpu" || profs[1].Priority >= profs[0].Priority || profs[1].MinMemoryGB == 0 {
t.Errorf("profiles: %+v", profs)
}
// A file too big for any card or any node's memory: refused, saying so.
s.deps.Pool = func() []modelplan.PoolNode { return []modelplan.PoolNode{{ID: "c", HostMemGB: 16, HostFreeGB: 14}} }
job, _ = s.Start(Request{Repo: "o/m-GGUF", File: "m-F16.gguf", Name: "big"})
if j := wait(t, s, job.ID); j.State != Failed || !strings.Contains(j.Error, "fits nowhere") {
t.Errorf("%+v", j)
}
}
+175
View File
@@ -0,0 +1,175 @@
package modelserve
import (
"context"
"errors"
"fmt"
"strings"
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
// Planning: which of a repository's files suits the cluster's hardware, and
// how — one card, several of a node's, several nodes' over a fabric, or a
// CPU node — worked out from the files themselves before any is fetched.
// The model's shape is read from one GGUF file's header, a few megabytes
// from the registry, or from its config.json; the hardware is the fleet as
// it is now.
// PlanRequest is what to plan.
type PlanRequest struct {
Repo string `json:"repo"`
Format string `json:"format,omitempty"` // gguf (default) or safetensors
// Context is the token pool: llama.cpp's --ctx-size, or vLLM's
// --max-model-len. 0 takes 16384 and 8192.
Context int `json:"context,omitempty"`
}
// PlannedArtifact is one file a repository offers, and the best way to run it.
type PlannedArtifact struct {
modelplan.Artifact
Best *modelplan.Option `json:"best,omitempty"`
}
// Plan is every way a repository's files could run here, best first.
type Plan struct {
Repo string `json:"repo"`
Revision string `json:"revision"`
Format string `json:"format"`
Arch string `json:"architecture,omitempty"`
License string `json:"license,omitempty"`
Params int64 `json:"params,omitempty"`
Context int `json:"context"`
Gated bool `json:"gated,omitempty"`
// Servable is false when the nodes' engine cannot run the model at all,
// and Reason says why.
Servable bool `json:"servable"`
Reason string `json:"reason,omitempty"`
// Note is anything the plan could not know, said once.
Note string `json:"note,omitempty"`
Projectors []string `json:"projectors,omitempty"`
Artifacts []PlannedArtifact `json:"artifacts"`
Options []modelplan.Option `json:"options"`
}
// Recommended is the best option of all, if any runs.
func (p Plan) Recommended() (modelplan.Option, bool) {
if len(p.Options) == 0 {
return modelplan.Option{}, false
}
return p.Options[0], true
}
// ggufSets are a repository's GGUF weights: models, not projectors or
// importance matrices.
func ggufSets(repo hf.Repo) []hf.FileSet {
var weights []hf.File
for _, f := range repo.Files {
lower := strings.ToLower(f.Path)
if strings.HasSuffix(lower, ".gguf") && !strings.Contains(lower, "mmproj") && !strings.Contains(lower, "imatrix") {
weights = append(weights, f)
}
}
return hf.Group(weights)
}
// Plan works out where a repository's files could run.
func (s *Server) Plan(ctx context.Context, req PlanRequest) (Plan, error) {
if s.deps.Pool == nil {
return Plan{}, errors.New("this control plane cannot see its fleet here")
}
req.Repo = strings.Trim(strings.TrimSpace(req.Repo), "/")
if !repoPattern.MatchString(req.Repo) {
return Plan{}, fmt.Errorf("%q is not a repository; it looks like owner/name", req.Repo)
}
repo, err := s.deps.Registry.Repo(ctx, req.Repo)
if err != nil {
return Plan{}, err
}
return s.plan(ctx, repo, req)
}
func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan, error) {
p := Plan{Repo: repo.ID, Revision: repo.Revision, Format: req.Format, Arch: repo.Architecture,
License: repo.License, Params: repo.Params, Context: req.Context, Gated: repo.Gated.Is(), Servable: true,
Projectors: Projectors(repo)}
var arts []modelplan.Artifact
var shape modelplan.Shape
switch req.Format {
case "", "gguf":
p.Format = "gguf"
if p.Context <= 0 {
p.Context = 16384
}
sets := ggufSets(repo)
if len(sets) == 0 {
return p, fmt.Errorf("%s has no GGUF files", repo.ID)
}
small := sets[0]
for _, st := range sets {
arts = append(arts, modelplan.Artifact{Name: st.Name, Format: "gguf", Bytes: st.Bytes,
Shards: st.Shards, Quant: modelplan.QuantOf(st.Name)})
if st.Bytes < small.Bytes {
small = st
}
}
// Every quantisation of a model has its shape: the smallest file's
// header says it, read from the registry without fetching the file.
base := ""
if b, ok := s.deps.Registry.(interface{ Base() string }); ok {
base = b.Base()
}
v := gguf.Check(ctx, repo.DownloadURL(base, small.Files[0].Path), s.deps.LlamaVersion)
if v.Checked && !v.Servable {
p.Servable, p.Reason = false, v.Reason
}
shape.KVBytesPerToken = v.Header.KVBytesPerToken()
shape.KVHeads = int(v.Header.HeadCountKV) //nolint:gosec // a header's head count
if p.Arch == "" {
p.Arch = v.Header.Architecture
}
case "safetensors":
if p.Context <= 0 {
p.Context = 8192
}
set, err := pickSafetensors(repo, Request{})
if err != nil {
return p, err
}
arts = append(arts, modelplan.Artifact{Name: "original", Format: "safetensors", Bytes: set.Bytes,
Shards: len(set.Files), Quant: "bf16"})
m, err := modelplan.FetchModel(ctx, repo.ID)
if err == nil {
shape.KVBytesPerToken = m.KVBytesPerToken(modelplan.BF16)
shape.KVHeads = m.KVHeads
if p.Params == 0 {
p.Params = m.Params
}
} else {
p.Note = "its config.json could not be read: " + err.Error()
}
default:
return p, fmt.Errorf("the format is gguf or safetensors, not %q", req.Format)
}
if shape.KVBytesPerToken == 0 && p.Note == "" {
p.Note = "the model's shape was not readable, so the context's memory is not counted"
}
shape.Params = p.Params
p.Options = modelplan.Options(arts, shape, s.deps.Pool(), p.Context)
if !p.Servable {
p.Options = nil
}
for _, a := range arts {
pa := PlannedArtifact{Artifact: a}
for i := range p.Options {
if p.Options[i].Artifact == a.Name {
pa.Best = &p.Options[i]
break
}
}
p.Artifacts = append(p.Artifacts, pa)
}
return p, nil
}