Plan a repository's files against the fleet, and serve the one that suits it the way it fits
The control plane reads the model's shape from one GGUF header or its config.json and lists, for every file, the best way it runs here: one card, several of a node's (by layer for llama.cpp, tensor-parallel for vLLM), several nodes' over one fabric (vLLM, pipeline-parallel between them), or a CPU node; what each card holds and whether it is free now. ctl model plan prints it; model serve takes its file and way unless told, gives a GGUF on GPUs a GPU profile, and with --fallbacks adds the next best ways. The console shows it per file and asks for the way and the fallback. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
630d2788c0
commit
54db7aecb7
15 files changed
+1224
-158
No files matched your search
@@ -0,0 +1,356 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Options: the ways the files a repository actually has could run on the
|
||||
// cluster as it is — each file on one card, on several of a node's cards, on
|
||||
// several nodes' cards over a fabric, or on a CPU node — with what each card
|
||||
// would hold, and whether the cards are free now or only exist.
|
||||
|
||||
// Artifact is something a repository offers to serve: one GGUF file (or one
|
||||
// split into parts), or its original safetensors weights.
|
||||
type Artifact struct {
|
||||
// Name is the file, or the set's name; for original weights, "original".
|
||||
Name string `json:"name"`
|
||||
Format string `json:"format"` // "gguf" or "safetensors"
|
||||
Bytes int64 `json:"bytes"`
|
||||
Shards int `json:"shards,omitempty"`
|
||||
// Quant is the quantisation read from the name, like Q4_K_M; "bf16" for
|
||||
// original weights.
|
||||
Quant string `json:"quant,omitempty"`
|
||||
}
|
||||
|
||||
// Shape is what sizes a model's KV cache, and constrains how it splits.
|
||||
type Shape struct {
|
||||
// KVBytesPerToken is one token of context, the whole model.
|
||||
KVBytesPerToken int64
|
||||
// KVHeads must divide by a tensor-parallel width; 0 is unknown.
|
||||
KVHeads int
|
||||
// Params is the parameter count, for bits per weight; 0 is unknown.
|
||||
Params int64
|
||||
}
|
||||
|
||||
// PoolNode is one ready node, as a plan sees it.
|
||||
type PoolNode struct {
|
||||
ID string
|
||||
Class string // "" for a node with no GPU
|
||||
Fabric string
|
||||
GPUMemGB int
|
||||
GPUs int
|
||||
FreeGPUs int
|
||||
// HostMemGB is its RAM, and HostFreeGB what is unused now.
|
||||
HostMemGB int
|
||||
HostFreeGB int
|
||||
}
|
||||
|
||||
// Option is one way to run one artifact.
|
||||
type Option struct {
|
||||
Artifact string `json:"artifact"`
|
||||
Format string `json:"format"`
|
||||
Engine string `json:"engine"`
|
||||
CPU bool `json:"cpu,omitempty"`
|
||||
Class string `json:"gpu_class,omitempty"`
|
||||
Fabric string `json:"fabric,omitempty"`
|
||||
// GPUsPerNode cards on each of Nodes nodes.
|
||||
GPUsPerNode int `json:"gpus_per_node,omitempty"`
|
||||
Nodes int `json:"nodes,omitempty"`
|
||||
// NeedGB is what one card holds, or for CPU the whole replica in RAM;
|
||||
// HaveGB what a card, or the node, offers for it.
|
||||
NeedGB float64 `json:"need_gb"`
|
||||
HaveGB float64 `json:"have_gb"`
|
||||
// Hosts is how many nodes could run it at all, and FreeNow whether one
|
||||
// (or enough, together) could right now.
|
||||
Hosts int `json:"hosts"`
|
||||
FreeNow bool `json:"free_now"`
|
||||
// BitsPerWeight is the artifact's precision, 0 when unknown.
|
||||
BitsPerWeight float64 `json:"bits_per_weight,omitempty"`
|
||||
}
|
||||
|
||||
// GPUs is every card one replica takes.
|
||||
func (o Option) GPUs() int { return o.GPUsPerNode * max(o.Nodes, 1) }
|
||||
|
||||
// Label says where it runs, in a few words.
|
||||
func (o Option) Label() string {
|
||||
if o.CPU {
|
||||
return "CPU"
|
||||
}
|
||||
if o.Nodes > 1 {
|
||||
return fmt.Sprintf("%d nodes × %d %s on %s", o.Nodes, o.GPUsPerNode, o.Class, o.Fabric)
|
||||
}
|
||||
return fmt.Sprintf("%d × %s", o.GPUsPerNode, o.Class)
|
||||
}
|
||||
|
||||
// gpuShare is how much of a card a plan may count on, as in Fits.
|
||||
const gpuShare = usableFraction
|
||||
|
||||
// hostShare is how much of a node's RAM one replica may be planned to use:
|
||||
// the system and the agent keep the rest.
|
||||
const hostShare = 0.85
|
||||
|
||||
// ggufOverhead is llama.cpp's own cost beyond weights and cache, as the GGUF
|
||||
// header's MemoryBytes counts it: compute buffers and the server.
|
||||
func ggufOverhead(weights int64) int64 { return 1<<30 + weights/20 }
|
||||
|
||||
// Options lists every way each artifact could run, best first.
|
||||
//
|
||||
// context is the token pool: llama.cpp's --ctx-size, or vLLM's
|
||||
// --max-model-len for one sequence.
|
||||
func Options(arts []Artifact, sh Shape, nodes []PoolNode, context int) []Option {
|
||||
var out []Option
|
||||
for _, a := range arts {
|
||||
bpw := quantBits[strings.ToUpper(a.Quant)]
|
||||
if sh.Params > 0 {
|
||||
bpw = float64(a.Bytes) * 8 / float64(sh.Params)
|
||||
}
|
||||
kv := int64(context) * sh.KVBytesPerToken
|
||||
switch a.Format {
|
||||
case "gguf":
|
||||
out = append(out, ggufOptions(a, kv, nodes, bpw)...)
|
||||
case "safetensors":
|
||||
out = append(out, vllmOptions(a, kv, sh, nodes, bpw)...)
|
||||
}
|
||||
}
|
||||
Rank(out)
|
||||
return out
|
||||
}
|
||||
|
||||
func ggufOptions(a Artifact, kv int64, nodes []PoolNode, bpw float64) []Option {
|
||||
var out []Option
|
||||
// The CPU: the whole replica in a node's RAM.
|
||||
total := a.Bytes + kv + ggufOverhead(a.Bytes)
|
||||
cpu := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", CPU: true,
|
||||
NeedGB: GB(total), BitsPerWeight: bpw}
|
||||
for _, n := range nodes {
|
||||
have := float64(n.HostMemGB) * hostShare
|
||||
if n.HostMemGB <= 0 || GB(total) > have {
|
||||
continue
|
||||
}
|
||||
cpu.Hosts++
|
||||
cpu.HaveGB = max(cpu.HaveGB, have)
|
||||
if float64(n.HostFreeGB) >= GB(total) {
|
||||
cpu.FreeNow = true
|
||||
}
|
||||
}
|
||||
if cpu.Hosts > 0 {
|
||||
out = append(out, cpu)
|
||||
}
|
||||
// GPUs: one node's cards, the model split across them by layer.
|
||||
for _, c := range classes(nodes) {
|
||||
budget := float64(c.mem) * gpuShare
|
||||
for width := 1; width <= c.maxGPUs; width++ {
|
||||
per := (a.Bytes+kv)/int64(width) + ggufOverhead(a.Bytes/int64(width))
|
||||
if GB(per) > budget {
|
||||
continue
|
||||
}
|
||||
o := Option{Artifact: a.Name, Format: a.Format, Engine: "llamacpp", Class: c.class,
|
||||
GPUsPerNode: width, Nodes: 1, NeedGB: GB(per), HaveGB: budget, BitsPerWeight: bpw}
|
||||
for _, n := range nodes {
|
||||
if n.Class == c.class && n.GPUs >= width {
|
||||
o.Hosts++
|
||||
if n.FreeGPUs >= width {
|
||||
o.FreeNow = true
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, o)
|
||||
break // more cards than the fewest that hold it buys nothing here
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func vllmOptions(a Artifact, kv int64, sh Shape, nodes []PoolNode, bpw float64) []Option {
|
||||
var out []Option
|
||||
perCard := func(width int) int64 {
|
||||
return (a.Bytes+kv)/int64(width) + overheadBytes(a.Bytes/int64(width))
|
||||
}
|
||||
// Tensor parallel within a node must divide the KV heads.
|
||||
divides := func(w int) bool { return sh.KVHeads == 0 || (w <= sh.KVHeads && sh.KVHeads%w == 0) }
|
||||
for _, c := range classes(nodes) {
|
||||
budget := float64(c.mem) * gpuShare
|
||||
opt := func(perNode, n int, fabric string) Option {
|
||||
return Option{Artifact: a.Name, Format: a.Format, Engine: "vllm", Class: c.class, Fabric: fabric,
|
||||
GPUsPerNode: perNode, Nodes: n, NeedGB: GB(perCard(perNode * n)), HaveGB: budget, BitsPerWeight: bpw}
|
||||
}
|
||||
// One node: the narrowest power of two that holds it.
|
||||
single := false
|
||||
for w := 1; w <= c.maxGPUs && divides(w); w *= 2 {
|
||||
if GB(perCard(w)) > budget {
|
||||
continue
|
||||
}
|
||||
o := opt(w, 1, "")
|
||||
for _, fab := range c.fabrics {
|
||||
for _, nd := range fab.nodes {
|
||||
if nd.GPUs >= w {
|
||||
o.Hosts++
|
||||
o.FreeNow = o.FreeNow || nd.FreeGPUs >= w
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, o)
|
||||
single = true
|
||||
break
|
||||
}
|
||||
if single {
|
||||
continue
|
||||
}
|
||||
// Several nodes, a node's full width on each, pipeline-parallel
|
||||
// between them: only within one fabric.
|
||||
w := 1
|
||||
for w*2 <= c.maxGPUs {
|
||||
w *= 2
|
||||
}
|
||||
if !divides(w) {
|
||||
continue
|
||||
}
|
||||
for _, fab := range c.fabrics {
|
||||
if fab.name == "" {
|
||||
continue
|
||||
}
|
||||
for n := 2; n <= len(fab.nodes); n *= 2 {
|
||||
if GB(perCard(w*n)) > budget {
|
||||
continue
|
||||
}
|
||||
o := opt(w, n, fab.name)
|
||||
free := 0
|
||||
for _, nd := range fab.nodes {
|
||||
if nd.GPUs >= w {
|
||||
o.Hosts++
|
||||
}
|
||||
if nd.FreeGPUs >= w {
|
||||
free++
|
||||
}
|
||||
}
|
||||
if o.Hosts < n {
|
||||
break
|
||||
}
|
||||
o.FreeNow = free >= n
|
||||
out = append(out, o)
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type classPool struct {
|
||||
class string
|
||||
mem int
|
||||
maxGPUs int
|
||||
fabrics []fabricPool
|
||||
}
|
||||
|
||||
type fabricPool struct {
|
||||
name string
|
||||
nodes []PoolNode
|
||||
}
|
||||
|
||||
// classes groups GPU nodes by card, and each by fabric: a replica spans
|
||||
// nodes only within one.
|
||||
func classes(nodes []PoolNode) []classPool {
|
||||
by := map[string]*classPool{}
|
||||
var order []string
|
||||
for _, n := range nodes {
|
||||
if n.Class == "" || n.GPUs == 0 || n.GPUMemGB == 0 {
|
||||
continue
|
||||
}
|
||||
c := by[n.Class]
|
||||
if c == nil {
|
||||
c = &classPool{class: n.Class, mem: n.GPUMemGB}
|
||||
by[n.Class] = c
|
||||
order = append(order, n.Class)
|
||||
}
|
||||
c.maxGPUs = max(c.maxGPUs, n.GPUs)
|
||||
i := -1
|
||||
for j := range c.fabrics {
|
||||
if c.fabrics[j].name == n.Fabric {
|
||||
i = j
|
||||
}
|
||||
}
|
||||
if i < 0 {
|
||||
c.fabrics = append(c.fabrics, fabricPool{name: n.Fabric})
|
||||
i = len(c.fabrics) - 1
|
||||
}
|
||||
c.fabrics[i].nodes = append(c.fabrics[i].nodes, n)
|
||||
}
|
||||
sort.Strings(order)
|
||||
out := make([]classPool, 0, len(order))
|
||||
for _, k := range order {
|
||||
out = append(out, *by[k])
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// quantBits is roughly what each quantisation spends a weight, for when the
|
||||
// parameter count is not known to work it out from the file's size.
|
||||
var quantBits = map[string]float64{
|
||||
"F32": 32, "F16": 16, "BF16": 16, "FP16": 16, "Q8_0": 8.5, "Q6_K": 6.6,
|
||||
"Q5_K_M": 5.7, "Q5_K_S": 5.5, "Q5_0": 5.5, "Q5_1": 6, "Q4_K_M": 4.8, "Q4_K_S": 4.6, "Q4_0": 4.5, "Q4_1": 5,
|
||||
"IQ4_XS": 4.3, "IQ4_NL": 4.5, "Q3_K_L": 4.3, "Q3_K_M": 3.9, "Q3_K_S": 3.5, "IQ3_M": 3.7, "IQ3_XS": 3.3,
|
||||
"IQ3_XXS": 3.1, "Q2_K": 3.0, "IQ2_M": 2.7, "IQ2_XS": 2.3, "IQ2_XXS": 2.1, "IQ1_M": 1.8, "IQ1_S": 1.6,
|
||||
}
|
||||
|
||||
// qualityCap is the precision past which a bigger file is not a better
|
||||
// model worth its memory: eight bits keep what sixteen do, near enough.
|
||||
const qualityCap = 8.6
|
||||
|
||||
func quality(o Option) float64 {
|
||||
if o.BitsPerWeight == 0 {
|
||||
return 0
|
||||
}
|
||||
return min(o.BitsPerWeight, qualityCap)
|
||||
}
|
||||
|
||||
// Rank orders options best first: what can start now; on a GPU before the
|
||||
// CPU; the fewest cards, then nodes; then the better precision, up to eight
|
||||
// bits; then the most room left.
|
||||
func Rank(opts []Option) {
|
||||
sort.SliceStable(opts, func(i, j int) bool {
|
||||
a, b := opts[i], opts[j]
|
||||
if a.FreeNow != b.FreeNow {
|
||||
return a.FreeNow
|
||||
}
|
||||
if a.CPU != b.CPU {
|
||||
return !a.CPU
|
||||
}
|
||||
if a.GPUs() != b.GPUs() {
|
||||
return a.GPUs() < b.GPUs()
|
||||
}
|
||||
if a.Nodes != b.Nodes {
|
||||
return a.Nodes < b.Nodes
|
||||
}
|
||||
if qa, qb := quality(a), quality(b); qa != qb {
|
||||
return qa > qb
|
||||
}
|
||||
return a.HaveGB-a.NeedGB > b.HaveGB-b.NeedGB
|
||||
})
|
||||
}
|
||||
|
||||
var quantPattern = regexp.MustCompile(`(?i)(IQ\d_[A-Z]+|Q\d_K_[SML]|Q\d_K|Q\d_\d|Q\d|BF16|F16|F32|FP16)`)
|
||||
|
||||
// QuantOf reads a quantisation out of a file name.
|
||||
func QuantOf(name string) string {
|
||||
m := quantPattern.FindAllString(name, -1)
|
||||
if len(m) == 0 {
|
||||
return ""
|
||||
}
|
||||
return strings.ToUpper(m[len(m)-1])
|
||||
}
|
||||
|
||||
// MemFromClass reads the card size out of a class label such as "h100-80g".
|
||||
func MemFromClass(class string) int {
|
||||
i := strings.LastIndex(class, "-")
|
||||
if i < 0 || !strings.HasSuffix(class, "g") {
|
||||
return 0
|
||||
}
|
||||
var gb int
|
||||
if _, err := fmt.Sscanf(class[i+1:], "%dg", &gb); err != nil {
|
||||
return 0
|
||||
}
|
||||
return gb
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
const gib = int64(1) << 30
|
||||
|
||||
// A 14B GGUF: 48 layers, 8 KV heads of 128 — 192 KB a token at f16.
|
||||
var shape14B = Shape{KVBytesPerToken: 48 * 8 * 256 * 2, KVHeads: 8, Params: 14_800_000_000}
|
||||
|
||||
func labels(opts []Option) []string {
|
||||
var out []string
|
||||
for _, o := range opts {
|
||||
out = append(out, o.Artifact+" "+o.Label())
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestEachFileIsMatchedToTheCardsThatHoldIt(t *testing.T) {
|
||||
nodes := []PoolNode{
|
||||
{ID: "gpu1", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 64, HostFreeGB: 40},
|
||||
{ID: "cpu1", HostMemGB: 32, HostFreeGB: 30},
|
||||
}
|
||||
arts := []Artifact{
|
||||
{Name: "q4.gguf", Format: "gguf", Bytes: 9 * gib},
|
||||
{Name: "q8.gguf", Format: "gguf", Bytes: 15 * gib},
|
||||
{Name: "f16.gguf", Format: "gguf", Bytes: 29 * gib},
|
||||
}
|
||||
opts := Options(arts, shape14B, nodes, 16384)
|
||||
got := strings.Join(labels(opts), "; ")
|
||||
// The best: the better precision on the one card, free now.
|
||||
if opts[0].Artifact != "q8.gguf" || opts[0].CPU || !opts[0].FreeNow {
|
||||
t.Errorf("first: %+v (all: %s)", opts[0], got)
|
||||
}
|
||||
// f16 fits no card, only a CPU node with the RAM.
|
||||
for _, o := range opts {
|
||||
if o.Artifact == "f16.gguf" && !o.CPU {
|
||||
t.Errorf("f16 placed on a card: %+v", o)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(got, "f16.gguf CPU") {
|
||||
t.Errorf("f16 not offered the CPU: %s", got)
|
||||
}
|
||||
// The card busy: what is free comes first, which is the CPU.
|
||||
nodes[0].FreeGPUs = 0
|
||||
opts = Options(arts, shape14B, nodes, 16384)
|
||||
if !opts[0].CPU || !opts[0].FreeNow {
|
||||
t.Errorf("with the card busy: %+v", opts[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestALlamaCppModelSpreadsOverANodesCardsByLayer(t *testing.T) {
|
||||
nodes := []PoolNode{{ID: "a", Class: "l4-24g", GPUMemGB: 24, GPUs: 4, FreeGPUs: 4}}
|
||||
opts := Options([]Artifact{{Name: "70b-q4.gguf", Format: "gguf", Bytes: 40 * gib}},
|
||||
Shape{KVBytesPerToken: 80 * 8 * 256 * 2}, nodes, 8192)
|
||||
if len(opts) != 1 || opts[0].GPUsPerNode != 3 || opts[0].Nodes != 1 {
|
||||
t.Fatalf("%+v", opts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginalWeightsSpanNodesOnlyOverAFabric(t *testing.T) {
|
||||
var nodes []PoolNode
|
||||
for _, id := range []string{"a", "b"} {
|
||||
nodes = append(nodes, PoolNode{ID: id, Class: "h100-80g", GPUMemGB: 80, GPUs: 8, FreeGPUs: 8, Fabric: "ib-0"})
|
||||
}
|
||||
llama70 := Shape{KVBytesPerToken: 80 * 8 * 256 * 2, KVHeads: 8, Params: 70e9}
|
||||
opts := Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 140 * gib}}, llama70, nodes, 8192)
|
||||
if len(opts) != 1 || opts[0].GPUsPerNode != 4 || opts[0].Nodes != 1 || opts[0].Engine != "vllm" {
|
||||
t.Fatalf("70B: %+v", opts)
|
||||
}
|
||||
big := Shape{KVBytesPerToken: 126 * 8 * 256 * 2, KVHeads: 8, Params: 405e9}
|
||||
opts = Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 810 * gib}}, big, nodes, 8192)
|
||||
if len(opts) != 1 || opts[0].Nodes != 2 || opts[0].GPUsPerNode != 8 || opts[0].Fabric != "ib-0" || !opts[0].FreeNow {
|
||||
t.Fatalf("405B: %+v", opts)
|
||||
}
|
||||
// No fabric, no replica across the two.
|
||||
for i := range nodes {
|
||||
nodes[i].Fabric = ""
|
||||
}
|
||||
if opts = Options([]Artifact{{Name: "original", Format: "safetensors", Bytes: 810 * gib}}, big, nodes, 8192); len(opts) != 0 {
|
||||
t.Fatalf("spanned nodes with no fabric: %+v", opts)
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuantisationsAreReadFromNames(t *testing.T) {
|
||||
for name, want := range map[string]string{
|
||||
"Qwen2.5-7B-Instruct-Q4_K_M.gguf": "Q4_K_M",
|
||||
"qwen2.5-coder-14b-instruct-q4_k_m-00001-of-00002.gguf": "Q4_K_M",
|
||||
"phi-4-IQ3_XS.gguf": "IQ3_XS",
|
||||
"model-f16.gguf": "F16",
|
||||
"JiRackUltra_1b.Q6_K.gguf": "Q6_K",
|
||||
} {
|
||||
if got := QuantOf(name); got != want {
|
||||
t.Errorf("%s: %s", name, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user