CHANGELOG: one list
This commit is contained in:
1 parent
7465c24830
commit
d6f8f433f4
24 files changed
+437
-22
No files matched your search
@@ -53,6 +53,8 @@ type PoolNode struct {
|
||||
// card beside it another. Empty means Class, GPUMemGB, GPUs and FreeGPUs
|
||||
// say it all.
|
||||
Kinds []PoolKind
|
||||
// Engines are the engines the node has installed.
|
||||
Engines []string
|
||||
}
|
||||
|
||||
// PoolKind is one class of a node's devices.
|
||||
@@ -110,6 +112,11 @@ type Option struct {
|
||||
// cards, whole, rather than slicing each across them: vLLM's
|
||||
// --enable-expert-parallel, for a model of experts on more than one card.
|
||||
ExpertParallel bool `json:"expert_parallel,omitempty"`
|
||||
// StrataFamily and StrataSize are Strata's --family and --size, for an
|
||||
// option on Strata: one card, and the experts in the node's RAM, which
|
||||
// NeedGB and HaveGB are then about.
|
||||
StrataFamily string `json:"strata_family,omitempty"`
|
||||
StrataSize string `json:"strata_size,omitempty"`
|
||||
}
|
||||
|
||||
// GPUs is every card one replica takes.
|
||||
@@ -130,7 +137,7 @@ func (o Option) Fits() bool { return o.NeedGB <= o.HaveGB }
|
||||
// WithContext is the option for another context: kvPerToken a token's KV
|
||||
// cache, the whole model; the weights and the engine's share stay.
|
||||
func (o Option) WithContext(kvPerToken int64, tokens int) Option {
|
||||
if kvPerToken <= 0 || tokens <= 0 {
|
||||
if kvPerToken <= 0 || tokens <= 0 || o.Engine == "strata" {
|
||||
return o
|
||||
}
|
||||
ctx := GB(int64(tokens) * kvPerToken)
|
||||
@@ -144,6 +151,9 @@ func (o Option) WithContext(kvPerToken int64, tokens int) Option {
|
||||
|
||||
// MemoryText says what it needs, part by part.
|
||||
func (o Option) MemoryText() string {
|
||||
if o.Engine == "strata" {
|
||||
return fmt.Sprintf("%.0f GB of the node's RAM for its experts, of %.0f usable, and one card", o.NeedGB, o.HaveGB)
|
||||
}
|
||||
parts := fmt.Sprintf("weights %.1f + context %.1f + engine %.1f", o.WeightsGB, o.ContextGB, o.EngineGB)
|
||||
if o.WeightsGB == 0 && o.ContextGB == 0 {
|
||||
parts = ""
|
||||
@@ -174,6 +184,9 @@ func (o Option) Label() string {
|
||||
if o.Nodes > 1 {
|
||||
return fmt.Sprintf("%d nodes × %d %s on %s", o.Nodes, o.GPUsPerNode, o.Class, o.Fabric)
|
||||
}
|
||||
if o.Engine == "strata" {
|
||||
return fmt.Sprintf("1 × %s with Strata, experts in RAM", o.Class)
|
||||
}
|
||||
return fmt.Sprintf("%d × %s", o.GPUsPerNode, o.Class)
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package modelplan
|
||||
|
||||
import (
|
||||
"slices"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
|
||||
)
|
||||
|
||||
// Strata runs a few mixture-of-experts models with their experts in the
|
||||
// node's RAM and one card for the rest: a model that fits no card, or only
|
||||
// many, served from one. Offered for the files of a family it runs, on nodes
|
||||
// that have it installed, a card of at least strataMinCardGB and the RAM.
|
||||
|
||||
// strataMinCardGB is the smallest card Strata is worth running on: what its
|
||||
// guide recommends.
|
||||
const strataMinCardGB = 12
|
||||
|
||||
// StrataOptions are the ways a family's files could run on Strata here.
|
||||
func StrataOptions(family string, arts []Artifact, nodes []PoolNode) []Option {
|
||||
var out []Option
|
||||
for _, a := range arts {
|
||||
size, ok := strata.SizeForFile(family, a.Name)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
need := float64(size.MemoryGB())
|
||||
byClass := map[string]*Option{}
|
||||
var order []string
|
||||
for _, n := range nodes {
|
||||
if !slices.Contains(n.Engines, "strata") || n.HostMemGB <= 0 {
|
||||
continue
|
||||
}
|
||||
have := float64(n.HostMemGB) * hostShare
|
||||
if need > have {
|
||||
continue
|
||||
}
|
||||
for _, k := range n.kinds() {
|
||||
if k.Class == "" || isMIG(k.Class) || k.GPUMemGB < strataMinCardGB || k.GPUs < 1 {
|
||||
continue
|
||||
}
|
||||
o := byClass[k.Class]
|
||||
if o == nil {
|
||||
o = &Option{Artifact: a.Name, Format: a.Format, Engine: "strata", Class: k.Class,
|
||||
GPUsPerNode: 1, Nodes: 1, NeedGB: need, StrataFamily: family, StrataSize: size.Name}
|
||||
byClass[k.Class] = o
|
||||
order = append(order, k.Class)
|
||||
}
|
||||
o.Hosts++
|
||||
o.HaveGB = max(o.HaveGB, have)
|
||||
if k.FreeGPUs > 0 && float64(n.HostFreeGB) >= need {
|
||||
o.FreeNow = true
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, c := range order {
|
||||
out = append(out, *byClass[c])
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,32 @@
|
||||
package modelplan
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestStrataIsOfferedOnANodeWithItTheCardAndTheRAM(t *testing.T) {
|
||||
arts := []Artifact{{Name: "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf", Format: "gguf", Bytes: 68e9},
|
||||
{Name: "something-else-Q4_K_M.gguf", Format: "gguf", Bytes: 5e9}}
|
||||
nodes := []PoolNode{
|
||||
{ID: "cuda01", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 251, HostFreeGB: 200,
|
||||
Engines: []string{"llamacpp", "strata"}},
|
||||
{ID: "no-strata", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 251, Engines: []string{"llamacpp"}},
|
||||
{ID: "small-ram", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 32, Engines: []string{"strata"}},
|
||||
{ID: "small-card", Class: "t4-8g", GPUMemGB: 8, GPUs: 1, FreeGPUs: 1, HostMemGB: 251, Engines: []string{"strata"}},
|
||||
}
|
||||
opts := StrataOptions("qwen", arts, nodes)
|
||||
if len(opts) != 1 {
|
||||
t.Fatalf("%+v", opts)
|
||||
}
|
||||
o := opts[0]
|
||||
if o.Engine != "strata" || o.StrataSize != "IQ2_XS" || o.Hosts != 1 || !o.FreeNow || o.NeedGB != 52 || o.GPUs() != 1 {
|
||||
t.Errorf("%+v", o)
|
||||
}
|
||||
// It ranks before llama.cpp on the CPU, and says what it needs.
|
||||
all := append(Options(arts[:1], Shape{}, nodes, 16384), opts...)
|
||||
Rank(all)
|
||||
if all[0].Engine != "strata" {
|
||||
t.Errorf("first is %+v", all[0])
|
||||
}
|
||||
if o.Label() != "1 × 6000-24g with Strata, experts in RAM" || o.MemoryText() == "" {
|
||||
t.Errorf("%q %q", o.Label(), o.MemoryText())
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user