CHANGELOG: one list
CI / test (push) Successful in 9m18s

This commit is contained in:
Ilker Manap committed 2026-10-30 10:39:30 +01:00
1 parent 7465c24830
commit d6f8f433f4
24 files changed
+437 -22

No files matched your search

+14 -1
View File
@@ -53,6 +53,8 @@ type PoolNode struct {
// card beside it another. Empty means Class, GPUMemGB, GPUs and FreeGPUs
// say it all.
Kinds []PoolKind
// Engines are the engines the node has installed.
Engines []string
}
// PoolKind is one class of a node's devices.
@@ -110,6 +112,11 @@ type Option struct {
// cards, whole, rather than slicing each across them: vLLM's
// --enable-expert-parallel, for a model of experts on more than one card.
ExpertParallel bool `json:"expert_parallel,omitempty"`
// StrataFamily and StrataSize are Strata's --family and --size, for an
// option on Strata: one card, and the experts in the node's RAM, which
// NeedGB and HaveGB are then about.
StrataFamily string `json:"strata_family,omitempty"`
StrataSize string `json:"strata_size,omitempty"`
}
// GPUs is every card one replica takes.
@@ -130,7 +137,7 @@ func (o Option) Fits() bool { return o.NeedGB <= o.HaveGB }
// WithContext is the option for another context: kvPerToken a token's KV
// cache, the whole model; the weights and the engine's share stay.
func (o Option) WithContext(kvPerToken int64, tokens int) Option {
if kvPerToken <= 0 || tokens <= 0 {
if kvPerToken <= 0 || tokens <= 0 || o.Engine == "strata" {
return o
}
ctx := GB(int64(tokens) * kvPerToken)
@@ -144,6 +151,9 @@ func (o Option) WithContext(kvPerToken int64, tokens int) Option {
// MemoryText says what it needs, part by part.
func (o Option) MemoryText() string {
if o.Engine == "strata" {
return fmt.Sprintf("%.0f GB of the node's RAM for its experts, of %.0f usable, and one card", o.NeedGB, o.HaveGB)
}
parts := fmt.Sprintf("weights %.1f + context %.1f + engine %.1f", o.WeightsGB, o.ContextGB, o.EngineGB)
if o.WeightsGB == 0 && o.ContextGB == 0 {
parts = ""
@@ -174,6 +184,9 @@ func (o Option) Label() string {
if o.Nodes > 1 {
return fmt.Sprintf("%d nodes × %d %s on %s", o.Nodes, o.GPUsPerNode, o.Class, o.Fabric)
}
if o.Engine == "strata" {
return fmt.Sprintf("1 × %s with Strata, experts in RAM", o.Class)
}
return fmt.Sprintf("%d × %s", o.GPUsPerNode, o.Class)
}
+60
View File
@@ -0,0 +1,60 @@
package modelplan
import (
"slices"
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
)
// Strata runs a few mixture-of-experts models with their experts in the
// node's RAM and one card for the rest: a model that fits no card, or only
// many, served from one. Offered for the files of a family it runs, on nodes
// that have it installed, a card of at least strataMinCardGB and the RAM.
// strataMinCardGB is the smallest card Strata is worth running on: what its
// guide recommends.
const strataMinCardGB = 12
// StrataOptions are the ways a family's files could run on Strata here.
func StrataOptions(family string, arts []Artifact, nodes []PoolNode) []Option {
var out []Option
for _, a := range arts {
size, ok := strata.SizeForFile(family, a.Name)
if !ok {
continue
}
need := float64(size.MemoryGB())
byClass := map[string]*Option{}
var order []string
for _, n := range nodes {
if !slices.Contains(n.Engines, "strata") || n.HostMemGB <= 0 {
continue
}
have := float64(n.HostMemGB) * hostShare
if need > have {
continue
}
for _, k := range n.kinds() {
if k.Class == "" || isMIG(k.Class) || k.GPUMemGB < strataMinCardGB || k.GPUs < 1 {
continue
}
o := byClass[k.Class]
if o == nil {
o = &Option{Artifact: a.Name, Format: a.Format, Engine: "strata", Class: k.Class,
GPUsPerNode: 1, Nodes: 1, NeedGB: need, StrataFamily: family, StrataSize: size.Name}
byClass[k.Class] = o
order = append(order, k.Class)
}
o.Hosts++
o.HaveGB = max(o.HaveGB, have)
if k.FreeGPUs > 0 && float64(n.HostFreeGB) >= need {
o.FreeNow = true
}
}
}
for _, c := range order {
out = append(out, *byClass[c])
}
}
return out
}
+32
View File
@@ -0,0 +1,32 @@
package modelplan
import "testing"
func TestStrataIsOfferedOnANodeWithItTheCardAndTheRAM(t *testing.T) {
arts := []Artifact{{Name: "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf", Format: "gguf", Bytes: 68e9},
{Name: "something-else-Q4_K_M.gguf", Format: "gguf", Bytes: 5e9}}
nodes := []PoolNode{
{ID: "cuda01", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 251, HostFreeGB: 200,
Engines: []string{"llamacpp", "strata"}},
{ID: "no-strata", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 251, Engines: []string{"llamacpp"}},
{ID: "small-ram", Class: "6000-24g", GPUMemGB: 24, GPUs: 1, FreeGPUs: 1, HostMemGB: 32, Engines: []string{"strata"}},
{ID: "small-card", Class: "t4-8g", GPUMemGB: 8, GPUs: 1, FreeGPUs: 1, HostMemGB: 251, Engines: []string{"strata"}},
}
opts := StrataOptions("qwen", arts, nodes)
if len(opts) != 1 {
t.Fatalf("%+v", opts)
}
o := opts[0]
if o.Engine != "strata" || o.StrataSize != "IQ2_XS" || o.Hosts != 1 || !o.FreeNow || o.NeedGB != 52 || o.GPUs() != 1 {
t.Errorf("%+v", o)
}
// It ranks before llama.cpp on the CPU, and says what it needs.
all := append(Options(arts[:1], Shape{}, nodes, 16384), opts...)
Rank(all)
if all[0].Engine != "strata" {
t.Errorf("first is %+v", all[0])
}
if o.Label() != "1 × 6000-24g with Strata, experts in RAM" || o.MemoryText() == "" {
t.Errorf("%q %q", o.Label(), o.MemoryText())
}
}