diff --git a/BACKLOG.md b/BACKLOG.md index 7fc21f5..f33d8b3 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -1876,12 +1876,14 @@ served only from a definition written by hand. In order: - N2. Its engine runs in a session of its own and survived a stop, holding the card and the RAM: the supervisor now finds what left the group before signalling it. Done 2026-10-30. - N3. Host memory: the experts live in RAM (40 GB for Qwen3.8-Flash-Next IQ2_XS, 72–136 GB for Unsloth's) and - nothing reserved it. A Strata profile added now must say `min_memory_gb`. Done 2026-10-30; derived from a table: B. + nothing reserved it. A Strata profile added now reserves what its size takes, from the table (N5), unless it says + `min_memory_gb`. Done 2026-10-30. - N4. Pascal and Volta: installed (CUDA 12.9 build) and then refused (7.0 floor). Floor now 6.0. Done 2026-10-30. **B. Offered where models are chosen.** - N5. A table of Strata's families and sizes (qwen, coder, swift, unsloth, gyro, nvfp4, qwen36, deepseek41) with - RAM+VRAM, disk and measured speed; the catalog refuses an unknown family/size (and requires `--size`). + download and RAM, from Strata 0.2.7's setup (internal/strata/models.go); the catalog refuses an unknown family or + size, and requires `--size`. Done 2026-10-30. Speeds measured: Qwen3.8-Flash-Next IQ2_XS 70–87 tok/s on an RTX 6000. - N6. `modelplan`, `ctl model plan` and `ctl model serve` offer "Strata on one card + N GB RAM" for a model it runs, with its speed; a large MoE that fits no card is not "fits nowhere". - N7. The web serve page, the explorer's fit, the console's serve form: the same choice. diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index b38fa62..058c1ce 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -22,6 +22,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/facility" "gitea.mynodes.xyz/ilker/llmgw/internal/control/placement" "gitea.mynodes.xyz/ilker/llmgw/internal/guard" + "gitea.mynodes.xyz/ilker/llmgw/internal/strata" ) // ErrExists is a model added only if absent that was there already. @@ -1201,10 +1202,14 @@ func (m Model) validate() error { if p.Nodes > 1 || p.TP > 1 || p.PP > 1 || p.GPUsPerNode > 1 { return fmt.Errorf("model %q profile %q: strata runs a replica on one GPU of one node", m.Name, p.Name) } - if !slices.ContainsFunc(p.Args, func(a string) bool { return a == "--family" || strings.HasPrefix(a, "--family=") }) { + family, size := strata.ArgValue(p.Args, "--family"), strata.ArgValue(p.Args, "--size") + if family == "" || size == "" { return fmt.Errorf("model %q profile %q: strata needs the model's family and size in args "+ "(--family qwen --size IQ2_XS)", m.Name, p.Name) } + if _, err := strata.SizeOf(family, size); err != nil { + return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err) + } if strings.HasSuffix(strings.ToLower(strings.TrimSuffix(m.SourceOf(p), "/")), ".gguf") { return fmt.Errorf("model %q profile %q: strata prepares a model from all of its GGUF files; "+ "give the directory they are in as the source", m.Name, p.Name) diff --git a/internal/control/catalog/strata_test.go b/internal/control/catalog/strata_test.go index 199bf8c..193237c 100644 --- a/internal/control/catalog/strata_test.go +++ b/internal/control/catalog/strata_test.go @@ -20,11 +20,17 @@ func TestAStrataProfileNamesItsFamilyAndRunsOnOneCard(t *testing.T) { "family and size": {Name: "x", Source: "http://mirror:8081/x/1/", Profiles: []Profile{{ Name: "p", Engine: "strata", GPUClass: "6000-24g", Args: []string{"--size", "IQ2_XS"}}}}, "all of its GGUF files": {Name: "x", Source: "http://mirror:8081/x/1/x-00001-of-00002.gguf", Profiles: []Profile{{ + Name: "p", Engine: "strata", GPUClass: "6000-24g", Args: []string{"--family", "qwen", "--size", "IQ2_XS"}}}}, + "no family \"llama\"": {Name: "x", Source: "http://mirror:8081/x/1/", Profiles: []Profile{{ + Name: "p", Engine: "strata", GPUClass: "6000-24g", Args: []string{"--family", "llama", "--size", "IQ2_XS"}}}}, + "comes in IQ1_M": {Name: "x", Source: "http://mirror:8081/x/1/", Profiles: []Profile{{ + Name: "p", Engine: "strata", GPUClass: "6000-24g", Args: []string{"--family", "coder", "--size", "IQ2_XS"}}}}, + "family and size ": {Name: "x", Source: "http://mirror:8081/x/1/", Profiles: []Profile{{ Name: "p", Engine: "strata", GPUClass: "6000-24g", Args: []string{"--family", "qwen"}}}}, "--gguf-dir": {Name: "x", Source: "http://mirror:8081/x/1/", Profiles: []Profile{{ Name: "p", Engine: "strata", GPUClass: "6000-24g", Args: []string{"--family", "qwen", "--gguf-dir", "/etc"}}}}, } { - if _, err := New([]Model{bad}); err == nil || !strings.Contains(err.Error(), want) { + if _, err := New([]Model{bad}); err == nil || !strings.Contains(err.Error(), strings.TrimSpace(want)) { t.Errorf("expected %q, got %v", want, err) } } diff --git a/internal/control/modelsvc/modelsvc.go b/internal/control/modelsvc/modelsvc.go index d2356b0..21036da 100644 --- a/internal/control/modelsvc/modelsvc.go +++ b/internal/control/modelsvc/modelsvc.go @@ -27,6 +27,7 @@ import ( "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/strata" ) // Store is the persistence this needs. @@ -129,15 +130,21 @@ func (s *Service) add(ctx context.Context, m catalog.Model, by string, onlyNew b // fits everywhere as far as placement can tell — however many of it // there already are. Required of what is added now; models stored before // this still load, and say what they are missing. - for _, p := range m.Profiles { + m.Profiles = slices.Clone(m.Profiles) + for i := range m.Profiles { // Strata keeps a model's experts in the host's memory, tens of // gigabytes of it: placed as if it took none, two replicas, or one - // beside a CPU model, filled a node until one was killed. - if p.Engine == "strata" && p.MinMemoryGB <= 0 { - return fmt.Errorf("model %q profile %q runs on Strata, which keeps the experts in host memory, and declares "+ - "none (min_memory_gb): give what Strata's model table says the size takes, e.g. 40 for Qwen3.8-Flash-Next IQ2_XS", - m.Name, p.Name) + // beside a CPU model, filled a node until one was killed. What its + // size takes, from Strata's table, unless the model says. + if p := &m.Profiles[i]; p.Engine == "strata" && p.MinMemoryGB <= 0 { + size, err := strata.SizeOf(strata.ArgValue(p.Args, "--family"), strata.ArgValue(p.Args, "--size")) + if err != nil { + return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err) + } + p.MinMemoryGB = size.MemoryGB() } + } + for _, p := range m.Profiles { if p.CPUOnly() && p.MinMemoryGB <= 0 { return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+ "placement could not tell when a node is full", m.Name, p.Name) diff --git a/internal/control/modelsvc/modelsvc_test.go b/internal/control/modelsvc/modelsvc_test.go index 8d8ddae..f06fa43 100644 --- a/internal/control/modelsvc/modelsvc_test.go +++ b/internal/control/modelsvc/modelsvc_test.go @@ -439,17 +439,22 @@ func TestAStoppedModelRunsNoneAndStartsAsItWas(t *testing.T) { } } -func TestAStrataModelMustSayTheMemoryItsExpertsTake(t *testing.T) { - s, _, _ := newService(t) +func TestAStrataModelReservesTheMemoryItsExpertsTake(t *testing.T) { + s, _, rec := newService(t) m := model("big") m.Source = "http://mirror:8081/big/v1/" m.Profiles = []catalog.Profile{{Name: "gpu", Engine: "strata", GPUClass: "6000-24g", GPUsPerNode: 1, Args: []string{"--family", "qwen", "--size", "IQ2_XS"}}} - if err := s.Add(t.Context(), m, "ilker"); err == nil || !strings.Contains(err.Error(), "min_memory_gb") { - t.Fatalf("added without its memory: %v", err) - } - m.Profiles[0].MinMemoryGB = 40 if err := s.Add(t.Context(), m, "ilker"); err != nil { t.Fatal(err) } + got, _ := rec.last.Resolve("big") + if got.Profiles[0].MinMemoryGB != 52 { + t.Errorf("IQ2_XS reserves %d GB, want Strata's 48 and 4 for the engine", got.Profiles[0].MinMemoryGB) + } + m.Name = "big2" + m.Profiles[0].Args = []string{"--family", "qwen", "--size", "IQ9"} + if err := s.Add(t.Context(), m, "ilker"); err == nil || !strings.Contains(err.Error(), "comes in") { + t.Errorf("an unknown size: %v", err) + } } diff --git a/internal/strata/models.go b/internal/strata/models.go new file mode 100644 index 0000000..b46ecf6 --- /dev/null +++ b/internal/strata/models.go @@ -0,0 +1,161 @@ +package strata + +import ( + "fmt" + "slices" + "strings" +) + +// What Strata runs, as its setup (0.2.7, the version llmgw installs) lists +// it: a few model families, each in a few sizes, with what a size takes. +// Written down here so a model is checked when it is added rather than when +// a node's setup refuses it minutes later, and so the plan can offer Strata +// for a model it runs, with the memory to reserve. + +// Family is a model Strata runs, from one publisher's files. +type Family struct { + // Name is setup's --family. + Name string + Title string + // Repo is the Hugging Face repository its files are in; empty when + // Strata fetches them itself from several. + Repo string + // Subdir is whether each size's files are in a directory of its own, + // named as the size, in the repository. + Subdir bool + // Engine is a family that runs on an engine path of its own, with a + // download of its own (DeepSeek V4.1). + Engine bool +} + +// Size is one of a family's sizes: setup's --model, which llmgw's profile +// says as --size. +type Size struct { + Name string + Families []string + // DownloadGB is what is fetched; DiskGB what it takes on the node once + // prepared, when more than the download. + DownloadGB, DiskGB float64 + // RAMGB is the host memory that keeps every expert in RAM: what a + // replica reserves. Strata can read the rest from the disk with less, + // much slower. + RAMGB float64 + // NVIDIAOnly is a size whose kernels only NVIDIA cards have. + NVIDIAOnly bool +} + +// Families are the families Strata 0.2.7 runs. +var Families = []Family{ + {Name: "qwen", Title: "Qwen3.8-Flash-Next", Repo: "ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF", Subdir: true}, + {Name: "swift", Title: "Swift 1.5 (Qwen3.8-Flash-Next, shorter thinking)", Repo: "ukisai/Swift-1.5-Qwen3.8-Flash-Next-GSQ-RCO-GGUF"}, + {Name: "coder", Title: "Qwen3.8-Flash-Next Coder", Repo: "ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-Coder-GGUF", Subdir: true}, + {Name: "unsloth", Title: "Qwen3.8-Flash-Next (Unsloth)", Repo: "unsloth/Qwen3.8-Flash-Next-GGUF", Subdir: true}, + {Name: "gyro", Title: "Qwen3.8-Flash-Next (Gyro)", Repo: "agentionai/Qwen3.8-Flash-Next-Gyro-GGUF"}, + {Name: "nvfp4", Title: "Qwen3.8-Flash-Next (NVFP4)"}, + {Name: "qwen36", Title: "Qwen3.6-35B-A3B", Repo: "unsloth/Qwen3.6-35B-A3B-MTP-GGUF"}, + {Name: "deepseek41", Title: "DeepSeek V4.1-Flash", Repo: "ISTA-DASLab/DeepSeek-V4.1-Flash-1017-GSQ-RCO-GGUF", Engine: true}, +} + +// Sizes are the sizes Strata 0.2.7 runs, and which families have each. +var Sizes = []Size{ + {Name: "Q2_0", Families: []string{"qwen"}, DownloadGB: 66.4, RAMGB: 48}, + {Name: "IQ2_XS", Families: []string{"qwen", "swift"}, DownloadGB: 68.0, RAMGB: 48}, + {Name: "IQ3_XXS", Families: []string{"qwen", "swift"}, DownloadGB: 75.8, RAMGB: 60}, + {Name: "IQ3_S", Families: []string{"qwen", "swift"}, DownloadGB: 83.6, RAMGB: 62}, + {Name: "IQ1_M", Families: []string{"coder"}, DownloadGB: 58.4, RAMGB: 32}, + {Name: "UD-IQ4_XS", Families: []string{"unsloth"}, DownloadGB: 93.7, RAMGB: 80}, + {Name: "UD-Q4_K_XL", Families: []string{"unsloth", "qwen36"}, DownloadGB: 111.3, RAMGB: 94}, + {Name: "UD-Q5_K_XL", Families: []string{"unsloth"}, DownloadGB: 158.3, RAMGB: 108}, + {Name: "UD-Q6_K_XL", Families: []string{"unsloth", "qwen36"}, DownloadGB: 169.2, RAMGB: 118}, + {Name: "Q8_0", Families: []string{"unsloth", "qwen36"}, DownloadGB: 188.2, RAMGB: 136}, + {Name: "Gyro-S", Families: []string{"gyro"}, DownloadGB: 58.5, RAMGB: 27.6}, + {Name: "Gyro-M", Families: []string{"gyro"}, DownloadGB: 92.0, RAMGB: 35}, + {Name: "NVFP4", Families: []string{"nvfp4"}, DownloadGB: 128, RAMGB: 80, NVIDIAOnly: true}, + {Name: "NVFP4-NVIDIA", Families: []string{"nvfp4"}, DownloadGB: 128, RAMGB: 80, NVIDIAOnly: true}, + {Name: "NVFP4-QAD", Families: []string{"nvfp4"}, DownloadGB: 102.2, DiskGB: 212.2, RAMGB: 80, NVIDIAOnly: true}, + {Name: "IST-DAS-1.75bpw", Families: []string{"deepseek41"}, DownloadGB: 239, RAMGB: 34}, + {Name: "IST-DAS-2.00bpw", Families: []string{"deepseek41"}, DownloadGB: 257, RAMGB: 40}, + {Name: "IST-DAS-2.25bpw", Families: []string{"deepseek41"}, DownloadGB: 274, RAMGB: 44}, + {Name: "UD-Q8_K_XL", Families: []string{"deepseek41"}, DownloadGB: 520, RAMGB: 306}, +} + +// qwen36 is smaller than the Flash-Next sizes of the same names: its own +// figures. +var qwen36RAM = map[string]float64{"UD-Q4_K_XL": 32, "UD-Q6_K_XL": 44, "Q8_0": 52} +var qwen36Download = map[string]float64{"UD-Q4_K_XL": 22.9, "UD-Q6_K_XL": 32.6, "Q8_0": 37.8} + +// FamilyNamed is a family by its --family. +func FamilyNamed(name string) (Family, bool) { + i := slices.IndexFunc(Families, func(f Family) bool { return f.Name == name }) + if i < 0 { + return Family{}, false + } + return Families[i], true +} + +// SizeOf is a family's size by its --size, with that family's figures. +func SizeOf(family, size string) (Size, error) { + if _, ok := FamilyNamed(family); !ok { + return Size{}, fmt.Errorf("strata runs no family %q; it runs %s", family, strings.Join(familyNames(), ", ")) + } + var names []string + for _, s := range Sizes { + if !slices.Contains(s.Families, family) { + continue + } + names = append(names, s.Name) + if s.Name == size { + if family == "qwen36" { + s.RAMGB, s.DownloadGB = qwen36RAM[size], qwen36Download[size] + } + return s, nil + } + } + return Size{}, fmt.Errorf("strata's %s comes in %s, not %q", family, strings.Join(names, ", "), size) +} + +// ForRepo is the family whose files a Hugging Face repository holds. +func ForRepo(repo string) (Family, bool) { + i := slices.IndexFunc(Families, func(f Family) bool { return f.Repo != "" && strings.EqualFold(f.Repo, repo) }) + if i < 0 { + return Family{}, false + } + return Families[i], true +} + +// SizesOf are a family's sizes, with its figures. +func SizesOf(family string) []Size { + var out []Size + for _, s := range Sizes { + if slices.Contains(s.Families, family) { + got, _ := SizeOf(family, s.Name) + out = append(out, got) + } + } + return out +} + +// MemoryGB is what a replica of a size reserves of the host: its experts, +// and the engine and server beside them. +func (s Size) MemoryGB() int { return int(s.RAMGB+0.999) + 4 } + +func familyNames() []string { + out := make([]string, len(Families)) + for i, f := range Families { + out[i] = f.Name + } + return out +} + +// ArgValue is a flag's value in a profile's arguments, "" when absent. +func ArgValue(args []string, flag string) string { + for i, a := range args { + if a == flag && i+1 < len(args) { + return args[i+1] + } + if v, ok := strings.CutPrefix(a, flag+"="); ok { + return v + } + } + return "" +} diff --git a/internal/strata/models_test.go b/internal/strata/models_test.go new file mode 100644 index 0000000..2dbff65 --- /dev/null +++ b/internal/strata/models_test.go @@ -0,0 +1,29 @@ +package strata + +import ( + "strings" + "testing" +) + +func TestASizeIsKnownForTheFamiliesThatHaveIt(t *testing.T) { + if s, err := SizeOf("qwen", "IQ2_XS"); err != nil || s.RAMGB != 48 || s.MemoryGB() != 52 { + t.Errorf("%+v %v", s, err) + } + if s, err := SizeOf("qwen36", "Q8_0"); err != nil || s.RAMGB != 52 { + t.Errorf("qwen36's own figures: %+v %v", s, err) + } + if _, err := SizeOf("coder", "IQ2_XS"); err == nil || !strings.Contains(err.Error(), "IQ1_M") { + t.Errorf("coder in IQ2_XS: %v", err) + } + if _, err := SizeOf("llama", "Q8_0"); err == nil || !strings.Contains(err.Error(), "deepseek41") { + t.Errorf("an unknown family: %v", err) + } + if f, ok := ForRepo("istA-daslab/qwen3.8-flash-next-gsq-rco-gguf"); !ok || f.Name != "qwen" { + t.Errorf("by repository: %+v", f) + } + for _, f := range Families { + if len(SizesOf(f.Name)) == 0 { + t.Errorf("%s has no sizes", f.Name) + } + } +}