CHANGELOG: one list
CI / test (push) Successful in 9m18s

This commit is contained in:
Ilker Manap committed 2026-10-30 10:39:30 +01:00
1 parent 7465c24830
commit d6f8f433f4
24 files changed
+437 -22

No files matched your search

+1 -1
View File
@@ -1939,7 +1939,7 @@ func PoolNodesOf(nodes []NodeView) []modelplan.PoolNode {
continue
}
p := modelplan.PoolNode{ID: n.ID, GPUs: n.GPUTotal, FreeGPUs: n.GPUFree, HostMemGB: n.MemoryGB,
HostFreeGB: n.MemoryGB}
HostFreeGB: n.MemoryGB, Engines: n.Engines}
if len(n.Classes) > 0 {
p.Class = n.Classes[0]
p.GPUMemGB = modelplan.MemFromClass(p.Class)
+43 -2
View File
@@ -134,6 +134,9 @@ type Request struct {
// Fallbacks adds the next best ways, lower in priority, so a model whose
// first choice is busy runs on another rather than waiting.
Fallbacks bool `json:"fallbacks,omitempty"`
// Engine is the engine the file runs on: "strata" for a family Strata
// runs, "llamacpp" to keep it off Strata; empty, the plan's best.
Engine string `json:"engine,omitempty"`
// Name is the catalog name; the repository suggests one otherwise.
Name string `json:"name,omitempty"`
Aliases []string `json:"aliases,omitempty"`
@@ -488,6 +491,23 @@ func (s *Server) run(j *Job) {
}
// 3. Whether the nodes' engine loads it, and how much memory it needs.
if layout[0].Engine == "strata" {
// Strata prepares it on the node from the directory of its files,
// and keeps the experts in the RAM the plan counted.
if !strings.HasSuffix(source, "/") {
source = strings.TrimSuffix(source, mirror.LastPathElement(source))
}
if !req.sized {
req.ContextLength = strataContext
}
s.step(j, "check", Done, fmt.Sprintf("Strata's %s %s, prepared on the node; %.0f GB of RAM for its experts",
layout[0].StrataFamily, layout[0].StrataSize, layout[0].NeedGB))
s.mu.Lock()
j.Request = req
s.mu.Unlock()
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0), hardwareExists(layout))
return
}
if req.Format == "safetensors" {
// vLLM reads the model's shape from its config.json, which came
// with the weights.
@@ -566,10 +586,21 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
}
var opts []modelplan.Option
for _, o := range plan.Options {
if o.Artifact == artifact {
onStrata := o.Engine == "strata"
if o.Artifact == artifact && (req.Engine == "" || (req.Engine == "strata") == onStrata) {
opts = append(opts, o)
}
}
if req.Engine == "strata" {
// Strata's way only, on the class asked for if one was.
i := slices.IndexFunc(opts, func(o modelplan.Option) bool { return req.GPUClass == "" || o.Class == req.GPUClass })
if i < 0 {
return nil, fmt.Errorf("Strata cannot run %s here: it runs a few families only (`llmgw ctl model plan %s` "+
"says whether this is one), on a node with Strata installed, a card of 12 GB or more and the RAM for its experts",
set.Name, plan.Repo)
}
return []modelplan.Option{opts[i]}, nil
}
var out []modelplan.Option
switch {
case req.CPU || req.GPUClass != "":
@@ -600,7 +631,7 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
default:
out = append(out, modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: true})
}
if req.Fallbacks {
if req.Fallbacks && out[0].Engine != "strata" {
shape := func(o modelplan.Option) string {
return fmt.Sprintf("%v/%s/%d/%d", o.CPU, o.Class, o.GPUsPerNode, o.Nodes)
}
@@ -625,6 +656,12 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
for i, o := range layout {
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i, Env: req.Env}
switch {
case o.Engine == "strata":
p.Name = "strata-" + o.Class
p.GPUClass, p.GPUsPerNode = o.Class, 1
p.Args = []string{"--family", o.StrataFamily, "--size", o.StrataSize,
"--context", strconv.Itoa(req.ContextLength), "--vision", "no"}
p.MinMemoryGB = int(o.NeedGB)
case o.CPU:
p.Name = "cpu"
p.Args = llamaArgs(req)
@@ -674,6 +711,10 @@ func llamaArgs(req Request) []string {
return args
}
// strataContext is a Strata model's context when its caller did not say:
// Strata's own default.
const strataContext = 32768
// An embedding model's slots, when its caller did not size them: passages
// of up to a thousand tokens, thirty-two at a time.
const (
+13
View File
@@ -9,6 +9,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
)
// Planning: which of a repository's files suits the cluster's hardware, and
@@ -168,6 +169,18 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
if !p.Servable {
p.Options = nil
}
// A family Strata runs: its files on one card with the experts in RAM.
// Strata reads them itself, whether or not llama.cpp could.
if fam, ok := strata.ForRepo(repo.ID); ok && p.Format == "gguf" {
if so := modelplan.StrataOptions(fam.Name, arts, s.deps.Pool()); len(so) > 0 {
p.Options = append(p.Options, so...)
modelplan.Rank(p.Options)
if !p.Servable {
p.Note = strings.TrimSpace(p.Note + " llama.cpp cannot run it (" + p.Reason + "); Strata can.")
p.Servable, p.Reason = true, ""
}
}
}
for _, a := range arts {
pa := PlannedArtifact{Artifact: a}
for i := range p.Options {
@@ -0,0 +1,37 @@
package modelserve
import (
"slices"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
)
func TestAStrataWayIsChosenAndMadeAProfile(t *testing.T) {
set := hf.FileSet{Name: "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf"}
strataWay := modelplan.Option{Artifact: set.Name, Format: "gguf", Engine: "strata", Class: "6000-24g",
GPUsPerNode: 1, Nodes: 1, NeedGB: 52, HaveGB: 213, Hosts: 1, StrataFamily: "qwen", StrataSize: "IQ2_XS"}
cpuWay := modelplan.Option{Artifact: set.Name, Format: "gguf", Engine: "llamacpp", CPU: true, NeedGB: 75, HaveGB: 213, Hosts: 1}
plan := Plan{Repo: "ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF", Options: []modelplan.Option{strataWay, cpuWay}}
got, err := chooseLayout(Request{Format: "gguf", Fallbacks: true}, plan, true, set)
if err != nil || len(got) != 1 || got[0].Engine != "strata" {
t.Fatalf("the plan's best, without llama.cpp fallbacks: %+v %v", got, err)
}
if got, _ := chooseLayout(Request{Format: "gguf", Engine: "llamacpp"}, plan, true, set); len(got) != 1 || !got[0].CPU {
t.Errorf("kept off Strata: %+v", got)
}
if _, err := chooseLayout(Request{Format: "gguf", Engine: "strata", GPUClass: "h100-80g"}, plan, true, set); err == nil {
t.Error("Strata on a class it was not planned for")
}
ps := profilesFor(got, Request{ContextLength: 32768}, 0, 0)
p := ps[0]
if p.Engine != "strata" || p.GPUClass != "6000-24g" || p.GPUsPerNode != 1 || p.MinMemoryGB != 52 ||
!slices.Equal(p.Args, []string{"--family", "qwen", "--size", "IQ2_XS", "--context", "32768", "--vision", "no"}) ||
!strings.HasPrefix(p.Name, "strata-") {
t.Errorf("%+v", p)
}
}