1 parent
7465c24830
commit
d6f8f433f4
24 files changed
+437
-22
No files matched your search
@@ -1939,7 +1939,7 @@ func PoolNodesOf(nodes []NodeView) []modelplan.PoolNode {
|
||||
continue
|
||||
}
|
||||
p := modelplan.PoolNode{ID: n.ID, GPUs: n.GPUTotal, FreeGPUs: n.GPUFree, HostMemGB: n.MemoryGB,
|
||||
HostFreeGB: n.MemoryGB}
|
||||
HostFreeGB: n.MemoryGB, Engines: n.Engines}
|
||||
if len(n.Classes) > 0 {
|
||||
p.Class = n.Classes[0]
|
||||
p.GPUMemGB = modelplan.MemFromClass(p.Class)
|
||||
|
||||
@@ -134,6 +134,9 @@ type Request struct {
|
||||
// Fallbacks adds the next best ways, lower in priority, so a model whose
|
||||
// first choice is busy runs on another rather than waiting.
|
||||
Fallbacks bool `json:"fallbacks,omitempty"`
|
||||
// Engine is the engine the file runs on: "strata" for a family Strata
|
||||
// runs, "llamacpp" to keep it off Strata; empty, the plan's best.
|
||||
Engine string `json:"engine,omitempty"`
|
||||
// Name is the catalog name; the repository suggests one otherwise.
|
||||
Name string `json:"name,omitempty"`
|
||||
Aliases []string `json:"aliases,omitempty"`
|
||||
@@ -488,6 +491,23 @@ func (s *Server) run(j *Job) {
|
||||
}
|
||||
|
||||
// 3. Whether the nodes' engine loads it, and how much memory it needs.
|
||||
if layout[0].Engine == "strata" {
|
||||
// Strata prepares it on the node from the directory of its files,
|
||||
// and keeps the experts in the RAM the plan counted.
|
||||
if !strings.HasSuffix(source, "/") {
|
||||
source = strings.TrimSuffix(source, mirror.LastPathElement(source))
|
||||
}
|
||||
if !req.sized {
|
||||
req.ContextLength = strataContext
|
||||
}
|
||||
s.step(j, "check", Done, fmt.Sprintf("Strata's %s %s, prepared on the node; %.0f GB of RAM for its experts",
|
||||
layout[0].StrataFamily, layout[0].StrataSize, layout[0].NeedGB))
|
||||
s.mu.Lock()
|
||||
j.Request = req
|
||||
s.mu.Unlock()
|
||||
s.addAndStart(ctx, j, repo, set, name, source, profilesFor(layout, req, 0, 0), hardwareExists(layout))
|
||||
return
|
||||
}
|
||||
if req.Format == "safetensors" {
|
||||
// vLLM reads the model's shape from its config.json, which came
|
||||
// with the weights.
|
||||
@@ -566,10 +586,21 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
|
||||
}
|
||||
var opts []modelplan.Option
|
||||
for _, o := range plan.Options {
|
||||
if o.Artifact == artifact {
|
||||
onStrata := o.Engine == "strata"
|
||||
if o.Artifact == artifact && (req.Engine == "" || (req.Engine == "strata") == onStrata) {
|
||||
opts = append(opts, o)
|
||||
}
|
||||
}
|
||||
if req.Engine == "strata" {
|
||||
// Strata's way only, on the class asked for if one was.
|
||||
i := slices.IndexFunc(opts, func(o modelplan.Option) bool { return req.GPUClass == "" || o.Class == req.GPUClass })
|
||||
if i < 0 {
|
||||
return nil, fmt.Errorf("Strata cannot run %s here: it runs a few families only (`llmgw ctl model plan %s` "+
|
||||
"says whether this is one), on a node with Strata installed, a card of 12 GB or more and the RAM for its experts",
|
||||
set.Name, plan.Repo)
|
||||
}
|
||||
return []modelplan.Option{opts[i]}, nil
|
||||
}
|
||||
var out []modelplan.Option
|
||||
switch {
|
||||
case req.CPU || req.GPUClass != "":
|
||||
@@ -600,7 +631,7 @@ func chooseLayout(req Request, plan Plan, planned bool, set hf.FileSet) ([]model
|
||||
default:
|
||||
out = append(out, modelplan.Option{Artifact: artifact, Format: req.Format, Engine: engine, CPU: true})
|
||||
}
|
||||
if req.Fallbacks {
|
||||
if req.Fallbacks && out[0].Engine != "strata" {
|
||||
shape := func(o modelplan.Option) string {
|
||||
return fmt.Sprintf("%v/%s/%d/%d", o.CPU, o.Class, o.GPUsPerNode, o.Nodes)
|
||||
}
|
||||
@@ -625,6 +656,12 @@ func profilesFor(layout []modelplan.Option, req Request, memGB, cores int) []cat
|
||||
for i, o := range layout {
|
||||
p := catalog.Profile{Engine: o.Engine, Weight: 1, Priority: 100 - 10*i, Env: req.Env}
|
||||
switch {
|
||||
case o.Engine == "strata":
|
||||
p.Name = "strata-" + o.Class
|
||||
p.GPUClass, p.GPUsPerNode = o.Class, 1
|
||||
p.Args = []string{"--family", o.StrataFamily, "--size", o.StrataSize,
|
||||
"--context", strconv.Itoa(req.ContextLength), "--vision", "no"}
|
||||
p.MinMemoryGB = int(o.NeedGB)
|
||||
case o.CPU:
|
||||
p.Name = "cpu"
|
||||
p.Args = llamaArgs(req)
|
||||
@@ -674,6 +711,10 @@ func llamaArgs(req Request) []string {
|
||||
return args
|
||||
}
|
||||
|
||||
// strataContext is a Strata model's context when its caller did not say:
|
||||
// Strata's own default.
|
||||
const strataContext = 32768
|
||||
|
||||
// An embedding model's slots, when its caller did not size them: passages
|
||||
// of up to a thousand tokens, thirty-two at a time.
|
||||
const (
|
||||
|
||||
@@ -9,6 +9,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/strata"
|
||||
)
|
||||
|
||||
// Planning: which of a repository's files suits the cluster's hardware, and
|
||||
@@ -168,6 +169,18 @@ func (s *Server) plan(ctx context.Context, repo hf.Repo, req PlanRequest) (Plan,
|
||||
if !p.Servable {
|
||||
p.Options = nil
|
||||
}
|
||||
// A family Strata runs: its files on one card with the experts in RAM.
|
||||
// Strata reads them itself, whether or not llama.cpp could.
|
||||
if fam, ok := strata.ForRepo(repo.ID); ok && p.Format == "gguf" {
|
||||
if so := modelplan.StrataOptions(fam.Name, arts, s.deps.Pool()); len(so) > 0 {
|
||||
p.Options = append(p.Options, so...)
|
||||
modelplan.Rank(p.Options)
|
||||
if !p.Servable {
|
||||
p.Note = strings.TrimSpace(p.Note + " llama.cpp cannot run it (" + p.Reason + "); Strata can.")
|
||||
p.Servable, p.Reason = true, ""
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, a := range arts {
|
||||
pa := PlannedArtifact{Artifact: a}
|
||||
for i := range p.Options {
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||||
)
|
||||
|
||||
func TestAStrataWayIsChosenAndMadeAProfile(t *testing.T) {
|
||||
set := hf.FileSet{Name: "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf"}
|
||||
strataWay := modelplan.Option{Artifact: set.Name, Format: "gguf", Engine: "strata", Class: "6000-24g",
|
||||
GPUsPerNode: 1, Nodes: 1, NeedGB: 52, HaveGB: 213, Hosts: 1, StrataFamily: "qwen", StrataSize: "IQ2_XS"}
|
||||
cpuWay := modelplan.Option{Artifact: set.Name, Format: "gguf", Engine: "llamacpp", CPU: true, NeedGB: 75, HaveGB: 213, Hosts: 1}
|
||||
plan := Plan{Repo: "ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF", Options: []modelplan.Option{strataWay, cpuWay}}
|
||||
|
||||
got, err := chooseLayout(Request{Format: "gguf", Fallbacks: true}, plan, true, set)
|
||||
if err != nil || len(got) != 1 || got[0].Engine != "strata" {
|
||||
t.Fatalf("the plan's best, without llama.cpp fallbacks: %+v %v", got, err)
|
||||
}
|
||||
if got, _ := chooseLayout(Request{Format: "gguf", Engine: "llamacpp"}, plan, true, set); len(got) != 1 || !got[0].CPU {
|
||||
t.Errorf("kept off Strata: %+v", got)
|
||||
}
|
||||
if _, err := chooseLayout(Request{Format: "gguf", Engine: "strata", GPUClass: "h100-80g"}, plan, true, set); err == nil {
|
||||
t.Error("Strata on a class it was not planned for")
|
||||
}
|
||||
|
||||
ps := profilesFor(got, Request{ContextLength: 32768}, 0, 0)
|
||||
p := ps[0]
|
||||
if p.Engine != "strata" || p.GPUClass != "6000-24g" || p.GPUsPerNode != 1 || p.MinMemoryGB != 52 ||
|
||||
!slices.Equal(p.Args, []string{"--family", "qwen", "--size", "IQ2_XS", "--context", "32768", "--vision", "no"}) ||
|
||||
!strings.HasPrefix(p.Name, "strata-") {
|
||||
t.Errorf("%+v", p)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user