1 parent
7465c24830
commit
d6f8f433f4
24 files changed
+437
-22
No files matched your search
@@ -12,6 +12,7 @@ import (
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
)
|
||||
|
||||
@@ -554,3 +555,38 @@ func (f *fakeCatalog) Stop(_ context.Context, name, _ string) (catalog.Model, er
|
||||
func (f *fakeCatalog) Start(_ context.Context, name, _ string) (catalog.Model, error) {
|
||||
return catalog.Model{Name: name, Scaling: catalog.Scaling{Min: 1, Max: 1}}, f.err
|
||||
}
|
||||
|
||||
// A version of a family Strata runs, on a fleet with Strata, is served on it:
|
||||
// one card, its family and size, the experts' memory, from the directory.
|
||||
func TestAStrataFamilysVersionIsServedOnStrata(t *testing.T) {
|
||||
st := held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00001-of-00002.gguf")
|
||||
st.Source = "https://huggingface.co/ISTA-DASLab/Qwen3.8-Flash-Next-GSQ-RCO-GGUF/resolve/abc/IQ2_XS/" + st.File
|
||||
files := []mirror.Status{st, held("q38", "abc123", "Qwen3.8-Flash-Next-GSQ-RCO-IQ2_XS-00002-of-00002.gguf")}
|
||||
f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp", "strata"}, memoryGB: 251},
|
||||
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} })
|
||||
if f.Engine != "strata" || f.StrataFamily != "qwen" || f.StrataSize != "IQ2_XS" || f.Refusal != "" ||
|
||||
f.MinMemoryGB != 52 || f.Context != 32768 || !strings.HasSuffix(f.Source, "/q38/abc123/") {
|
||||
t.Fatalf("%+v", f)
|
||||
}
|
||||
// Without Strata on any node, llama.cpp as before, and its refusal.
|
||||
if f := planServe(english, st, files, "http://mirror:8081", fakeFleet{gpus: 1, engines: []string{"llamacpp"}},
|
||||
nil, func(string) gguf.Verdict { return gguf.Verdict{Checked: true, Reason: "unknown architecture"} }); f.Engine != "llamacpp" || f.Refusal == "" {
|
||||
t.Errorf("%+v", f)
|
||||
}
|
||||
form := url.Values{"name": {"q38"}, "source": {f.Source}, "engine": {"strata"}, "replicas": {"1"},
|
||||
"context_length": {"32768"}, "placement": {"gpu:6000-24g"}, "min_memory_gb": {"52"},
|
||||
"strata_family": {"qwen"}, "strata_size": {"IQ2_XS"}}
|
||||
m, err := buildModel(english, form.Get)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
p := m.Profiles[0]
|
||||
if p.Engine != "strata" || p.GPUClass != "6000-24g" || p.GPUsPerNode != 1 || p.MinMemoryGB != 52 ||
|
||||
strings.Join(p.Args, " ") != "--family qwen --size IQ2_XS --context 32768 --vision no" {
|
||||
t.Errorf("%+v", p)
|
||||
}
|
||||
form.Set("strata_size", "IQ9")
|
||||
if _, err := buildModel(english, form.Get); err == nil {
|
||||
t.Error("an unknown size was taken")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user