Strata named by hand is reported, its engine stopped with it, its memory required, Pascal allowed
CI / test (push) Successful in 9m28s
CI / test (push) Successful in 9m28s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
73091d7c97
commit
91f668fe7e
8 files changed
+244
-2
No files matched your search
@@ -130,6 +130,14 @@ func (s *Service) add(ctx context.Context, m catalog.Model, by string, onlyNew b
|
||||
// there already are. Required of what is added now; models stored before
|
||||
// this still load, and say what they are missing.
|
||||
for _, p := range m.Profiles {
|
||||
// Strata keeps a model's experts in the host's memory, tens of
|
||||
// gigabytes of it: placed as if it took none, two replicas, or one
|
||||
// beside a CPU model, filled a node until one was killed.
|
||||
if p.Engine == "strata" && p.MinMemoryGB <= 0 {
|
||||
return fmt.Errorf("model %q profile %q runs on Strata, which keeps the experts in host memory, and declares "+
|
||||
"none (min_memory_gb): give what Strata's model table says the size takes, e.g. 40 for Qwen3.8-Flash-Next IQ2_XS",
|
||||
m.Name, p.Name)
|
||||
}
|
||||
if p.CPUOnly() && p.MinMemoryGB <= 0 {
|
||||
return fmt.Errorf("model %q profile %q runs without a GPU and declares no memory (min_memory_gb): "+
|
||||
"placement could not tell when a node is full", m.Name, p.Name)
|
||||
|
||||
@@ -438,3 +438,18 @@ func TestAStoppedModelRunsNoneAndStartsAsItWas(t *testing.T) {
|
||||
t.Error("still stopped after a scale")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStrataModelMustSayTheMemoryItsExpertsTake(t *testing.T) {
|
||||
s, _, _ := newService(t)
|
||||
m := model("big")
|
||||
m.Source = "http://mirror:8081/big/v1/"
|
||||
m.Profiles = []catalog.Profile{{Name: "gpu", Engine: "strata", GPUClass: "6000-24g", GPUsPerNode: 1,
|
||||
Args: []string{"--family", "qwen", "--size", "IQ2_XS"}}}
|
||||
if err := s.Add(t.Context(), m, "ilker"); err == nil || !strings.Contains(err.Error(), "min_memory_gb") {
|
||||
t.Fatalf("added without its memory: %v", err)
|
||||
}
|
||||
m.Profiles[0].MinMemoryGB = 40
|
||||
if err := s.Add(t.Context(), m, "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user