Replicas counted per profile: seven on cards and four on CPUs under one name
A profile's replicas, set on any of a model's profiles, is how many of its replicas run there; the scaling becomes the sum. The reconciler starts the ones short of their profile before stopping the ones over, so a change of split never runs fewer. ctl model replicas sets them, adding a profile on CPUs to a GGUF model; the router already fills the higher-priority pool first, which the CPU profile is put below. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6403e61f4b
commit
e23732fbbb
13 files changed
+585
-5
No files matched your search
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -458,3 +459,44 @@ func TestAStrataModelReservesTheMemoryItsExpertsTake(t *testing.T) {
|
||||
t.Errorf("an unknown size: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A GGUF model on a card gets a profile on CPUs, and seven replicas on cards
|
||||
// beside four on CPUs; a scaling of another total puts it back on whatever
|
||||
// fits.
|
||||
func TestReplicasCountedPerProfile(t *testing.T) {
|
||||
s, _, _ := newService(t)
|
||||
m := catalog.Model{Name: "g", Source: "/models/g.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "gpu", Engine: "llamacpp", GPUClass: "l40s-48g", GPUsPerNode: 2,
|
||||
Priority: 100, Args: []string{"--ctx-size", "16384", "--n-gpu-layers", "999", "--split-mode=layer"}}}}
|
||||
if err := s.Add(t.Context(), m, "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.SetProfileReplicas(t.Context(), "g", map[string]int{"gpu": 7, "cpu": 4}, nil, "ilker"); err == nil {
|
||||
t.Fatal("a count for a profile it does not have was taken")
|
||||
}
|
||||
got, err := s.SetProfileReplicas(t.Context(), "g", map[string]int{"gpu": 7, "cpu": 4},
|
||||
&CPUProfile{MemoryGB: 24, CPUCores: 8}, "ilker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Profiles) != 2 || got.Scaling.Min != 11 || got.Scaling.Max != 11 {
|
||||
t.Fatalf("profiles %+v, scaling %+v", got.Profiles, got.Scaling)
|
||||
}
|
||||
cpu := got.Profiles[1]
|
||||
if cpu.Name != "cpu" || cpu.GPUClass != "" || cpu.GPUsPerNode != 0 || cpu.Replicas != 4 || cpu.Priority >= 100 ||
|
||||
cpu.MinMemoryGB != 24 || cpu.CPUCores != 8 || !slices.Equal(cpu.Args, []string{"--ctx-size", "16384"}) {
|
||||
t.Errorf("the profile on CPUs: %+v", cpu)
|
||||
}
|
||||
// Stopped and started, it comes back split.
|
||||
_, _ = s.Stop(t.Context(), "g", "ilker")
|
||||
if got, _ = s.Start(t.Context(), "g", "ilker"); got.Scaling.Min != 11 || got.Profiles[0].Replicas != 7 {
|
||||
t.Errorf("started: %+v", got)
|
||||
}
|
||||
// The same total keeps the split; another one ends it.
|
||||
if got, _ = s.Scale(t.Context(), "g", catalog.Scaling{Min: 11, Max: 11}, "ilker"); !got.Split() {
|
||||
t.Error("the same total ended the split")
|
||||
}
|
||||
if got, _ = s.Scale(t.Context(), "g", catalog.Scaling{Min: 3, Max: 3}, "ilker"); got.Split() {
|
||||
t.Error("another total kept the split")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user