Replicas counted per profile: seven on cards and four on CPUs under one name
A profile's replicas, set on any of a model's profiles, is how many of its replicas run there; the scaling becomes the sum. The reconciler starts the ones short of their profile before stopping the ones over, so a change of split never runs fewer. ctl model replicas sets them, adding a profile on CPUs to a GGUF model; the router already fills the higher-priority pool first, which the CPU profile is put below. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
6403e61f4b
commit
e23732fbbb
13 files changed
+585
-5
No files matched your search
@@ -22,6 +22,7 @@ import (
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
@@ -249,6 +250,13 @@ func (s *Service) Remove(ctx context.Context, name string) error {
|
||||
// Scale changes how many replicas a model added at runtime runs.
|
||||
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error {
|
||||
// Replicas counted per profile stay so only for the same total, one
|
||||
// way: any other scaling is the model's again, on whatever fits.
|
||||
if m.Split() && (sc.Min != sc.Max || sc.Min != m.SplitTotal()) {
|
||||
for i := range m.Profiles {
|
||||
m.Profiles[i].Replicas = 0
|
||||
}
|
||||
}
|
||||
m.Scaling = sc
|
||||
// A number set is what it runs now: a stopped model is started by it.
|
||||
m.Stopped = nil
|
||||
@@ -261,6 +269,98 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
|
||||
return final, err
|
||||
}
|
||||
|
||||
// CPUProfile is a profile on CPUs to add to a llama.cpp model, for replicas
|
||||
// counted per profile: what a replica of it reserves.
|
||||
type CPUProfile struct {
|
||||
Name string `json:"name,omitempty"`
|
||||
MemoryGB int `json:"memory_gb"`
|
||||
CPUCores int `json:"cpu_cores,omitempty"`
|
||||
}
|
||||
|
||||
// SetProfileReplicas sets how many of a model's replicas run on each of its
|
||||
// profiles: seven on cards and four on CPUs, under the one name. A profile
|
||||
// not named keeps its count. cpu, for a llama.cpp model with no profile on
|
||||
// CPUs, adds one — its card profile without the card, preferred below every
|
||||
// other, so the router fills the cards first and the CPUs after. Running
|
||||
// replicas carry on; those over a count stop once the ones short have
|
||||
// started.
|
||||
func (s *Service) SetProfileReplicas(ctx context.Context, name string, counts map[string]int, cpu *CPUProfile,
|
||||
by string,
|
||||
) (catalog.Model, error) {
|
||||
final, err := s.update(ctx, name, by, "count its replicas per profile", func(m *catalog.Model) error {
|
||||
if cpu != nil {
|
||||
if cpu.Name == "" {
|
||||
cpu.Name = "cpu"
|
||||
}
|
||||
if !slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Name == cpu.Name }) {
|
||||
p, err := cpuProfileOf(*m, *cpu)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
m.Profiles = append(m.Profiles, p)
|
||||
}
|
||||
}
|
||||
for prof, n := range counts {
|
||||
i := slices.IndexFunc(m.Profiles, func(p catalog.Profile) bool { return p.Name == prof })
|
||||
if i < 0 {
|
||||
return fmt.Errorf("model %q has no profile %q", name, prof)
|
||||
}
|
||||
if n < 0 {
|
||||
return fmt.Errorf("profile %q: %d replicas", prof, n)
|
||||
}
|
||||
m.Profiles[i].Replicas = n
|
||||
}
|
||||
n := m.SplitTotal()
|
||||
if n == 0 {
|
||||
return fmt.Errorf("none of %q's profiles would run a replica; llmgw ctl model stop %s keeps it without", name, name)
|
||||
}
|
||||
sc := catalog.Scaling{Min: n, Max: n, IdleTimeout: m.Scaling.IdleTimeout}
|
||||
if m.Stopped != nil {
|
||||
sc.IdleTimeout = m.Stopped.IdleTimeout
|
||||
m.Stopped = &sc
|
||||
} else {
|
||||
m.Scaling = sc
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err == nil {
|
||||
s.log.Info("replicas counted per profile", "model", name, "counts", counts, "by", by)
|
||||
}
|
||||
return final, err
|
||||
}
|
||||
|
||||
// cpuProfileOf is a model's llama.cpp profile on CPUs: the same engine and
|
||||
// arguments, without the card or what spreads layers over cards.
|
||||
func cpuProfileOf(m catalog.Model, cpu CPUProfile) (catalog.Profile, error) {
|
||||
i := slices.IndexFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" })
|
||||
if i < 0 {
|
||||
return catalog.Profile{}, fmt.Errorf("model %q is not served by llama.cpp; only a GGUF model runs on CPUs", m.Name)
|
||||
}
|
||||
if cpu.MemoryGB <= 0 {
|
||||
return catalog.Profile{}, errors.New("a profile on CPUs needs the memory a replica reserves")
|
||||
}
|
||||
p := m.Profiles[i]
|
||||
lowest := p.Priority
|
||||
for _, o := range m.Profiles {
|
||||
lowest = min(lowest, o.Priority)
|
||||
}
|
||||
out := catalog.Profile{Name: cpu.Name, Engine: p.Engine, EngineVersion: p.EngineVersion, Source: p.Source,
|
||||
Env: p.Env, NodeSelector: p.NodeSelector, StartupTimeout: p.StartupTimeout, Weight: p.Weight,
|
||||
Priority: lowest - 10, MinMemoryGB: cpu.MemoryGB, CPUCores: cpu.CPUCores}
|
||||
for j := 0; j < len(p.Args); j++ {
|
||||
switch a := p.Args[j]; {
|
||||
case slices.Contains([]string{"--n-gpu-layers", "-ngl", "--split-mode", "-sm", "--tensor-split", "-ts",
|
||||
"--main-gpu", "-mg"}, a):
|
||||
j++ // and its value
|
||||
case strings.HasPrefix(a, "--n-gpu-layers=") || strings.HasPrefix(a, "--split-mode=") ||
|
||||
strings.HasPrefix(a, "--tensor-split=") || strings.HasPrefix(a, "--main-gpu="):
|
||||
default:
|
||||
out.Args = append(out.Args, a)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Stop takes a model out of service and keeps it: none of its replicas run,
|
||||
// and what it was scaled to is kept for Start.
|
||||
func (s *Service) Stop(ctx context.Context, name, by string) (catalog.Model, error) {
|
||||
|
||||
@@ -5,6 +5,7 @@ import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"log/slog"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
@@ -458,3 +459,44 @@ func TestAStrataModelReservesTheMemoryItsExpertsTake(t *testing.T) {
|
||||
t.Errorf("an unknown size: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A GGUF model on a card gets a profile on CPUs, and seven replicas on cards
|
||||
// beside four on CPUs; a scaling of another total puts it back on whatever
|
||||
// fits.
|
||||
func TestReplicasCountedPerProfile(t *testing.T) {
|
||||
s, _, _ := newService(t)
|
||||
m := catalog.Model{Name: "g", Source: "/models/g.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
|
||||
Profiles: []catalog.Profile{{Name: "gpu", Engine: "llamacpp", GPUClass: "l40s-48g", GPUsPerNode: 2,
|
||||
Priority: 100, Args: []string{"--ctx-size", "16384", "--n-gpu-layers", "999", "--split-mode=layer"}}}}
|
||||
if err := s.Add(t.Context(), m, "ilker"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.SetProfileReplicas(t.Context(), "g", map[string]int{"gpu": 7, "cpu": 4}, nil, "ilker"); err == nil {
|
||||
t.Fatal("a count for a profile it does not have was taken")
|
||||
}
|
||||
got, err := s.SetProfileReplicas(t.Context(), "g", map[string]int{"gpu": 7, "cpu": 4},
|
||||
&CPUProfile{MemoryGB: 24, CPUCores: 8}, "ilker")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(got.Profiles) != 2 || got.Scaling.Min != 11 || got.Scaling.Max != 11 {
|
||||
t.Fatalf("profiles %+v, scaling %+v", got.Profiles, got.Scaling)
|
||||
}
|
||||
cpu := got.Profiles[1]
|
||||
if cpu.Name != "cpu" || cpu.GPUClass != "" || cpu.GPUsPerNode != 0 || cpu.Replicas != 4 || cpu.Priority >= 100 ||
|
||||
cpu.MinMemoryGB != 24 || cpu.CPUCores != 8 || !slices.Equal(cpu.Args, []string{"--ctx-size", "16384"}) {
|
||||
t.Errorf("the profile on CPUs: %+v", cpu)
|
||||
}
|
||||
// Stopped and started, it comes back split.
|
||||
_, _ = s.Stop(t.Context(), "g", "ilker")
|
||||
if got, _ = s.Start(t.Context(), "g", "ilker"); got.Scaling.Min != 11 || got.Profiles[0].Replicas != 7 {
|
||||
t.Errorf("started: %+v", got)
|
||||
}
|
||||
// The same total keeps the split; another one ends it.
|
||||
if got, _ = s.Scale(t.Context(), "g", catalog.Scaling{Min: 11, Max: 11}, "ilker"); !got.Split() {
|
||||
t.Error("the same total ended the split")
|
||||
}
|
||||
if got, _ = s.Scale(t.Context(), "g", catalog.Scaling{Min: 3, Max: 3}, "ilker"); got.Split() {
|
||||
t.Error("another total kept the split")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user