Replicas counted per profile: seven on cards and four on CPUs under one name

A profile's replicas, set on any of a model's profiles, is how many of its
replicas run there; the scaling becomes the sum. The reconciler starts the
ones short of their profile before stopping the ones over, so a change of
split never runs fewer. ctl model replicas sets them, adding a profile on
CPUs to a GGUF model; the router already fills the higher-priority pool
first, which the CPU profile is put below.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-03 09:21:50 +01:00
1 parent 6403e61f4b
commit e23732fbbb
13 files changed
+585 -5

No files matched your search

+100
View File
@@ -22,6 +22,7 @@ import (
"fmt"
"log/slog"
"slices"
"strings"
"sync"
"time"
@@ -249,6 +250,13 @@ func (s *Service) Remove(ctx context.Context, name string) error {
// Scale changes how many replicas a model added at runtime runs.
func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by string) (catalog.Model, error) {
final, err := s.update(ctx, name, by, "change its scaling", func(m *catalog.Model) error {
// Replicas counted per profile stay so only for the same total, one
// way: any other scaling is the model's again, on whatever fits.
if m.Split() && (sc.Min != sc.Max || sc.Min != m.SplitTotal()) {
for i := range m.Profiles {
m.Profiles[i].Replicas = 0
}
}
m.Scaling = sc
// A number set is what it runs now: a stopped model is started by it.
m.Stopped = nil
@@ -261,6 +269,98 @@ func (s *Service) Scale(ctx context.Context, name string, sc catalog.Scaling, by
return final, err
}
// CPUProfile is a profile on CPUs to add to a llama.cpp model, for replicas
// counted per profile: what a replica of it reserves.
type CPUProfile struct {
Name string `json:"name,omitempty"`
MemoryGB int `json:"memory_gb"`
CPUCores int `json:"cpu_cores,omitempty"`
}
// SetProfileReplicas sets how many of a model's replicas run on each of its
// profiles: seven on cards and four on CPUs, under the one name. A profile
// not named keeps its count. cpu, for a llama.cpp model with no profile on
// CPUs, adds one — its card profile without the card, preferred below every
// other, so the router fills the cards first and the CPUs after. Running
// replicas carry on; those over a count stop once the ones short have
// started.
func (s *Service) SetProfileReplicas(ctx context.Context, name string, counts map[string]int, cpu *CPUProfile,
by string,
) (catalog.Model, error) {
final, err := s.update(ctx, name, by, "count its replicas per profile", func(m *catalog.Model) error {
if cpu != nil {
if cpu.Name == "" {
cpu.Name = "cpu"
}
if !slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Name == cpu.Name }) {
p, err := cpuProfileOf(*m, *cpu)
if err != nil {
return err
}
m.Profiles = append(m.Profiles, p)
}
}
for prof, n := range counts {
i := slices.IndexFunc(m.Profiles, func(p catalog.Profile) bool { return p.Name == prof })
if i < 0 {
return fmt.Errorf("model %q has no profile %q", name, prof)
}
if n < 0 {
return fmt.Errorf("profile %q: %d replicas", prof, n)
}
m.Profiles[i].Replicas = n
}
n := m.SplitTotal()
if n == 0 {
return fmt.Errorf("none of %q's profiles would run a replica; llmgw ctl model stop %s keeps it without", name, name)
}
sc := catalog.Scaling{Min: n, Max: n, IdleTimeout: m.Scaling.IdleTimeout}
if m.Stopped != nil {
sc.IdleTimeout = m.Stopped.IdleTimeout
m.Stopped = &sc
} else {
m.Scaling = sc
}
return nil
})
if err == nil {
s.log.Info("replicas counted per profile", "model", name, "counts", counts, "by", by)
}
return final, err
}
// cpuProfileOf is a model's llama.cpp profile on CPUs: the same engine and
// arguments, without the card or what spreads layers over cards.
func cpuProfileOf(m catalog.Model, cpu CPUProfile) (catalog.Profile, error) {
i := slices.IndexFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" })
if i < 0 {
return catalog.Profile{}, fmt.Errorf("model %q is not served by llama.cpp; only a GGUF model runs on CPUs", m.Name)
}
if cpu.MemoryGB <= 0 {
return catalog.Profile{}, errors.New("a profile on CPUs needs the memory a replica reserves")
}
p := m.Profiles[i]
lowest := p.Priority
for _, o := range m.Profiles {
lowest = min(lowest, o.Priority)
}
out := catalog.Profile{Name: cpu.Name, Engine: p.Engine, EngineVersion: p.EngineVersion, Source: p.Source,
Env: p.Env, NodeSelector: p.NodeSelector, StartupTimeout: p.StartupTimeout, Weight: p.Weight,
Priority: lowest - 10, MinMemoryGB: cpu.MemoryGB, CPUCores: cpu.CPUCores}
for j := 0; j < len(p.Args); j++ {
switch a := p.Args[j]; {
case slices.Contains([]string{"--n-gpu-layers", "-ngl", "--split-mode", "-sm", "--tensor-split", "-ts",
"--main-gpu", "-mg"}, a):
j++ // and its value
case strings.HasPrefix(a, "--n-gpu-layers=") || strings.HasPrefix(a, "--split-mode=") ||
strings.HasPrefix(a, "--tensor-split=") || strings.HasPrefix(a, "--main-gpu="):
default:
out.Args = append(out.Args, a)
}
}
return out, nil
}
// Stop takes a model out of service and keeps it: none of its replicas run,
// and what it was scaled to is kept for Start.
func (s *Service) Stop(ctx context.Context, name, by string) (catalog.Model, error) {
@@ -5,6 +5,7 @@ import (
"encoding/json"
"errors"
"log/slog"
"slices"
"strings"
"testing"
"time"
@@ -458,3 +459,44 @@ func TestAStrataModelReservesTheMemoryItsExpertsTake(t *testing.T) {
t.Errorf("an unknown size: %v", err)
}
}
// A GGUF model on a card gets a profile on CPUs, and seven replicas on cards
// beside four on CPUs; a scaling of another total puts it back on whatever
// fits.
func TestReplicasCountedPerProfile(t *testing.T) {
s, _, _ := newService(t)
m := catalog.Model{Name: "g", Source: "/models/g.gguf", Scaling: catalog.Scaling{Min: 1, Max: 1},
Profiles: []catalog.Profile{{Name: "gpu", Engine: "llamacpp", GPUClass: "l40s-48g", GPUsPerNode: 2,
Priority: 100, Args: []string{"--ctx-size", "16384", "--n-gpu-layers", "999", "--split-mode=layer"}}}}
if err := s.Add(t.Context(), m, "ilker"); err != nil {
t.Fatal(err)
}
if _, err := s.SetProfileReplicas(t.Context(), "g", map[string]int{"gpu": 7, "cpu": 4}, nil, "ilker"); err == nil {
t.Fatal("a count for a profile it does not have was taken")
}
got, err := s.SetProfileReplicas(t.Context(), "g", map[string]int{"gpu": 7, "cpu": 4},
&CPUProfile{MemoryGB: 24, CPUCores: 8}, "ilker")
if err != nil {
t.Fatal(err)
}
if len(got.Profiles) != 2 || got.Scaling.Min != 11 || got.Scaling.Max != 11 {
t.Fatalf("profiles %+v, scaling %+v", got.Profiles, got.Scaling)
}
cpu := got.Profiles[1]
if cpu.Name != "cpu" || cpu.GPUClass != "" || cpu.GPUsPerNode != 0 || cpu.Replicas != 4 || cpu.Priority >= 100 ||
cpu.MinMemoryGB != 24 || cpu.CPUCores != 8 || !slices.Equal(cpu.Args, []string{"--ctx-size", "16384"}) {
t.Errorf("the profile on CPUs: %+v", cpu)
}
// Stopped and started, it comes back split.
_, _ = s.Stop(t.Context(), "g", "ilker")
if got, _ = s.Start(t.Context(), "g", "ilker"); got.Scaling.Min != 11 || got.Profiles[0].Replicas != 7 {
t.Errorf("started: %+v", got)
}
// The same total keeps the split; another one ends it.
if got, _ = s.Scale(t.Context(), "g", catalog.Scaling{Min: 11, Max: 11}, "ilker"); !got.Split() {
t.Error("the same total ended the split")
}
if got, _ = s.Scale(t.Context(), "g", catalog.Scaling{Min: 3, Max: 3}, "ilker"); got.Split() {
t.Error("another total kept the split")
}
}