Replicas counted per profile: seven on cards and four on CPUs under one name

A profile's replicas, set on any of a model's profiles, is how many of its
replicas run there; the scaling becomes the sum. The reconciler starts the
ones short of their profile before stopping the ones over, so a change of
split never runs fewer. ctl model replicas sets them, adding a profile on
CPUs to a GGUF model; the router already fills the higher-priority pool
first, which the CPU profile is put below.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-03 09:21:50 +01:00
1 parent 6403e61f4b
commit e23732fbbb
13 files changed
+585 -5

No files matched your search

+30 -4
View File
@@ -886,14 +886,25 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
}
desired := r.desiredFor(m)
// Replicas counted per profile: one short on its profile is started even
// with the total right — before one of a profile with too many is
// stopped, which the total going over then does — so changing seven on
// cards and four on CPUs to nine and two never runs fewer than eleven.
var deficit map[string]bool
var surplus []string
split := m.Split() && desired == m.SplitTotal()
if split {
deficit, surplus = r.splitGaps(m)
}
needUp := len(liveIDs) < desired || split && len(deficit) > 0 && len(liveIDs) == desired
outcome := notTried
switch {
case len(liveIDs) < desired && isParked:
case needUp && isParked:
// Not placed. Its failure is on every status the operator reads.
case len(liveIDs) < desired && r.isHeld(m.Name):
case needUp && r.isHeld(m.Name):
// It gave up room to a model of a higher class, which is placed
// first; taking it back now would undo the preemption.
case len(liveIDs) < desired:
case needUp:
if r.now().Before(blockedUntil) {
// Still in backoff. Rebuilding immediately would hot-loop on a
// model that cannot start — a missing weights path, say — and bury
@@ -912,7 +923,11 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
attribute.String("llmgw.model", m.Name),
attribute.Int("llmgw.replicas", len(liveIDs)),
attribute.Int("llmgw.desired", desired))
r.scaleDown(sctx, liveIDs[desired:], "more replicas than desired")
excess := liveIDs[desired:]
if split {
excess = pickSurplus(surplus, liveIDs, len(liveIDs)-desired)
}
r.scaleDown(sctx, excess, "more replicas than desired")
span.End()
}
@@ -976,6 +991,17 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
attribute.String("llmgw.model", m.Name),
attribute.StringSlice("llmgw.avoid", avoid))
profiles := r.orderedProfiles(m)
if m.Split() {
// Only the profiles short of their replicas.
deficit, _ := r.splitGaps(m)
profiles = slices.DeleteFunc(profiles, func(p placement.Profile) bool {
return !deficit[catalog.ProfileBase(p.Name)]
})
if len(profiles) == 0 {
pspan.End()
return notTried
}
}
res := r.sched.Place(profiles, avoid)
if res.Placement == nil && len(avoid) > 0 {
// Spreading across failure domains is a preference, not a rule. If it
+64
View File
@@ -0,0 +1,64 @@
package reconciler
import (
"slices"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// splitGaps is, for a model whose replicas are counted per profile, the
// profiles short of theirs and the replicas over: of a profile with more
// than it asks, or of one it no longer has. Counted as reconcileModel
// counts — a replica being retired is not one it has — so a replica
// restarted is replaced on its own profile.
func (r *Reconciler) splitGaps(m catalog.Model) (deficit map[string]bool, surplus []string) {
live := map[string][]string{}
r.mu.Lock()
for _, rep := range r.replicas {
if rep.Model == m.Name && rep.State.Counts() && rep.RetiringAt.IsZero() {
b := catalog.ProfileBase(rep.Profile)
live[b] = append(live[b], rep.ID)
}
}
r.mu.Unlock()
deficit = map[string]bool{}
for _, p := range m.Profiles {
ids := live[p.Name]
delete(live, p.Name)
slices.Sort(ids)
switch {
case len(ids) < p.Replicas:
deficit[p.Name] = true
case len(ids) > p.Replicas:
surplus = append(surplus, ids[p.Replicas:]...)
}
}
for _, ids := range live {
surplus = append(surplus, ids...)
}
slices.Sort(surplus)
return deficit, surplus
}
// pickSurplus is n of the live replicas to stop: those over their
// profile's count first, then any.
func pickSurplus(surplus, live []string, n int) []string {
out := make([]string, 0, n)
for _, id := range surplus {
if len(out) == n {
return out
}
if slices.Contains(live, id) {
out = append(out, id)
}
}
for _, id := range live {
if len(out) == n {
break
}
if !slices.Contains(out, id) {
out = append(out, id)
}
}
return out
}
+85
View File
@@ -0,0 +1,85 @@
package reconciler
import (
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
// splitModel runs gpu replicas on cards and cpu on CPUs, under one name.
func splitModel(gpu, cpu int) catalog.Model {
return catalog.Model{Name: "m", Source: "/models/m.gguf",
Scaling: catalog.Scaling{Min: gpu + cpu, Max: gpu + cpu},
Profiles: []catalog.Profile{
{Name: "gpu", Engine: "llamacpp", GPUClass: "h100-80g", GPUsPerNode: 1, Priority: 100, Replicas: gpu},
{Name: "cpu", Engine: "llamacpp", Priority: 10, Replicas: cpu},
}}
}
// byProfile counts the model's live replicas on each profile.
func byProfile(h *harness) map[string]int {
out := map[string]int{}
for _, rep := range h.r.Replicas() {
if rep.State.Counts() && rep.RetiringAt.IsZero() {
out[catalog.ProfileBase(rep.Profile)]++
}
}
return out
}
// Seven on cards and four on CPUs is what runs, not eleven on whatever fits
// best: with cards free for all eleven, four still go to the CPUs.
func TestReplicasRunAsManyOnEachProfileAsItAsks(t *testing.T) {
h := newHarness(t, splitModel(7, 4), gpuNode("1", 8), gpuNode("2", 8), cpuNode("3"), cpuNode("4"))
for range 14 {
h.r.Reconcile(t.Context())
h.markReady()
h.advance(time.Second)
}
if got := byProfile(h); got["gpu"] != 7 || got["cpu"] != 4 {
t.Fatalf("running %v, want 7 on cards and 4 on CPUs", got)
}
}
// Moving replicas from one profile to the other starts the new ones before
// stopping the old: the model never runs fewer than it asks.
func TestChangingTheSplitStartsBeforeItStops(t *testing.T) {
h := newHarness(t, splitModel(2, 1), gpuNode("1", 8), cpuNode("3"), cpuNode("4"))
for range 6 {
h.r.Reconcile(t.Context())
h.markReady()
h.advance(time.Second)
}
if got := byProfile(h); got["gpu"] != 2 || got["cpu"] != 1 {
t.Fatalf("running %v before the change", got)
}
h.setModels(t, splitModel(1, 2))
for range 10 {
h.r.Reconcile(t.Context())
h.markReady()
h.advance(h.r.cfg.DrainGrace + time.Second)
if n := replicasOf(h, "m"); n < 3 {
t.Fatalf("only %d replicas while the split changed", n)
}
}
if got := byProfile(h); got["gpu"] != 1 || got["cpu"] != 2 {
t.Errorf("running %v, want 1 on cards and 2 on CPUs", got)
}
}
// The scaling has to be the profiles' sum, so a count in one place cannot
// disagree with the other.
func TestReplicasPerProfileAndScalingAgree(t *testing.T) {
m := splitModel(2, 1)
m.Scaling = catalog.Scaling{Min: 1, Max: 5}
if _, err := catalog.New([]catalog.Model{m}); err == nil {
t.Error("profiles asking for 3 with scaling 1..5 were accepted")
}
stopped := splitModel(2, 1)
was := stopped.Scaling
stopped.Stopped, stopped.Scaling = &was, catalog.Scaling{}
if _, err := catalog.New([]catalog.Model{stopped}); err != nil {
t.Errorf("a stopped model with replicas per profile was refused: %v", err)
}
}