Ilker ManapandClaude Opus 5.5
e23732fbbb
A profile's replicas, set on any of a model's profiles, is how many of its replicas run there; the scaling becomes the sum. The reconciler starts the ones short of their profile before stopping the ones over, so a change of split never runs fewer. ctl model replicas sets them, adding a profile on CPUs to a GGUF model; the router already fills the higher-priority pool first, which the CPU profile is put below. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
86 lines
2.8 KiB
Go
86 lines
2.8 KiB
Go
package reconciler
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
)
|
|
|
|
// splitModel runs gpu replicas on cards and cpu on CPUs, under one name.
|
|
func splitModel(gpu, cpu int) catalog.Model {
|
|
return catalog.Model{Name: "m", Source: "/models/m.gguf",
|
|
Scaling: catalog.Scaling{Min: gpu + cpu, Max: gpu + cpu},
|
|
Profiles: []catalog.Profile{
|
|
{Name: "gpu", Engine: "llamacpp", GPUClass: "h100-80g", GPUsPerNode: 1, Priority: 100, Replicas: gpu},
|
|
{Name: "cpu", Engine: "llamacpp", Priority: 10, Replicas: cpu},
|
|
}}
|
|
}
|
|
|
|
// byProfile counts the model's live replicas on each profile.
|
|
func byProfile(h *harness) map[string]int {
|
|
out := map[string]int{}
|
|
for _, rep := range h.r.Replicas() {
|
|
if rep.State.Counts() && rep.RetiringAt.IsZero() {
|
|
out[catalog.ProfileBase(rep.Profile)]++
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Seven on cards and four on CPUs is what runs, not eleven on whatever fits
|
|
// best: with cards free for all eleven, four still go to the CPUs.
|
|
func TestReplicasRunAsManyOnEachProfileAsItAsks(t *testing.T) {
|
|
h := newHarness(t, splitModel(7, 4), gpuNode("1", 8), gpuNode("2", 8), cpuNode("3"), cpuNode("4"))
|
|
for range 14 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.advance(time.Second)
|
|
}
|
|
if got := byProfile(h); got["gpu"] != 7 || got["cpu"] != 4 {
|
|
t.Fatalf("running %v, want 7 on cards and 4 on CPUs", got)
|
|
}
|
|
}
|
|
|
|
// Moving replicas from one profile to the other starts the new ones before
|
|
// stopping the old: the model never runs fewer than it asks.
|
|
func TestChangingTheSplitStartsBeforeItStops(t *testing.T) {
|
|
h := newHarness(t, splitModel(2, 1), gpuNode("1", 8), cpuNode("3"), cpuNode("4"))
|
|
for range 6 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.advance(time.Second)
|
|
}
|
|
if got := byProfile(h); got["gpu"] != 2 || got["cpu"] != 1 {
|
|
t.Fatalf("running %v before the change", got)
|
|
}
|
|
h.setModels(t, splitModel(1, 2))
|
|
for range 10 {
|
|
h.r.Reconcile(t.Context())
|
|
h.markReady()
|
|
h.advance(h.r.cfg.DrainGrace + time.Second)
|
|
if n := replicasOf(h, "m"); n < 3 {
|
|
t.Fatalf("only %d replicas while the split changed", n)
|
|
}
|
|
}
|
|
if got := byProfile(h); got["gpu"] != 1 || got["cpu"] != 2 {
|
|
t.Errorf("running %v, want 1 on cards and 2 on CPUs", got)
|
|
}
|
|
}
|
|
|
|
// The scaling has to be the profiles' sum, so a count in one place cannot
|
|
// disagree with the other.
|
|
func TestReplicasPerProfileAndScalingAgree(t *testing.T) {
|
|
m := splitModel(2, 1)
|
|
m.Scaling = catalog.Scaling{Min: 1, Max: 5}
|
|
if _, err := catalog.New([]catalog.Model{m}); err == nil {
|
|
t.Error("profiles asking for 3 with scaling 1..5 were accepted")
|
|
}
|
|
stopped := splitModel(2, 1)
|
|
was := stopped.Scaling
|
|
stopped.Stopped, stopped.Scaling = &was, catalog.Scaling{}
|
|
if _, err := catalog.New([]catalog.Model{stopped}); err != nil {
|
|
t.Errorf("a stopped model with replicas per profile was refused: %v", err)
|
|
}
|
|
}
|