Canary of a model's newer weights: ctl model canary, promote and canaries, rolled back by itself when it serves worse
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0a4076928a
commit
a1dfd4df1d
13 files changed
+750
-18
No files matched your search
@@ -0,0 +1,101 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
"log/slog"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/canary"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
)
|
||||
|
||||
func canarySetup(t *testing.T) (*Server, *updModels, *restartFleet) {
|
||||
t.Helper()
|
||||
s, _, models, _, fleet := updSetup(t)
|
||||
models.onAdd = func(m catalog.Model) {
|
||||
fleet.mu.Lock()
|
||||
fleet.reps = append(fleet.reps, reconciler.Replica{ID: m.Name + "-1", Model: m.Name,
|
||||
State: reconciler.Ready, Endpoint: fleet.engine})
|
||||
fleet.mu.Unlock()
|
||||
}
|
||||
s.deps.Log = slog.New(slog.DiscardHandler)
|
||||
return s, models, fleet
|
||||
}
|
||||
|
||||
func TestACanaryRunsTheNewWeightsBesideAndIsPromoted(t *testing.T) {
|
||||
s, models, fleet := canarySetup(t)
|
||||
oldSource := models.models["qwen"].Source
|
||||
job, err := s.StartCanary("qwen", catalog.Canary{Percent: 10}, "tester")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
||||
}
|
||||
m, c := models.models["qwen"], models.models["qwen-canary"]
|
||||
if m.Source != oldSource || m.Canary == nil || m.Canary.Model != "qwen-canary" || m.Canary.Percent != 10 {
|
||||
t.Fatalf("the model was changed, or given no canary: %s %+v", m.Source, m.Canary)
|
||||
}
|
||||
if !strings.HasSuffix(c.Source, "/qwen/def98765/q4.gguf") || c.Origin.Revision != newRev || c.Scaling.Max != 1 {
|
||||
t.Fatalf("candidate: %s %+v %+v", c.Source, c.Origin, c.Scaling)
|
||||
}
|
||||
if _, err := s.StartCanary("qwen", catalog.Canary{Percent: 10}, "tester"); err == nil {
|
||||
t.Error("a second canary was started")
|
||||
}
|
||||
if _, err := s.SetCanaryPercent(context.Background(), "qwen", 50, "tester"); err != nil || models.models["qwen"].Canary.Percent != 50 {
|
||||
t.Errorf("share: %v", err)
|
||||
}
|
||||
|
||||
job, err = s.StartPromote("qwen", "tester")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
||||
}
|
||||
m = models.models["qwen"]
|
||||
if m.Source != c.Source || m.Canary != nil || m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource {
|
||||
t.Fatalf("promoted: %s %+v %+v", m.Source, m.Canary, m.Origin)
|
||||
}
|
||||
if _, ok := models.models["qwen-canary"]; ok {
|
||||
t.Error("the canary was left in the catalog")
|
||||
}
|
||||
for _, r := range fleet.Replicas() {
|
||||
if r.Model == "qwen" && !strings.Contains(r.ID, "new") {
|
||||
t.Errorf("%s was not replaced", r.ID)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestACanaryServingWorseIsRolledBackByItself(t *testing.T) {
|
||||
s, models, _ := canarySetup(t)
|
||||
job, _ := s.StartCanary("qwen", catalog.Canary{Percent: 10, MinRequests: 20}, "tester")
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
||||
}
|
||||
var stat canary.Stat
|
||||
stat.Canary = "qwen-canary"
|
||||
for i := range 200 {
|
||||
stat.Stable.Add(200, true, 100*time.Millisecond, time.Second)
|
||||
if i < 30 {
|
||||
status := 200
|
||||
if i%3 == 0 {
|
||||
status = 502
|
||||
}
|
||||
stat.Candidate.Add(status, true, 100*time.Millisecond, time.Second)
|
||||
}
|
||||
}
|
||||
reset := ""
|
||||
s.deps.CanaryStats = func() map[string]canary.Stat { return map[string]canary.Stat{"qwen": stat} }
|
||||
s.deps.ResetCanary = func(m string) { reset = m }
|
||||
s.checkCanaries(context.Background())
|
||||
if models.models["qwen"].Canary != nil || reset != "qwen" {
|
||||
t.Fatalf("a canary failing a third of its requests was kept: %+v", models.models["qwen"].Canary)
|
||||
}
|
||||
if _, ok := models.models["qwen-canary"]; ok {
|
||||
t.Error("the candidate was left in the catalog")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user