Ilker ManapandClaude Opus 5.5
a1dfd4df1d
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
102 lines
3.3 KiB
Go
102 lines
3.3 KiB
Go
package modelserve
|
|
|
|
import (
|
|
"context"
|
|
"log/slog"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/canary"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
|
)
|
|
|
|
func canarySetup(t *testing.T) (*Server, *updModels, *restartFleet) {
|
|
t.Helper()
|
|
s, _, models, _, fleet := updSetup(t)
|
|
models.onAdd = func(m catalog.Model) {
|
|
fleet.mu.Lock()
|
|
fleet.reps = append(fleet.reps, reconciler.Replica{ID: m.Name + "-1", Model: m.Name,
|
|
State: reconciler.Ready, Endpoint: fleet.engine})
|
|
fleet.mu.Unlock()
|
|
}
|
|
s.deps.Log = slog.New(slog.DiscardHandler)
|
|
return s, models, fleet
|
|
}
|
|
|
|
func TestACanaryRunsTheNewWeightsBesideAndIsPromoted(t *testing.T) {
|
|
s, models, fleet := canarySetup(t)
|
|
oldSource := models.models["qwen"].Source
|
|
job, err := s.StartCanary("qwen", catalog.Canary{Percent: 10}, "tester")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if j := wait(t, s, job.ID); j.State != Done {
|
|
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
|
}
|
|
m, c := models.models["qwen"], models.models["qwen-canary"]
|
|
if m.Source != oldSource || m.Canary == nil || m.Canary.Model != "qwen-canary" || m.Canary.Percent != 10 {
|
|
t.Fatalf("the model was changed, or given no canary: %s %+v", m.Source, m.Canary)
|
|
}
|
|
if !strings.HasSuffix(c.Source, "/qwen/def98765/q4.gguf") || c.Origin.Revision != newRev || c.Scaling.Max != 1 {
|
|
t.Fatalf("candidate: %s %+v %+v", c.Source, c.Origin, c.Scaling)
|
|
}
|
|
if _, err := s.StartCanary("qwen", catalog.Canary{Percent: 10}, "tester"); err == nil {
|
|
t.Error("a second canary was started")
|
|
}
|
|
if _, err := s.SetCanaryPercent(context.Background(), "qwen", 50, "tester"); err != nil || models.models["qwen"].Canary.Percent != 50 {
|
|
t.Errorf("share: %v", err)
|
|
}
|
|
|
|
job, err = s.StartPromote("qwen", "tester")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if j := wait(t, s, job.ID); j.State != Done {
|
|
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
|
}
|
|
m = models.models["qwen"]
|
|
if m.Source != c.Source || m.Canary != nil || m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource {
|
|
t.Fatalf("promoted: %s %+v %+v", m.Source, m.Canary, m.Origin)
|
|
}
|
|
if _, ok := models.models["qwen-canary"]; ok {
|
|
t.Error("the canary was left in the catalog")
|
|
}
|
|
for _, r := range fleet.Replicas() {
|
|
if r.Model == "qwen" && !strings.Contains(r.ID, "new") {
|
|
t.Errorf("%s was not replaced", r.ID)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestACanaryServingWorseIsRolledBackByItself(t *testing.T) {
|
|
s, models, _ := canarySetup(t)
|
|
job, _ := s.StartCanary("qwen", catalog.Canary{Percent: 10, MinRequests: 20}, "tester")
|
|
if j := wait(t, s, job.ID); j.State != Done {
|
|
t.Fatalf("%s: %+v", j.Error, j.Steps)
|
|
}
|
|
var stat canary.Stat
|
|
stat.Canary = "qwen-canary"
|
|
for i := range 200 {
|
|
stat.Stable.Add(200, true, 100*time.Millisecond, time.Second)
|
|
if i < 30 {
|
|
status := 200
|
|
if i%3 == 0 {
|
|
status = 502
|
|
}
|
|
stat.Candidate.Add(status, true, 100*time.Millisecond, time.Second)
|
|
}
|
|
}
|
|
reset := ""
|
|
s.deps.CanaryStats = func() map[string]canary.Stat { return map[string]canary.Stat{"qwen": stat} }
|
|
s.deps.ResetCanary = func(m string) { reset = m }
|
|
s.checkCanaries(context.Background())
|
|
if models.models["qwen"].Canary != nil || reset != "qwen" {
|
|
t.Fatalf("a canary failing a third of its requests was kept: %+v", models.models["qwen"].Canary)
|
|
}
|
|
if _, ok := models.models["qwen-canary"]; ok {
|
|
t.Error("the candidate was left in the catalog")
|
|
}
|
|
}
|