llmgw/internal/control/modelserve/gate_test.go

package modelserve

import (
	"context"
	"errors"
	"strings"
	"sync"
	"testing"

	"gitea.mynodes.xyz/ilker/llmgw/internal/bench/eval"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)

// fakeEval scores whatever it is asked what it is told to.
type fakeEval struct {
	mu    sync.Mutex
	score float64
	asked []string
}

func (f *fakeEval) Check(_ string, r eval.Request) error {
	if r.Suite == "missing" {
		return errors.New("no suite called missing")
	}
	return nil
}

func (f *fakeEval) Run(_ context.Context, model string, r eval.Request, by, job string) (store.EvalRun, error) {
	f.mu.Lock()
	defer f.mu.Unlock()
	f.asked = append(f.asked, model+" "+r.Named())
	threshold := r.MinScore
	if threshold == 0 {
		threshold = 0.8
	}
	below := 0
	if f.score < 1 {
		below = 1
	}
	return store.EvalRun{ID: "eval-1", Model: "qwen", Target: model, Suite: r.Named(), Score: f.score,
		Threshold: threshold, Passed: f.score >= threshold, Cases: 4, Below: below, By: by, Job: job}, nil
}

func (f *fakeEval) said() []string {
	f.mu.Lock()
	defer f.mu.Unlock()
	return append([]string{}, f.asked...)
}

func stepOf(j Job, name string) Step {
	for _, s := range j.Steps {
		if s.Name == name {
			return s
		}
	}
	return Step{}
}

func TestAPromoteBelowTheSuitesScoreIsRefusedAndTheCanaryStays(t *testing.T) {
	s, models, _ := canarySetup(t)
	ev := &fakeEval{score: 0.5}
	s.deps.Eval = ev
	job, _ := s.StartCanary("qwen", catalog.Canary{Percent: 10}, "tester")
	if j := wait(t, s, job.ID); j.State != Done {
		t.Fatalf("%s: %+v", j.Error, j.Steps)
	}
	oldSource := models.models["qwen"].Source
	gate := Change{Eval: &eval.Request{Suite: "arithmetic", MinScore: 0.8}}

	job, err := s.StartPromote("qwen", "tester", gate)
	if err != nil {
		t.Fatal(err)
	}
	j := wait(t, s, job.ID)
	if j.State != Failed || stepOf(j, "eval").State != Failed ||
		!strings.Contains(j.Error, "qwen-canary scored 0.50 on arithmetic") ||
		!strings.Contains(j.Error, "stays a canary") || stepOf(j, "catalog").Name != "" {
		t.Fatalf("a canary scoring 0.5 of 0.8 was promoted, or not said so: %s %+v", j.Error, j.Steps)
	}
	if j.Eval == nil || j.Eval.Score != 0.5 || j.Eval.Job != j.ID {
		t.Errorf("the job does not carry the score: %+v", j.Eval)
	}
	m := models.models["qwen"]
	if m.Source != oldSource || m.Canary == nil || m.Canary.Percent != 10 {
		t.Fatalf("the model moved, or lost its canary: %s %+v", m.Source, m.Canary)
	}
	if _, ok := models.models["qwen-canary"]; !ok {
		t.Fatal("the canary was taken out; a refused promotion leaves that to its owner")
	}
	if got := ev.said(); len(got) != 1 || got[0] != "qwen-canary arithmetic" {
		t.Errorf("asked %v", got)
	}

	ev.score = 0.9
	job, _ = s.StartPromote("qwen", "tester", gate)
	if j := wait(t, s, job.ID); j.State != Done || !strings.Contains(stepOf(j, "eval").Detail, "scored 0.90") {
		t.Fatalf("%s: %+v", j.Error, j.Steps)
	}
	if m := models.models["qwen"]; m.Source == oldSource || m.Canary != nil {
		t.Fatalf("a canary that passed was not promoted: %s %+v", m.Source, m.Canary)
	}
}

func TestAnUpdateHeldToTheModelsSuiteTriesTheNewWeightsFirstAndKeepsTheOldBelowIt(t *testing.T) {
	s, models, fleet := canarySetup(t)
	ev := &fakeEval{score: 0.4}
	s.deps.Eval = ev
	m := models.models["qwen"]
	m.Eval = &catalog.EvalGate{Suite: "arithmetic", MinScore: 0.75}
	models.models["qwen"] = m
	oldSource := m.Source

	job, err := s.StartUpdate("qwen", "tester", Change{})
	if err != nil {
		t.Fatal(err)
	}
	j := wait(t, s, job.ID)
	if j.State != Failed || !strings.Contains(j.Error, "scored 0.40") || !strings.Contains(j.Error, "taken out again") {
		t.Fatalf("%s: %+v", j.Error, j.Steps)
	}
	if got := ev.said(); len(got) != 1 || got[0] != "qwen-canary arithmetic" {
		t.Fatalf("the new weights were not what was asked: %v", got)
	}
	m = models.models["qwen"]
	if m.Source != oldSource || m.Canary != nil {
		t.Fatalf("the model moved to weights that failed, or kept their canary: %s %+v", m.Source, m.Canary)
	}
	if _, ok := models.models["qwen-canary"]; ok {
		t.Fatal("the failed weights were left in the catalog")
	}
	for _, r := range fleet.Replicas() {
		if r.Model == "qwen" && strings.Contains(r.ID, "new") {
			t.Fatalf("%s was replaced onto weights that failed", r.ID)
		}
	}

	// Scoring enough, they replace the old.
	ev.score = 1
	job, _ = s.StartUpdate("qwen", "tester", Change{})
	j = wait(t, s, job.ID)
	if j.State != Done || stepOf(j, "eval").State != Done || stepOf(j, "split").Detail != "0% of qwen's requests go to qwen-canary" {
		t.Fatalf("%s: %+v", j.Error, j.Steps)
	}
	if m := models.models["qwen"]; m.Source == oldSource || !strings.HasSuffix(m.Source, "/def98765/q4.gguf") ||
		m.Canary != nil || m.Origin.Previous == nil || m.Origin.Previous.Source != oldSource || m.Eval == nil {
		t.Fatalf("not moved to the new weights, or the gate was lost: %s %+v %+v", m.Source, m.Canary, m.Origin)
	}
	if _, ok := models.models["qwen-canary"]; ok {
		t.Error("the canary was left in the catalog")
	}
}

func TestAModelsSuiteCanBeSkippedOnceAndAMissingOneIsRefusedAtOnce(t *testing.T) {
	s, models, _ := canarySetup(t)
	ev := &fakeEval{score: 0}
	s.deps.Eval = ev
	m := models.models["qwen"]
	m.Eval = &catalog.EvalGate{Suite: "arithmetic"}
	models.models["qwen"] = m

	if _, err := s.StartUpdate("qwen", "tester", Change{Eval: &eval.Request{Suite: "missing"}}); err == nil ||
		!strings.Contains(err.Error(), "no suite called missing") {
		t.Fatalf("an update held to a suite that is not there was started: %v", err)
	}

	job, err := s.StartUpdate("qwen", "tester", Change{SkipEval: true})
	if err != nil {
		t.Fatal(err)
	}
	j := wait(t, s, job.ID)
	if j.State != Done || stepOf(j, "eval").State != Skipped || !strings.Contains(stepOf(j, "eval").Detail, "tester") {
		t.Fatalf("%s: %+v", j.Error, j.Steps)
	}
	if len(ev.said()) != 0 {
		t.Errorf("asked %v while skipped", ev.said())
	}

	// Without anything to run suites, a gate is refused rather than ignored.
	s.deps.Eval = nil
	if _, err := s.StartUpdate("qwen", "tester", Change{}); err == nil {
		t.Error("an update held to the model's suite started with nothing to run it")
	}
}