llmgw/internal/control/catalog/catalog_test.go

package catalog

import (
	"strings"
	"testing"
	"time"
)

func model(name string, profiles ...Profile) Model {
	if len(profiles) == 0 {
		profiles = []Profile{{Name: "p1", GPUClass: "h100-80g", TP: 4}}
	}
	return Model{Name: name, Source: "/models/" + name, Profiles: profiles}
}

func TestResolveNameAndAlias(t *testing.T) {
	m := model("llama-3.3-70b")
	m.Aliases = []string{"llama", "llama3"}
	c, err := New([]Model{m})
	if err != nil {
		t.Fatal(err)
	}

	for _, name := range []string{"llama-3.3-70b", "llama", "llama3"} {
		got, ok := c.Resolve(name)
		if !ok {
			t.Fatalf("%q did not resolve", name)
		}
		if got.Name != "llama-3.3-70b" {
			t.Errorf("%q resolved to %q", name, got.Name)
		}
	}
	if _, ok := c.Resolve("nope"); ok {
		t.Error("an unknown name resolved")
	}
}

func TestAliasCollisionRejected(t *testing.T) {
	// An alias shadowing another model would misroute every request for it,
	// silently, at request time.
	a := model("qwen")
	b := model("llama")
	b.Aliases = []string{"qwen"}

	_, err := New([]Model{a, b})
	if err == nil || !strings.Contains(err.Error(), "collides") {
		t.Errorf("collision not rejected: %v", err)
	}
}

func TestDuplicateAliasRejected(t *testing.T) {
	a := model("a")
	a.Aliases = []string{"shared"}
	b := model("b")
	b.Aliases = []string{"shared"}

	_, err := New([]Model{a, b})
	if err == nil || !strings.Contains(err.Error(), "claimed by both") {
		t.Errorf("duplicate alias not rejected: %v", err)
	}
}

func TestUnknownFallbackRejected(t *testing.T) {
	// A typo in a fallback should stop a deployment, not surface during the
	// outage when the fallback is finally needed.
	m := model("llama")
	m.Fallbacks = []string{"qwen-typo"}

	_, err := New([]Model{m})
	if err == nil || !strings.Contains(err.Error(), "fallback") {
		t.Errorf("unknown fallback not rejected: %v", err)
	}
}

func TestFallbackToAliasAccepted(t *testing.T) {
	a := model("qwen2.5-32b")
	a.Aliases = []string{"qwen"}
	b := model("llama")
	b.Fallbacks = []string{"qwen"}

	if _, err := New([]Model{a, b}); err != nil {
		t.Errorf("a fallback naming an alias was rejected: %v", err)
	}
}

func TestParallelismArithmeticValidated(t *testing.T) {
	// Wrong arithmetic hands the engine a spec it cannot satisfy, and it fails
	// minutes later during model loading rather than at config load.
	_, err := New([]Model{model("m", Profile{
		Name: "bad", GPUClass: "h100-80g", TP: 4, Nodes: 2, GPUsPerNode: 8,
	})})
	if err == nil || !strings.Contains(err.Error(), "does not match") {
		t.Errorf("bad arithmetic accepted: %v", err)
	}

	// 2 nodes x 8 GPUs = TP 16 is consistent.
	if _, err := New([]Model{model("m", Profile{
		Name: "ok", GPUClass: "h100-80g", TP: 16, Nodes: 2, GPUsPerNode: 8,
	})}); err != nil {
		t.Errorf("valid arithmetic rejected: %v", err)
	}
}

func TestDefaults(t *testing.T) {
	c, err := New([]Model{model("m", Profile{Name: "p", GPUClass: "h100-80g", TP: 4})})
	if err != nil {
		t.Fatal(err)
	}
	m, _ := c.Resolve("m")
	p := m.Profiles[0]

	if p.Engine != "vllm" {
		t.Errorf("engine=%q", p.Engine)
	}
	if p.Nodes != 1 {
		t.Errorf("nodes=%d", p.Nodes)
	}
	if p.GPUsPerNode != 4 {
		t.Errorf("gpus_per_node=%d; a single-node profile should default to TP", p.GPUsPerNode)
	}
	if p.StartupTimeout != 20*time.Minute {
		t.Errorf("startup_timeout=%v", p.StartupTimeout)
	}
	if p.Weight <= 0 {
		t.Errorf("weight=%v", p.Weight)
	}
	if m.Scaling.Max < 1 {
		t.Errorf("scaling max=%d", m.Scaling.Max)
	}
}

func TestRequiredFields(t *testing.T) {
	cases := map[string]Model{
		"no name":     {Source: "/m", Profiles: []Profile{{Name: "p", GPUClass: "c", TP: 1}}},
		"no source":   {Name: "m", Profiles: []Profile{{Name: "p", GPUClass: "c", TP: 1}}},
		"no profiles": {Name: "m", Source: "/m"},
		"no class":    {Name: "m", Source: "/m", Profiles: []Profile{{Name: "p", TP: 1}}},
	}
	for name, m := range cases {
		t.Run(name, func(t *testing.T) {
			if _, err := New([]Model{m}); err == nil {
				t.Error("accepted")
			}
		})
	}
}

func TestScalingBounds(t *testing.T) {
	m := model("m")
	m.Scaling = Scaling{Min: 5, Max: 2}
	if _, err := New([]Model{m}); err == nil {
		t.Error("max below min was accepted")
	}
}

func TestDuplicateProfileRejected(t *testing.T) {
	_, err := New([]Model{model("m",
		Profile{Name: "p", GPUClass: "h100-80g", TP: 1},
		Profile{Name: "p", GPUClass: "a100-40g", TP: 1},
	)})
	if err == nil || !strings.Contains(err.Error(), "twice") {
		t.Errorf("duplicate profile accepted: %v", err)
	}
}

func TestPlacementProfilesCarryPriority(t *testing.T) {
	c, _ := New([]Model{model("m",
		Profile{Name: "fast", GPUClass: "h100-80g", TP: 4, Priority: 100},
		Profile{Name: "slow", GPUClass: "a100-40g", TP: 4, Priority: 50},
	)})
	m, _ := c.Resolve("m")

	pp := m.PlacementProfiles()
	if len(pp) != 2 {
		t.Fatalf("got %d placement profiles", len(pp))
	}
	byName := map[string]int{}
	for _, p := range pp {
		byName[p.Name] = p.Priority
	}
	if byName["fast"] != 100 || byName["slow"] != 50 {
		t.Errorf("priorities lost in conversion: %v", byName)
	}
}

func TestEngineProfileCarriesModelIdentity(t *testing.T) {
	// The served name is what clients ask for and need not match the path.
	c, _ := New([]Model{model("llama-3.3-70b")})
	m, _ := c.Resolve("llama-3.3-70b")

	ep, ok := m.EngineProfile("p1")
	if !ok {
		t.Fatal("engine profile not found")
	}
	if ep.ModelPath != "/models/llama-3.3-70b" {
		t.Errorf("ModelPath=%q", ep.ModelPath)
	}
	if ep.ServedModelName != "llama-3.3-70b" {
		t.Errorf("ServedModelName=%q", ep.ServedModelName)
	}

	if _, ok := m.EngineProfile("nope"); ok {
		t.Error("an unknown profile resolved")
	}
}

func TestEngineProfileArgsAreCopied(t *testing.T) {
	// The reconciler hands these to an engine adapter; mutating them must not
	// corrupt the catalog for every later replica.
	c, _ := New([]Model{model("m", Profile{
		Name: "p", GPUClass: "h100-80g", TP: 1, Args: []string{"--max-model-len", "32768"},
	})})
	m, _ := c.Resolve("m")

	ep, _ := m.EngineProfile("p")
	ep.Args[0] = "tampered"

	again, _ := m.EngineProfile("p")
	if again.Args[0] != "--max-model-len" {
		t.Error("EngineProfile exposed the catalog's own slice")
	}
}

func TestNamesAndModelsOrder(t *testing.T) {
	c, _ := New([]Model{model("zeta"), model("alpha")})

	// Models keeps declaration order, so configuration reads as written.
	if got := c.Models(); got[0].Name != "zeta" {
		t.Errorf("Models order=%q", got[0].Name)
	}
	// Names is sorted, for stable CLI output.
	if got := c.Names(); got[0] != "alpha" {
		t.Errorf("Names order=%v", got)
	}
}

func TestSimulatedNeedsEveryProfileToBeSimulated(t *testing.T) {
	// The question the flag answers is "can this request come back as real
	// inference?". One real profile means yes, however many sim profiles sit
	// beside it, so a mixed model must not be labelled simulated.
	for _, tc := range []struct {
		name    string
		engines []string
		wantSim bool
	}{
		{"all simulated", []string{"sim", "sim"}, true},
		{"one real profile", []string{"sim", "llamacpp"}, false},
		{"no simulation", []string{"llamacpp"}, false},
		{"no profiles at all", nil, false},
	} {
		t.Run(tc.name, func(t *testing.T) {
			var m Model
			for _, e := range tc.engines {
				m.Profiles = append(m.Profiles, Profile{Engine: e})
			}
			if got := m.Simulated(); got != tc.wantSim {
				t.Errorf("Simulated() = %v, want %v", got, tc.wantSim)
			}
		})
	}
}