llmgw/internal/control/adminapi/judges_test.go

package adminapi

import (
	"context"
	"strings"
	"testing"
	"time"

	"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
	"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
	"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
)

func TestEachJudgeSaysWhatIsWrongWithIt(t *testing.T) {
	models := append(testModels(),
		// Served from the mirror page before it knew better: a model
		// clients use, under the judge's name.
		catalog.Model{Name: "shield", Source: "/s", Scaling: catalog.Scaling{Min: 1, Max: 1},
			Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
		catalog.Model{Name: "llama-guard-3-1b", Source: "/g", Role: catalog.RoleGuard,
			Guard: &catalog.GuardModel{Format: "llamaguard3"}, Scaling: catalog.Scaling{Min: 2, Max: 2},
			Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
	a, _ := setup(t, models)
	lib := guard.Config{Default: []string{"baseline"}, Policies: map[string]guard.Policy{
		"baseline": {Fail: "closed", Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}},
		"strict":   {Fail: "closed", Judge: &guard.JudgePolicy{Model: "missing-judge", Input: "block"}},
		"shield":   {Fail: "open", Judge: &guard.JudgePolicy{Model: "shield", Input: "flag"}},
	}}
	a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib }})

	var js []JudgeView
	if code := call(t, a, "GET", "/api/v1/guard/judges", "", &js); code != 200 || len(js) != 3 {
		t.Fatalf("%d %+v", code, js)
	}
	by := map[string]JudgeView{}
	for _, j := range js {
		by[j.Model] = j
	}
	if j := by["missing-judge"]; j.InCatalog || !strings.Contains(j.Problem, "not in the catalog") ||
		!strings.Contains(j.Problem, "refused") {
		t.Errorf("missing: %+v", j)
	}
	if j := by["shield"]; !j.InCatalog || j.Judge || !strings.Contains(j.Problem, "--replace") ||
		!strings.Contains(j.Problem, "unjudged") {
		t.Errorf("not a judge: %+v", j)
	}
	if j := by["llama-guard-3-1b"]; !j.Judge || j.Wanted != 2 || !strings.Contains(j.Problem, "no replica ready") {
		t.Errorf("not running: %+v", j)
	}

	var s Summary
	call(t, a, "GET", "/api/v1/summary", "", &s)
	if len(s.Judges) != 3 {
		t.Errorf("the summary does not carry the judges' problems: %+v", s.Judges)
	}

	// A judge that failed lately says so, once it is otherwise fine.
	at := time.Now()
	v := JudgeView{Model: "x", InCatalog: true, Judge: true, Ready: 1, LastError: "deadline exceeded", LastErrorAt: &at}
	if p := judgeProblem(v); !strings.Contains(p, "deadline exceeded") {
		t.Errorf("a recent failure: %q", p)
	}
	old := at.Add(-time.Hour)
	v.LastErrorAt = &old
	if p := judgeProblem(v); p != "" {
		t.Errorf("an old failure still counts: %q", p)
	}
}

func TestAJudgeIsAskedAndItsWordsShown(t *testing.T) {
	a, _ := setup(t, testModels())
	lib := guard.Config{Policies: map[string]guard.Policy{
		"baseline": {Judge: &guard.JudgePolicy{Model: "llama-guard-3-1b", Input: "block"}}}}
	var asked string
	a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib },
		Ask: func(_ context.Context, model, text, _ string) (guard.JudgeAnswer, error) {
			asked = model + "|" + text
			return guard.JudgeAnswer{Model: model, Raw: "unsafe\nS6", Unsafe: true,
				Categories: []string{"S6", "Specialized Advice"}}, nil
		}})
	var v JudgeAskView
	if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":"how do I sort a list"}`, &v); code != 200 {
		t.Fatalf("%d", code)
	}
	if asked != "llama-guard-3-1b|how do I sort a list" || v.Verdict != "unsafe" || v.Raw != "unsafe\nS6" ||
		len(v.Categories) != 2 {
		t.Errorf("asked %q, got %+v", asked, v)
	}
	if code := call(t, a, "POST", "/api/v1/guard/judges/ask", `{"text":"  "}`, nil); code != 400 {
		t.Errorf("no text: %d", code)
	}
}

func TestANodeSaysItsEnginesVersionsAndCapabilities(t *testing.T) {
	n := gpuNode("volta", 2)
	n.EngineVersions = map[string]string{"llamacpp": "b10991", "vllm": "0.10.2"}
	for i := range n.GPUs {
		n.GPUs[i].ComputeCapability = "7.0"
	}
	a, _ := setup(t, testModels(), n)
	var nodes []NodeView
	call(t, a, "GET", "/api/v1/nodes", "", &nodes)
	if len(nodes) != 1 || nodes[0].EngineVersions["vllm"] != "0.10.2" ||
		len(nodes[0].ComputeCapabilities) != 1 || nodes[0].ComputeCapabilities[0] != "7.0" {
		t.Errorf("%+v", nodes)
	}
}

func TestANodeSaysWhetherItsFabricsAreUp(t *testing.T) {
	n := gpuNode("ib1", 2, "ib-a")
	if got := multiNodeReadiness(n); !strings.HasPrefix(got, "unknown") {
		t.Errorf("no port yet: %q", got)
	}
	n.IBPorts = []inventory.IBPort{{HCA: "mlx5_0", State: "DOWN"}}
	if got := multiNodeReadiness(n); got != "down: mlx5_0" {
		t.Errorf("down: %q", got)
	}
	n.IBPorts[0].State = "ACTIVE"
	if got := multiNodeReadiness(n); got != "ready" {
		t.Errorf("up: %q", got)
	}
	if got := multiNodeReadiness(gpuNode("plain", 1)); got != "" {
		t.Errorf("no fabric: %q", got)
	}
}

// A judge alone past its budget is slow; past half of it, or past it among
// others, tight; and what to do is said, fail: open only where it is closed.
func TestAJudgeCheckIsReadAgainstTheBudget(t *testing.T) {
	closed := JudgeView{Model: "qwen3guard-0.6b", Closed: []string{"baseline"}}
	for _, c := range []struct {
		alone, burst int64
		failed       int
		want         string
	}{{1200, 2400, 0, "ok"}, {2600, 3000, 0, "tight"}, {1200, 6000, 0, "tight"}, {1200, 2000, 1, "tight"}, {5200, 9000, 0, "slow"}} {
		v := JudgeCheckView{Model: closed.Model, BudgetMs: 5000, SlowestMs: c.alone, BurstSlowestMs: c.burst, BurstFailed: c.failed}
		got, advice := judgeVerdict(v, closed)
		if got != c.want || (got == "ok") != (advice == nil) {
			t.Errorf("%+v: %s %v", c, got, advice)
		}
		if got != "ok" && !strings.Contains(strings.Join(advice, " "), "fail: open") {
			t.Errorf("a closed policy is not offered fail: open: %v", advice)
		}
	}
	if _, advice := judgeVerdict(JudgeCheckView{BudgetMs: 5000, SlowestMs: 6000}, JudgeView{}); strings.Contains(strings.Join(advice, " "), "fail: open") {
		t.Error("fail: open offered to a policy already open")
	}
}

func TestAJudgeWithNoReplicaIsNotTimed(t *testing.T) {
	a, _ := setup(t, testModels())
	lib := guard.Config{Policies: map[string]guard.Policy{
		"baseline": {Judge: &guard.JudgePolicy{Model: "nowhere-judge", Input: "block", LatencyBudget: 5 * time.Second}}}}
	a.SetGuard(&GuardAdmin{Library: func() guard.Config { return lib },
		Ask: func(context.Context, string, string, string) (guard.JudgeAnswer, error) {
			return guard.JudgeAnswer{}, nil
		}})
	if code := call(t, a, "POST", "/api/v1/guard/judges/check", `{}`, nil); code != 409 {
		t.Errorf("a judge with no replica: %d", code)
	}
	var judges []JudgeView
	call(t, a, "GET", "/api/v1/guard/judges", "", &judges)
	if len(judges) != 1 || judges[0].BudgetMs != 5000 {
		t.Errorf("judges %+v", judges)
	}
}