Ilker ManapandClaude Opus 5.5
c2be429554
CI / test (push) Successful in 4m14s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
452 lines
14 KiB
Go
452 lines
14 KiB
Go
package cli
|
||
|
||
import (
|
||
"encoding/json"
|
||
"io"
|
||
"net/http"
|
||
"net/http/httptest"
|
||
"os"
|
||
"strings"
|
||
"sync"
|
||
"testing"
|
||
|
||
tea "github.com/charmbracelet/bubbletea"
|
||
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/modelplan"
|
||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||
)
|
||
|
||
type seen struct {
|
||
mu sync.Mutex
|
||
reqs []string
|
||
}
|
||
|
||
func (s *seen) last() string {
|
||
s.mu.Lock()
|
||
defer s.mu.Unlock()
|
||
if len(s.reqs) == 0 {
|
||
return ""
|
||
}
|
||
return s.reqs[len(s.reqs)-1]
|
||
}
|
||
|
||
// adminStub answers every change with 204 and records it as "METHOD path body".
|
||
func adminStub(t *testing.T) (*seen, string) {
|
||
s := &seen{}
|
||
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if r.Method == http.MethodGet {
|
||
_, _ = w.Write([]byte("[]"))
|
||
return
|
||
}
|
||
b, _ := io.ReadAll(r.Body)
|
||
s.mu.Lock()
|
||
s.reqs = append(s.reqs, r.Method+" "+r.URL.Path+" "+strings.TrimSpace(string(b)))
|
||
s.mu.Unlock()
|
||
w.WriteHeader(http.StatusNoContent)
|
||
}))
|
||
t.Cleanup(srv.Close)
|
||
return s, srv.URL
|
||
}
|
||
|
||
// pressRun presses keys and runs, one level deep, whatever the last one
|
||
// returns that is not a tick, the way the program would.
|
||
func pressRun(m tea.Model, keys ...string) tea.Model {
|
||
m = press(m, keys[:len(keys)-1]...)
|
||
var msg tea.Msg = tea.KeyMsg{Type: tea.KeyEnter}
|
||
if k := keys[len(keys)-1]; k != "enter" {
|
||
msg = tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune(k)}
|
||
}
|
||
m, cmd := m.Update(msg)
|
||
if cmd != nil {
|
||
if done, ok := cmd().(actionDoneMsg); ok {
|
||
m, _ = m.Update(done)
|
||
}
|
||
}
|
||
return m
|
||
}
|
||
|
||
func consoleWith(t *testing.T, menu int) (consoleModel, *seen) {
|
||
s, url := adminStub(t)
|
||
m := newConsoleModel(url, "/nonexistent/server.yaml")
|
||
m.loaded, m.menu, m.inPanel = true, menu, true
|
||
m.data.models = []adminapi.ModelView{{Name: "qwen", Min: 1, Max: 1, Class: "standard", Profiles: []string{"cpu"}}}
|
||
m.data.nodes = []adminapi.NodeView{{ID: "n1", State: "ready"}}
|
||
m.data.replicas = []adminapi.ReplicaView{{ID: "qwen-1", Model: "qwen"}}
|
||
return m, s
|
||
}
|
||
|
||
func TestAModelsReplicasAreSetFromItsRow(t *testing.T) {
|
||
m, s := consoleWith(t, cModels)
|
||
var tm tea.Model = m
|
||
tm = press(tm, "enter")
|
||
if !strings.Contains(tm.View(), "Replicas (scale)") {
|
||
t.Fatalf("no action menu:\n%s", tm.View())
|
||
}
|
||
tm = press(tm, "down", "enter")
|
||
f := tm.(consoleModel).formD
|
||
if f == nil {
|
||
t.Fatal("no form")
|
||
}
|
||
f.fields[1].text.SetValue("3")
|
||
// A max below the min is refused in the form, and nothing is sent.
|
||
f.fields[0].text.SetValue("5")
|
||
tm = press(tm, "tab", "tab", "tab", "enter")
|
||
if f.err == "" || s.last() != "" {
|
||
t.Fatalf("a max below the min went through: %q %q", f.err, s.last())
|
||
}
|
||
f.fields[0].text.SetValue("2")
|
||
tm = pressRun(tm, "enter")
|
||
var body adminapi.ScaleRequest
|
||
got := s.last()
|
||
if !strings.HasPrefix(got, "POST /api/v1/models/qwen/scale ") ||
|
||
json.Unmarshal([]byte(strings.SplitN(got, " ", 3)[2]), &body) != nil || body.Min != 2 || body.Max != 3 {
|
||
t.Fatalf("sent %q", got)
|
||
}
|
||
if !strings.Contains(tm.View(), "keeps 2 to 3") {
|
||
t.Errorf("the outcome was not shown:\n%s", tm.View())
|
||
}
|
||
}
|
||
|
||
func TestANodeIsCordonedAndAModelRemovedOnlyWhenConfirmed(t *testing.T) {
|
||
m, s := consoleWith(t, cNodes)
|
||
var tm tea.Model = m
|
||
tm = pressRun(tm, "enter", "down", "enter")
|
||
if got := s.last(); got != `POST /api/v1/nodes/n1/state {"state":"cordoned"}` {
|
||
t.Fatalf("sent %q", got)
|
||
}
|
||
|
||
m, s = consoleWith(t, cModels)
|
||
tm = m
|
||
tm = press(tm, "enter")
|
||
n := len(tm.(consoleModel).menuD.items)
|
||
keys := []string{}
|
||
for range n - 1 {
|
||
keys = append(keys, "down")
|
||
}
|
||
tm = press(tm, append(keys, "enter")...)
|
||
if tm.(consoleModel).confirmD == nil {
|
||
t.Fatal("removing did not ask first")
|
||
}
|
||
tm = press(tm, "n")
|
||
if s.last() != "" {
|
||
t.Fatalf("removed though the answer was no: %q", s.last())
|
||
}
|
||
tm = press(tm, "enter")
|
||
tm = press(tm, append(keys, "enter")...)
|
||
_ = pressRun(tm, "y")
|
||
if got := s.last(); got != "DELETE /api/v1/models/qwen " {
|
||
t.Fatalf("sent %q", got)
|
||
}
|
||
}
|
||
|
||
func TestAQuotaIsAddedWithOnlyWhatIsGiven(t *testing.T) {
|
||
m, s := consoleWith(t, cQuotas)
|
||
var tm tea.Model = press(m, "a")
|
||
f := tm.(consoleModel).formD
|
||
f.fields[0].text.SetValue("team:acme/ml")
|
||
f.fields[4].text.SetValue("2")
|
||
for range len(f.fields) {
|
||
tm = press(tm, "tab")
|
||
}
|
||
_ = pressRun(tm, "enter")
|
||
if got := s.last(); got != `PUT /api/v1/quotas {"scope":"team:acme/ml","guarantee":{},"limit":{"gpus":2}}` {
|
||
t.Fatalf("sent %q", got)
|
||
}
|
||
}
|
||
|
||
func TestPeopleAreManagedFromTheConsoleAgainstItsStore(t *testing.T) {
|
||
dir := t.TempDir()
|
||
cfgPath := dir + "/server.yaml"
|
||
yaml := "auth:\n store: " + dir + "/llmgw.db\n"
|
||
if err := os.WriteFile(cfgPath, []byte(yaml), 0o600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
if _, err := config.Load(cfgPath); err != nil {
|
||
t.Fatalf("the test's configuration: %v", err)
|
||
}
|
||
m := newConsoleModel("127.0.0.1:1", cfgPath)
|
||
defer m.st.close()
|
||
m.loaded, m.inPanel = true, true
|
||
|
||
refresh := func(tm tea.Model) tea.Model {
|
||
cm := tm.(consoleModel)
|
||
msg := cm.fetchPeople()()
|
||
tm, _ = tm.Update(msg)
|
||
return tm
|
||
}
|
||
submit := func(tm tea.Model, vals ...string) tea.Model {
|
||
f := tm.(consoleModel).formD
|
||
if f == nil {
|
||
t.Fatal("no form")
|
||
}
|
||
for i, v := range vals {
|
||
f.fields[i].text.SetValue(v)
|
||
}
|
||
f.focus = len(f.fields)
|
||
tm = pressRun(tm, "enter")
|
||
if v := tm.View(); !strings.Contains(v, "✓") {
|
||
t.Fatalf("did not work:\n%s", v)
|
||
}
|
||
return refresh(press(tm, "x"))
|
||
}
|
||
|
||
m.menu = cOrgs
|
||
var tm tea.Model = m
|
||
tm = submit(press(tm, "a"), "Acme")
|
||
// Into the organization: a team, then a project in it.
|
||
tm = submit(press(tm, "enter", "enter"), "ml")
|
||
cm := tm.(consoleModel)
|
||
if len(cm.people.orgs) != 2 || cm.people.orgs[1].path != "Acme/ml" {
|
||
t.Fatalf("tree: %+v", cm.people.orgs)
|
||
}
|
||
tm = submit(press(tm, "down", "enter", "enter"), "training")
|
||
|
||
cm = tm.(consoleModel)
|
||
cm.menu, cm.row = cAccounts, 0
|
||
tm = submit(press(cm, "a"), "ayse@example.com", "a-long-s3cret-pass", "Acme/ml/training")
|
||
cm = tm.(consoleModel)
|
||
if len(cm.people.users) != 1 || cm.people.users[0].State != store.UserActive ||
|
||
!strings.Contains(strings.Join(cm.people.where[cm.people.users[0].ID], ","), "training") {
|
||
t.Fatalf("account: %+v %v", cm.people.users, cm.people.where)
|
||
}
|
||
|
||
cm.menu, cm.row = cKeys, 0
|
||
tm = press(cm, "a")
|
||
f := tm.(consoleModel).formD
|
||
f.fields[0].text.SetValue("ci")
|
||
f.fields[1].text.SetValue("Acme/ml/training")
|
||
f.focus = len(f.fields)
|
||
tm = pressRun(tm, "enter")
|
||
if v := tm.View(); !strings.Contains(v, "sk-") {
|
||
t.Fatalf("the new key was not shown:\n%s", v)
|
||
}
|
||
tm = refresh(press(tm, "x"))
|
||
if n := len(tm.(consoleModel).people.keys); n != 1 {
|
||
t.Fatalf("%d keys", n)
|
||
}
|
||
// Limits on the project, read back in its row.
|
||
cm = tm.(consoleModel)
|
||
cm.menu, cm.row = cOrgs, 2
|
||
tm = press(cm, "enter")
|
||
items := tm.(consoleModel).menuD.items
|
||
i := 0
|
||
for ; items[i].label != "Limits…"; i++ {
|
||
}
|
||
keys := []string{}
|
||
for range i {
|
||
keys = append(keys, "down")
|
||
}
|
||
tm = submit(press(tm, append(keys, "enter")...), "60")
|
||
if c := limitsCell(tm.(consoleModel).people.limits[store.LimitOwner{Scope: store.ScopeProject,
|
||
ID: tm.(consoleModel).people.orgs[2].id}.CounterKey()]); c != "60 rpm" {
|
||
t.Fatalf("limits: %q", c)
|
||
}
|
||
}
|
||
|
||
func TestTheSidebarFitsAShortScreen(t *testing.T) {
|
||
m, _ := consoleWith(t, cOrgs)
|
||
m.inPanel = false
|
||
for _, h := range []int{20, 30, 50} {
|
||
m.height = h
|
||
if n := len(strings.Split(m.View(), "\n")); n > h {
|
||
t.Errorf("height %d: %d lines drawn", h, n)
|
||
}
|
||
if !strings.Contains(m.View(), "Organizations") {
|
||
t.Errorf("height %d: the current entry is not shown", h)
|
||
}
|
||
}
|
||
}
|
||
|
||
func TestAModelIsFoundOnTheRegistryAndServedFromTheConsole(t *testing.T) {
|
||
reg := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
switch {
|
||
case r.URL.Path == "/api/models":
|
||
if r.URL.Query().Get("search") != "qwen" || r.URL.Query().Get("filter") != "gguf" {
|
||
_, _ = w.Write([]byte(`[]`))
|
||
return
|
||
}
|
||
_, _ = w.Write([]byte(`[{"id":"bartowski/Qwen2.5-3B-Instruct-GGUF","downloads":1200,"gguf":{"total":3000000000,"architecture":"qwen2"}}]`))
|
||
case r.URL.Path == "/api/models/bartowski/Qwen2.5-3B-Instruct-GGUF":
|
||
_, _ = w.Write([]byte(`{"id":"bartowski/Qwen2.5-3B-Instruct-GGUF","sha":"abc"}`))
|
||
case strings.HasPrefix(r.URL.Path, "/api/models/bartowski/Qwen2.5-3B-Instruct-GGUF/tree/"):
|
||
_, _ = w.Write([]byte(`[{"type":"file","path":"Qwen2.5-3B-Instruct-Q4_K_M.gguf","lfs":{"oid":"x","size":2000000000}},
|
||
{"type":"file","path":"Qwen2.5-3B-Instruct-Q8_0.gguf","lfs":{"oid":"y","size":3000000000}},
|
||
{"type":"file","path":"mmproj-f16.gguf","size":10}]`))
|
||
default:
|
||
http.NotFound(w, r)
|
||
}
|
||
}))
|
||
defer reg.Close()
|
||
dir := t.TempDir()
|
||
cfgPath := dir + "/server.yaml"
|
||
if err := os.WriteFile(cfgPath, []byte("mirror:\n api: http://127.0.0.1:1\n registry: "+reg.URL+"\n"), 0o600); err != nil {
|
||
t.Fatal(err)
|
||
}
|
||
// The control plane's plan of the repository's files.
|
||
plan := modelserve.Plan{Repo: "bartowski/Qwen2.5-3B-Instruct-GGUF", Revision: "abc", Format: "gguf", Context: 16384,
|
||
Servable: true, Artifacts: []modelserve.PlannedArtifact{
|
||
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q4_K_M.gguf", Format: "gguf", Bytes: 2 << 30}},
|
||
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-Q8_0.gguf", Format: "gguf", Bytes: 3 << 30}},
|
||
{Artifact: modelplan.Artifact{Name: "Qwen2.5-3B-Instruct-F16.gguf", Format: "gguf", Bytes: 6 << 30}},
|
||
}}
|
||
gpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", Class: "6000-24g", GPUsPerNode: 1, Nodes: 1,
|
||
NeedGB: 5, HaveGB: 21.6, Hosts: 1, FreeNow: true}
|
||
cpu := modelplan.Option{Format: "gguf", Engine: "llamacpp", CPU: true, NeedGB: 5, HaveGB: 27, Hosts: 2, FreeNow: true}
|
||
for _, o := range []struct {
|
||
art string
|
||
o modelplan.Option
|
||
}{{"Qwen2.5-3B-Instruct-Q8_0.gguf", gpu}, {"Qwen2.5-3B-Instruct-Q4_K_M.gguf", gpu},
|
||
{"Qwen2.5-3B-Instruct-Q8_0.gguf", cpu}, {"Qwen2.5-3B-Instruct-F16.gguf", cpu}} {
|
||
o.o.Artifact = o.art
|
||
plan.Options = append(plan.Options, o.o)
|
||
}
|
||
for i := range plan.Artifacts {
|
||
for j := range plan.Options {
|
||
if plan.Options[j].Artifact == plan.Artifacts[i].Name {
|
||
plan.Artifacts[i].Best = &plan.Options[j]
|
||
break
|
||
}
|
||
}
|
||
}
|
||
admin := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||
if r.URL.Path == "/api/v1/plan" && r.URL.Query().Get("repo") == plan.Repo {
|
||
_ = json.NewEncoder(w).Encode(plan)
|
||
return
|
||
}
|
||
_, _ = w.Write([]byte("[]"))
|
||
}))
|
||
defer admin.Close()
|
||
m := newConsoleModel(admin.URL, cfgPath)
|
||
m.loaded, m.menu, m.inPanel = true, cServe, true
|
||
var tm tea.Model = m
|
||
|
||
run := func(tm tea.Model, cmd tea.Cmd) tea.Model {
|
||
if cmd != nil {
|
||
tm, _ = tm.Update(cmd())
|
||
}
|
||
return tm
|
||
}
|
||
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("/")})
|
||
tm = press(tm, "q", "w", "e", "n")
|
||
tm, cmd := tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||
tm = run(tm, cmd)
|
||
if v := tm.View(); !strings.Contains(v, "bartowski/Qwen2.5-3B-Instruct-GGUF") || !strings.Contains(v, "3.0B") {
|
||
t.Fatalf("no result:\n%s", v)
|
||
}
|
||
tm, cmd = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||
tm = run(tm, cmd)
|
||
d := tm.(consoleModel).menuD
|
||
if d == nil || len(d.items) != 3 || !strings.Contains(d.items[0].label, "★") ||
|
||
!strings.Contains(d.items[0].label, "Q8_0") || !strings.Contains(d.items[0].label, "1 × 6000-24g") ||
|
||
!strings.Contains(d.items[2].label, "CPU") {
|
||
t.Fatalf("files offered: %+v", d)
|
||
}
|
||
tm = press(tm, "enter")
|
||
f := tm.(consoleModel).formD
|
||
if f == nil || f.fields[0].value() != "qwen2.5-3b-instruct" || len(f.fields[4].options) != 2 {
|
||
t.Fatalf("form: %+v", f)
|
||
}
|
||
f.focus = len(f.fields)
|
||
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyEnter})
|
||
act := tm.(consoleModel).action
|
||
got := strings.Join(act.targets, " ")
|
||
if act.kind != "run" || !strings.Contains(got, "model serve bartowski/Qwen2.5-3B-Instruct-GGUF --name=qwen2.5-3b-instruct --replicas=1") ||
|
||
!strings.Contains(got, "--file=Qwen2.5-3B-Instruct-Q8_0.gguf --gpu-class=6000-24g --gpus=1 --nodes=1 --fallbacks") {
|
||
t.Fatalf("would run %q %q", act.kind, got)
|
||
}
|
||
}
|
||
|
||
func TestTheManualHasEveryPageAndFindsInThem(t *testing.T) {
|
||
v := newDocViewer()
|
||
for _, k := range []string{"console", "install", "reference", "commands", "changes"} {
|
||
if findTopic(v.topics, k) < 0 {
|
||
t.Errorf("no %s page", k)
|
||
}
|
||
}
|
||
cmds := commandsManual()
|
||
for _, c := range []string{"llmgw ctl model update", "llmgw users approve", "llmgw docs"} {
|
||
if !strings.Contains(cmds, "# "+c) {
|
||
t.Errorf("the commands page lacks %s", c)
|
||
}
|
||
}
|
||
// Every line fits the width it was laid out for.
|
||
for _, tp := range v.topics {
|
||
for _, l := range renderMarkdown(tp.src(), 80) {
|
||
if !strings.HasPrefix(l.plain, " ") && len([]rune(l.plain)) > 80 {
|
||
t.Errorf("%s: a line of %d: %q", tp.key, len([]rune(l.plain)), l.plain)
|
||
break
|
||
}
|
||
}
|
||
}
|
||
v.show(findTopic(v.topics, "console"))
|
||
v.query = "drain"
|
||
v.find(80, 0, 1)
|
||
if l := v.lines(80)[v.scroll+2]; !strings.Contains(strings.ToLower(l.plain), "drain") {
|
||
t.Errorf("found at %q", l.plain)
|
||
}
|
||
v.openTOC(80)
|
||
if v.toc == nil || len(v.toc.items) < 5 {
|
||
t.Fatal("no contents")
|
||
}
|
||
}
|
||
|
||
func TestQuestionMarkOpensTheConsolesPage(t *testing.T) {
|
||
m, _ := consoleWith(t, cNodes)
|
||
tm := press(m, "?")
|
||
cm := tm.(consoleModel)
|
||
if cm.menu != cManual || !cm.detail || cm.docs.topics[cm.docs.open].key != "console" {
|
||
t.Fatalf("menu %d detail %v", cm.menu, cm.detail)
|
||
}
|
||
if !strings.Contains(tm.View(), "The console") {
|
||
t.Error("the page is not shown")
|
||
}
|
||
}
|
||
|
||
func TestNextFindsTheNextMatchAndComesRound(t *testing.T) {
|
||
v := newDocViewer()
|
||
v.show(findTopic(v.topics, "console"))
|
||
v.query = "revok"
|
||
lines := v.lines(80)
|
||
var hits []int
|
||
for i, l := range lines {
|
||
if strings.Contains(strings.ToLower(l.plain), "revok") {
|
||
hits = append(hits, i)
|
||
}
|
||
}
|
||
if len(hits) < 2 {
|
||
t.Fatalf("hits %v", hits)
|
||
}
|
||
v.find(80, 0, 1)
|
||
seen := []int{v.scroll + 2}
|
||
for range hits {
|
||
v.key(tea.KeyMsg{Type: tea.KeyRunes, Runes: []rune("n")}, 80, 30)
|
||
seen = append(seen, v.scroll+2)
|
||
}
|
||
if seen[0] != hits[0] || seen[1] != hits[1] || seen[len(hits)] != hits[0] {
|
||
t.Errorf("hits %v, went %v", hits, seen)
|
||
}
|
||
}
|
||
|
||
func TestTabAndArrowsGoRoundAFormFromApplyWithoutFalling(t *testing.T) {
|
||
m, _ := consoleWith(t, cQuotas)
|
||
var tm tea.Model = press(m, "a")
|
||
n := len(tm.(consoleModel).formD.fields)
|
||
// Round the form twice each way, through Apply.
|
||
for range 2 * (n + 1) {
|
||
tm = press(tm, "tab")
|
||
}
|
||
for range 2 * (n + 1) {
|
||
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyUp})
|
||
tm, _ = tm.Update(tea.KeyMsg{Type: tea.KeyShiftTab})
|
||
}
|
||
for range 2 * (n + 1) {
|
||
tm = press(tm, "down")
|
||
}
|
||
if f := tm.(consoleModel).formD; f == nil || f.focus < 0 || f.focus > n {
|
||
t.Fatalf("focus %+v", f)
|
||
}
|
||
}
|