package adminapi import ( "context" "errors" "path/filepath" "strings" "testing" "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) type fakeRegistry struct { repos map[string]hf.Repo asked int } func (f *fakeRegistry) RepoAt(_ context.Context, id, _ string) (hf.Repo, error) { f.asked++ r, ok := f.repos[id] if !ok { return hf.Repo{}, errors.New("404") } return r, nil } func TestAPortalTrainingFindsItsWeightsAndMemoryAndOneTrainsPerProject(t *testing.T) { ctx := context.Background() models := append(testModels(), catalog.Model{Name: "small", Source: "/s", Origin: &catalog.Origin{Repo: "bartowski/Small-GGUF"}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}, catalog.Model{Name: "gated", Source: "/g", Origin: &catalog.Origin{Repo: "meta/Gated"}, Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}, catalog.Model{Name: "local", Source: "/l", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}) a, _ := setup(t, models, gpuNode("cuda01", 1)) s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db")) if err != nil { t.Fatal(err) } defer s.Close() reg := &fakeRegistry{repos: map[string]hf.Repo{ "bartowski/Small-GGUF": {ID: "bartowski/Small-GGUF", Revision: "g1", BaseModel: []string{"Org/Small"}, Files: []hf.File{{Path: "small-Q4_K_M.gguf", Bytes: 400 << 20}}}, "Org/Small": {ID: "Org/Small", Revision: "abc", Files: []hf.File{{Path: "model.safetensors", Bytes: 1000 << 20}, {Path: "config.json", Bytes: 1 << 10}}}, "meta/Gated": {ID: "meta/Gated", Gated: hf.Gated{Reason: "manual"}, Files: []hf.File{{Path: "model.safetensors", Bytes: 1 << 30}}}, }} a.SetTrain(&TrainAdmin{Store: s, Service: train.New(s, noAgents{}, nil)}) a.SetTrainMirror(nil, reg) if got := a.TrainableModels(); strings.Join(got, ",") != "gated,small" { t.Errorf("trainable: %v", got) } data := train.NDJSON([]train.Pair{{Q: "a", A: "b"}}) req := TrainRequest{Name: "tone", Model: "small", Data: data, Owner: "project:p1", Project: "p1", Publish: true} j, err := a.StartTraining(ctx, req, "ayse@x") if err != nil { t.Fatal(err) } if j.BaseRepo != "Org/Small" || j.BaseRevision != "abc" || j.Node != "" || j.GPUMemoryMiB != TrainMemoryMiB(1000<<20) || !j.Publish || j.Project != "p1" || j.CreatedBy != "ayse@x" || !j.GGUF { t.Errorf("queued %+v", j) } if j.GPUMemoryMiB != 3072 { t.Errorf("1000 MiB of weights take %d MiB to train", j.GPUMemoryMiB) } req.Name = "tone2" if _, err := a.StartTraining(ctx, req, "mehmet@x"); err == nil || !strings.Contains(err.Error(), "already has a model training") { t.Errorf("a second in the project: %v", err) } req.Name, req.Project, req.Owner = "tone", "p2", "project:p2" if _, err := a.StartTraining(ctx, req, "x"); err == nil || !strings.Contains(err.Error(), "already being trained") { t.Errorf("the same name twice: %v", err) } req.Name = "tone3" asked := reg.asked if _, err := a.StartTraining(ctx, req, "x"); err != nil { t.Errorf("another project: %v", err) } if reg.asked != asked { t.Errorf("the registry was asked again: %d", reg.asked-asked) } for model, says := range map[string]string{"gated": "gated", "local": "did not come from the model registry"} { req.Model, req.Name, req.Project = model, "x-"+model, "p-"+model _, err := a.StartTraining(ctx, req, "x") if err == nil || !strings.Contains(err.Error(), says) { t.Errorf("%s: %v", model, err) } if msg, theirs := TrainRefusal(err); !theirs || msg == "" { t.Errorf("%s is not the request's fault: %q", model, msg) } } jobs, err := a.Trainings(ctx, []string{"p1"}, "nobody") if err != nil || len(jobs) != 1 || jobs[0].Name != "tone" { t.Errorf("p1's: %+v %v", jobs, err) } if err := a.ShareTraining(ctx, j.ID, "", "ayse@x"); err != nil { t.Fatal(err) } if got, _ := a.Training(ctx, j.ID); got.Owner != "" { t.Errorf("shared with everyone: %q", got.Owner) } if err := a.DeleteTraining(ctx, j.ID, "ayse@x"); err != nil { t.Fatal(err) } if _, err := a.Training(ctx, j.ID); !errors.Is(err, store.ErrNotFound) { t.Errorf("deleted: %v", err) } } func TestATrainingGoesToTheNodeWithTheMostFree(t *testing.T) { busy := gpuNode("busy", 1) busy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 23000}}} roomy := gpuNode("roomy", 1) roomy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 4000}}} cpu := inventory.Node{ID: "cpu", State: inventory.Ready} small := gpuNode("small", 1) small.GPUs = []discovery.GPU{{Index: 0, MemoryTotalMB: 8000}} a, _ := setup(t, testModels(), busy, roomy, cpu, small) if got := a.PickTrainNode(3000, nil); got != "roomy" { t.Errorf("picked %q", got) } if got := a.PickTrainNode(3000, map[string]bool{"roomy": true}); got != "small" { t.Errorf("with roomy busy, picked %q", got) } if got := a.PickTrainNode(30000, nil); got != "" { t.Errorf("none has 30 GB, picked %q", got) } }