A page for anyone in a project, built for the one who knows least: an Excel, CSV or JSONL file of questions and answers, previewed; a model; a name; who may use it; and a button. The weights to train on are found from where the model's came (a GGUF repository's base model), the card's memory from their size, the node as the one with the most free. When it is done the adapter is published by itself (a llama.cpp model's replicas replaced to load it) and is a model of its own in the chat and the API. The project's by default; its members share it wider (confirmed, and never wider than the knowledge bases its pairs came from) or delete it. One trains at a time per project. ctl train start needs only --model and --data now, and --publish publishes when done. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
138 lines
5.1 KiB
Go
138 lines
5.1 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
type fakeRegistry struct {
|
|
repos map[string]hf.Repo
|
|
asked int
|
|
}
|
|
|
|
func (f *fakeRegistry) RepoAt(_ context.Context, id, _ string) (hf.Repo, error) {
|
|
f.asked++
|
|
r, ok := f.repos[id]
|
|
if !ok {
|
|
return hf.Repo{}, errors.New("404")
|
|
}
|
|
return r, nil
|
|
}
|
|
|
|
func TestAPortalTrainingFindsItsWeightsAndMemoryAndOneTrainsPerProject(t *testing.T) {
|
|
ctx := context.Background()
|
|
models := append(testModels(),
|
|
catalog.Model{Name: "small", Source: "/s", Origin: &catalog.Origin{Repo: "bartowski/Small-GGUF"},
|
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
|
|
catalog.Model{Name: "gated", Source: "/g", Origin: &catalog.Origin{Repo: "meta/Gated"},
|
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
|
|
catalog.Model{Name: "local", Source: "/l", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
|
|
a, _ := setup(t, models, gpuNode("cuda01", 1))
|
|
s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
reg := &fakeRegistry{repos: map[string]hf.Repo{
|
|
"bartowski/Small-GGUF": {ID: "bartowski/Small-GGUF", Revision: "g1", BaseModel: []string{"Org/Small"},
|
|
Files: []hf.File{{Path: "small-Q4_K_M.gguf", Bytes: 400 << 20}}},
|
|
"Org/Small": {ID: "Org/Small", Revision: "abc", Files: []hf.File{{Path: "model.safetensors", Bytes: 1000 << 20},
|
|
{Path: "config.json", Bytes: 1 << 10}}},
|
|
"meta/Gated": {ID: "meta/Gated", Gated: hf.Gated{Reason: "manual"},
|
|
Files: []hf.File{{Path: "model.safetensors", Bytes: 1 << 30}}},
|
|
}}
|
|
a.SetTrain(&TrainAdmin{Store: s, Service: train.New(s, noAgents{}, nil)})
|
|
a.SetTrainMirror(nil, reg)
|
|
|
|
if got := a.TrainableModels(); strings.Join(got, ",") != "gated,small" {
|
|
t.Errorf("trainable: %v", got)
|
|
}
|
|
data := train.NDJSON([]train.Pair{{Q: "a", A: "b"}})
|
|
req := TrainRequest{Name: "tone", Model: "small", Data: data, Owner: "project:p1", Project: "p1", Publish: true}
|
|
j, err := a.StartTraining(ctx, req, "ayse@x")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if j.BaseRepo != "Org/Small" || j.BaseRevision != "abc" || j.Node != "" || j.GPUMemoryMiB != TrainMemoryMiB(1000<<20) ||
|
|
!j.Publish || j.Project != "p1" || j.CreatedBy != "ayse@x" || !j.GGUF {
|
|
t.Errorf("queued %+v", j)
|
|
}
|
|
if j.GPUMemoryMiB != 3072 {
|
|
t.Errorf("1000 MiB of weights take %d MiB to train", j.GPUMemoryMiB)
|
|
}
|
|
|
|
req.Name = "tone2"
|
|
if _, err := a.StartTraining(ctx, req, "mehmet@x"); err == nil || !strings.Contains(err.Error(), "already has a model training") {
|
|
t.Errorf("a second in the project: %v", err)
|
|
}
|
|
req.Name, req.Project, req.Owner = "tone", "p2", "project:p2"
|
|
if _, err := a.StartTraining(ctx, req, "x"); err == nil || !strings.Contains(err.Error(), "already being trained") {
|
|
t.Errorf("the same name twice: %v", err)
|
|
}
|
|
req.Name = "tone3"
|
|
asked := reg.asked
|
|
if _, err := a.StartTraining(ctx, req, "x"); err != nil {
|
|
t.Errorf("another project: %v", err)
|
|
}
|
|
if reg.asked != asked {
|
|
t.Errorf("the registry was asked again: %d", reg.asked-asked)
|
|
}
|
|
for model, says := range map[string]string{"gated": "gated", "local": "did not come from the model registry"} {
|
|
req.Model, req.Name, req.Project = model, "x-"+model, "p-"+model
|
|
_, err := a.StartTraining(ctx, req, "x")
|
|
if err == nil || !strings.Contains(err.Error(), says) {
|
|
t.Errorf("%s: %v", model, err)
|
|
}
|
|
if msg, theirs := TrainRefusal(err); !theirs || msg == "" {
|
|
t.Errorf("%s is not the request's fault: %q", model, msg)
|
|
}
|
|
}
|
|
|
|
jobs, err := a.Trainings(ctx, []string{"p1"}, "nobody")
|
|
if err != nil || len(jobs) != 1 || jobs[0].Name != "tone" {
|
|
t.Errorf("p1's: %+v %v", jobs, err)
|
|
}
|
|
if err := a.ShareTraining(ctx, j.ID, "", "ayse@x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if got, _ := a.Training(ctx, j.ID); got.Owner != "" {
|
|
t.Errorf("shared with everyone: %q", got.Owner)
|
|
}
|
|
if err := a.DeleteTraining(ctx, j.ID, "ayse@x"); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if _, err := a.Training(ctx, j.ID); !errors.Is(err, store.ErrNotFound) {
|
|
t.Errorf("deleted: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestATrainingGoesToTheNodeWithTheMostFree(t *testing.T) {
|
|
busy := gpuNode("busy", 1)
|
|
busy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 23000}}}
|
|
roomy := gpuNode("roomy", 1)
|
|
roomy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 4000}}}
|
|
cpu := inventory.Node{ID: "cpu", State: inventory.Ready}
|
|
small := gpuNode("small", 1)
|
|
small.GPUs = []discovery.GPU{{Index: 0, MemoryTotalMB: 8000}}
|
|
a, _ := setup(t, testModels(), busy, roomy, cpu, small)
|
|
if got := a.PickTrainNode(3000, nil); got != "roomy" {
|
|
t.Errorf("picked %q", got)
|
|
}
|
|
if got := a.PickTrainNode(3000, map[string]bool{"roomy": true}); got != "small" {
|
|
t.Errorf("with roomy busy, picked %q", got)
|
|
}
|
|
if got := a.PickTrainNode(30000, nil); got != "" {
|
|
t.Errorf("none has 30 GB, picked %q", got)
|
|
}
|
|
}
|