Customise a model from the portal: a file, a model, a button

A page for anyone in a project, built for the one who knows least: an
Excel, CSV or JSONL file of questions and answers, previewed; a model; a
name; who may use it; and a button. The weights to train on are found
from where the model's came (a GGUF repository's base model), the card's
memory from their size, the node as the one with the most free. When it
is done the adapter is published by itself (a llama.cpp model's replicas
replaced to load it) and is a model of its own in the chat and the API.

The project's by default; its members share it wider (confirmed, and
never wider than the knowledge bases its pairs came from) or delete it.
One trains at a time per project. ctl train start needs only --model and
--data now, and --publish publishes when done.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-11-19 11:31:12 +01:00
1 parent fcda4d3f85
commit a53b2f4be9
33 files changed
+2199 -134

No files matched your search

+347 -72
View File
@@ -10,15 +10,20 @@ import (
"errors"
"fmt"
"io"
"log/slog"
"net/http"
"regexp"
"slices"
"strconv"
"strings"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
@@ -36,6 +41,11 @@ type TrainStore interface {
TrainArtifact(ctx context.Context, job, kind string) ([]byte, error)
TrainArtifactKinds(ctx context.Context, job string) ([]string, error)
DeleteTrainJob(ctx context.Context, id string) error
QueuedTrainJobs(ctx context.Context) ([]store.TrainJob, error)
ActiveTrainJobs(ctx context.Context) ([]store.TrainJob, error)
TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]store.TrainJob, error)
SetTrainOwner(ctx context.Context, id, owner string) error
SetTrainNote(ctx context.Context, id, message string) error
}
// TrainAdmin is what the training endpoints need.
@@ -45,6 +55,27 @@ type TrainAdmin struct {
// Mirror keeps what a job made where the nodes fetch weights; nil
// leaves publishing off.
Mirror TrainMirror
// Registry finds the weights to train a catalog model on, from where
// its own came; nil has every job name them.
Registry TrainRegistry
// Log says what publishing by itself could not do.
Log *slog.Logger
mu sync.Mutex
bases map[string]trainBaseInfo // by the model's origin repository
}
// TrainRegistry reads a repository of the model registry.
type TrainRegistry interface {
RepoAt(ctx context.Context, id, rev string) (hf.Repo, error)
}
// trainBaseInfo is the safetensors weights a model is trained on.
type trainBaseInfo struct {
repo, revision string
bytes int64
err error
at time.Time
}
// TrainMirror is the model mirror, as publishing needs it.
@@ -66,11 +97,11 @@ type TrainPublishView struct {
// SetTrain enables the training endpoints.
func (a *API) SetTrain(t *TrainAdmin) { a.train = t }
// SetTrainMirror gives publishing the mirror, which is set up after the
// training endpoints are.
func (a *API) SetTrainMirror(m TrainMirror) {
// SetTrainMirror gives publishing the mirror, and finding a model's
// weights the registry, which are set up after the training endpoints are.
func (a *API) SetTrainMirror(m TrainMirror, reg TrainRegistry) {
if a.train != nil {
a.train.Mirror = m
a.train.Mirror, a.train.Registry = m, reg
}
}
@@ -81,13 +112,23 @@ type TrainRequest struct {
// Model is the catalog model the adapter is for.
Model string `json:"model"`
// BaseRepo and BaseRevision are the weights trained on: the model's own
// on its registry, in safetensors (Hugging Face), not a GGUF.
BaseRepo string `json:"base_repo"`
// on its registry, in safetensors (Hugging Face), not a GGUF. Empty
// finds them from where the model's came: the repository itself, or the
// one its card says it was made from.
BaseRepo string `json:"base_repo,omitempty"`
BaseRevision string `json:"base_revision,omitempty"`
// Node trains it; GPUMemoryMiB is what it needs of a card, which can be
// what an engine there leaves free.
Node string `json:"node"`
GPUMemoryMiB int `json:"gpu_memory_mib"`
// Node trains it; empty is the first with a card that has enough free.
// GPUMemoryMiB is what it needs of a card, which can be what an engine
// there leaves free; 0 is worked out from the weights' size.
Node string `json:"node,omitempty"`
GPUMemoryMiB int `json:"gpu_memory_mib,omitempty"`
// Project is the project the job is started in: its members see and
// manage it, and only one of its jobs trains at a time.
Project string `json:"project,omitempty"`
// Publish puts what the job makes on the mirror and makes it the
// model's adapter by itself when the job is done, restarting a llama.cpp
// model's replicas to load it.
Publish bool `json:"publish,omitempty"`
// Owner is who the adapter will belong to: org:, team:, project: (by
// path or ID), user: or key:. Only callers there see it. When the
// pairs come from one knowledge base, it defaults to that collection's
@@ -129,6 +170,8 @@ type TrainJobView struct {
StartedAt time.Time `json:"started_at,omitzero"`
FinishedAt time.Time `json:"finished_at,omitzero"`
CreatedBy string `json:"created_by,omitempty"`
Project string `json:"project,omitempty"`
Publish bool `json:"publish,omitempty"`
// Artifacts are what it made: peft (a gzipped tar), gguf.
Artifacts []string `json:"artifacts,omitempty"`
}
@@ -137,7 +180,8 @@ func trainView(j store.TrainJob) TrainJobView {
return TrainJobView{ID: j.ID, Name: j.Name, Model: j.Model, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision,
Node: j.Node, Owner: j.Owner, GPUMemoryMiB: j.GPUMemoryMiB, Params: j.Params, GGUF: j.GGUF, Pairs: j.Pairs, State: j.State,
Step: j.Step, Steps: j.Steps, Loss: j.Loss, Message: j.Message, PeakMiB: j.PeakMiB, CreatedAt: j.CreatedAt,
StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy}
StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy,
Project: j.Project, Publish: j.Publish}
}
func (a *API) registerTrain(mux Mux) {
@@ -178,50 +222,122 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) {
writeError(w, http.StatusBadRequest, err.Error())
return
}
bad := func(format string, args ...any) {
writeError(w, http.StatusBadRequest, fmt.Sprintf(format, args...))
j, err := a.StartTraining(r.Context(), in, actorOf(r))
if err != nil {
writeTrainErr(w, err)
return
}
writeJSON(w, http.StatusCreated, trainView(j))
}
// trainErr is a refusal, with the status it is answered with.
type trainErr struct {
code int
msg string
}
func (e *trainErr) Error() string { return e.msg }
func refused(code int, format string, args ...any) error {
return &trainErr{code: code, msg: fmt.Sprintf(format, args...)}
}
func writeTrainErr(w http.ResponseWriter, err error) {
var te *trainErr
if errors.As(err, &te) {
writeError(w, te.code, te.msg)
return
}
writeError(w, http.StatusInternalServerError, err.Error())
}
// TrainRefusal is the reason a training request was refused, for someone
// to read, and whether it was the request's fault rather than the
// cluster's.
func TrainRefusal(err error) (string, bool) {
if err == nil {
return "", false
}
var te *trainErr
if errors.As(err, &te) {
return te.msg, te.code < 500
}
return err.Error(), false
}
// StartTraining checks a job and queues it, as actor.
func (a *API) StartTraining(ctx context.Context, in TrainRequest, actor string) (store.TrainJob, error) {
if a.train == nil {
return store.TrainJob{}, refused(http.StatusNotImplemented,
"training needs the store and the nodes' agents, which this control plane runs without")
}
bad := func(format string, args ...any) (store.TrainJob, error) {
return store.TrainJob{}, refused(http.StatusBadRequest, format, args...)
}
if !adapterName.MatchString(in.Name) {
bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name)
return
return bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name)
}
m, ok := a.ctl.Catalog().Resolve(in.Model)
if !ok {
bad("no model %s in the catalog", in.Model)
return
return bad("no model %s in the catalog", in.Model)
}
if _, _, _, taken := a.ctl.Catalog().ResolveAdapter(in.Name); taken || in.Name == m.Name {
bad("%s is already a model's or an adapter's name", in.Name)
return
return bad("%s is already a model's or an adapter's name", in.Name)
}
if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") {
bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo)
return
if _, taken := a.ctl.Catalog().Resolve(in.Name); taken {
return bad("%s is already a model's or an adapter's name", in.Name)
}
if _, ok := a.ctl.Inventory().Get(in.Node); !ok {
bad("no node %s", in.Node)
return
queued, err := a.train.Store.QueuedTrainJobs(ctx)
if err != nil {
return store.TrainJob{}, err
}
active, err := a.train.Store.ActiveTrainJobs(ctx)
if err != nil {
return store.TrainJob{}, err
}
for _, j := range append(queued, active...) {
if j.Name == in.Name {
return bad("%s is already being trained (job %s)", in.Name, j.ID)
}
if in.Project != "" && j.Project == in.Project {
return store.TrainJob{}, refused(http.StatusConflict,
"this project already has a model training (%s): one at a time, so that one project does not hold every card", j.Name)
}
}
var weights int64
if in.BaseRepo == "" {
b := a.trainBase(ctx, m)
if b.err != nil {
return store.TrainJob{}, b.err
}
in.BaseRepo, in.BaseRevision, weights = b.repo, b.revision, b.bytes
} else if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") {
return bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo)
}
if in.Node != "" {
if _, ok := a.ctl.Inventory().Get(in.Node); !ok {
return bad("no node %s", in.Node)
}
}
if in.GPUMemoryMiB <= 0 {
bad("gpu_memory_mib is what the job needs of a card, in MiB")
return
if weights <= 0 {
return bad("gpu_memory_mib is what the job needs of a card, in MiB")
}
in.GPUMemoryMiB = TrainMemoryMiB(weights)
}
pairs, colls, err := countPairs(in.Data)
if err != nil {
bad("%v", err)
return
return bad("%v", err)
}
owner, err := a.trainOwner(r.Context(), in, colls)
owner, err := a.trainOwner(ctx, in, colls)
if err != nil {
bad("%v", err)
return
return bad("%v", err)
}
for k, v := range in.Params {
lim, known := trainParams[k]
f, isNum := v.(float64)
if !known || !isNum || f < lim[0] || f > lim[1] {
bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k)
return
return bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k)
}
}
params := ""
@@ -240,15 +356,132 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) {
}
j := store.TrainJob{ID: store.NewID("train-"), Name: in.Name, Model: m.Name, BaseRepo: in.BaseRepo,
BaseRevision: in.BaseRevision, Node: in.Node, Owner: owner, GPUMemoryMiB: in.GPUMemoryMiB, Params: params, GGUF: gguf,
Data: []byte(in.Data), Pairs: pairs, CreatedBy: actorOf(r)}
if err := a.train.Store.CreateTrainJob(r.Context(), j); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
Data: []byte(in.Data), Pairs: pairs, CreatedBy: actor, Project: in.Project, Publish: in.Publish}
if err := a.train.Store.CreateTrainJob(ctx, j); err != nil {
return store.TrainJob{}, err
}
// At once rather than at the next look, when the node is here.
go a.train.Service.Dispatch(context.WithoutCancel(r.Context()))
go a.train.Service.Dispatch(context.WithoutCancel(ctx))
j.State = store.TrainQueued
writeJSON(w, http.StatusCreated, trainView(j))
return j, nil
}
// TrainMemoryMiB is what training a LoRA adapter on weights of this size
// takes of a card: the weights in 16 bits, as much again for what passes
// through them, and the CUDA context. Measured at 2.3 GB for a 0.5B model.
func TrainMemoryMiB(weightBytes int64) int {
mib := int(weightBytes>>20)*2 + 1024
return (mib + 255) / 256 * 256
}
// trainBaseFor is how long what the registry said of a model's weights is
// kept: it changes when the model is updated, which is rare.
const trainBaseFor = time.Hour
// trainBase is the safetensors weights to train m on: its own origin's,
// when they are safetensors, or those its card says it was made from — a
// GGUF's base model.
func (a *API) trainBase(ctx context.Context, m catalog.Model) trainBaseInfo {
if m.Origin == nil || m.Origin.Repo == "" {
return trainBaseInfo{err: refused(http.StatusBadRequest, "%s's weights did not come from the model registry, so "+
"what to train it on is not known: an administrator can name them (llmgw ctl train start --base-repo)", m.Name)}
}
if a.train.Registry == nil {
return trainBaseInfo{err: refused(http.StatusNotImplemented, "the model registry is not reachable from here "+
"(mirror.registry): the weights to train %s on cannot be found", m.Name)}
}
a.train.mu.Lock()
b, ok := a.train.bases[m.Origin.Repo]
a.train.mu.Unlock()
if ok && time.Since(b.at) < trainBaseFor {
return b
}
b = a.findTrainBase(ctx, m)
if b.err != nil {
var te *trainErr
if !errors.As(b.err, &te) {
return b // the registry did not answer: asked again next time
}
}
b.at = time.Now()
a.train.mu.Lock()
if a.train.bases == nil {
a.train.bases = map[string]trainBaseInfo{}
}
a.train.bases[m.Origin.Repo] = b
a.train.mu.Unlock()
return b
}
func (a *API) findTrainBase(ctx context.Context, m catalog.Model) trainBaseInfo {
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
repo, err := a.train.Registry.RepoAt(ctx, m.Origin.Repo, "")
if err != nil {
return trainBaseInfo{err: fmt.Errorf("asking the model registry about %s: %w", m.Origin.Repo, err)}
}
candidates := []hf.Repo{repo}
for _, id := range repo.BaseModel {
if id == repo.ID || strings.Count(id, "/") != 1 {
continue
}
r, err := a.train.Registry.RepoAt(ctx, id, "")
if err != nil {
continue
}
candidates = append(candidates, r)
}
gated := ""
for _, r := range candidates {
var n int64
for _, f := range r.Files {
if strings.HasSuffix(strings.ToLower(f.Path), ".safetensors") {
n += f.Bytes
}
}
if n == 0 {
continue
}
if r.Gated.Is() {
gated = r.ID
continue
}
return trainBaseInfo{repo: r.ID, revision: r.Revision, bytes: n}
}
if gated != "" {
return trainBaseInfo{err: refused(http.StatusBadRequest, "the weights to train %s on, %s, are gated on the registry: "+
"the nodes cannot fetch them without an account that accepted their terms", m.Name, gated)}
}
return trainBaseInfo{err: refused(http.StatusBadRequest, "neither %s, where %s's weights came from, nor the model its card "+
"names has safetensors weights to train on", m.Origin.Repo, m.Name)}
}
// PickTrainNode is the node whose freest card has the most memory free,
// if that is need, among those ready and not busy.
func (a *API) PickTrainNode(need int, busy map[string]bool) string {
best, bestFree := "", 0
for _, n := range a.ctl.Inventory().Nodes() {
if busy[n.ID] || n.State != inventory.Ready {
continue
}
free := 0
if n.Load != nil {
for _, g := range n.Load.GPUs {
if g.MemoryTotalMB > 0 {
free = max(free, int(g.MemoryTotalMB-g.MemoryUsedMB))
}
}
}
if free == 0 {
for _, g := range n.GPUs {
free = max(free, g.MemoryTotalMB-g.MemoryUsedMB)
}
}
if free >= need && free > bestFree {
best, bestFree = n.ID, free
}
}
return best
}
// trainOwner is who a job's adapter will belong to, as scope:ID: one that
@@ -453,73 +686,74 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) {
if !ok {
return
}
if a.train.Mirror == nil {
writeError(w, http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+
"llmgw ctl train download, then llmgw mirror import there, does it by hand")
v, err := a.publishTrained(r.Context(), j, actorOf(r))
if err != nil {
writeTrainErr(w, err)
return
}
writeJSON(w, http.StatusOK, v)
}
// publishTrained puts what a job made on the mirror and makes it the
// model's adapter: the GGUF for a model llama.cpp serves, the PEFT
// directory for one vLLM serves.
func (a *API) publishTrained(ctx context.Context, j store.TrainJob, actor string) (TrainPublishView, error) {
var none TrainPublishView
if a.train.Mirror == nil {
return none, refused(http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+
"llmgw ctl train download, then llmgw mirror import there, does it by hand")
}
if a.models == nil {
writeError(w, http.StatusNotImplemented, "this control plane has no store for the catalog")
return
return none, refused(http.StatusNotImplemented, "this control plane has no store for the catalog")
}
if j.State != store.TrainDone {
writeError(w, http.StatusConflict, "the job has not finished: it is "+j.State)
return
return none, refused(http.StatusConflict, "%s", "the job has not finished: it is "+j.State)
}
m, found := a.ctl.Catalog().Resolve(j.Model)
if !found {
writeError(w, http.StatusConflict, "the model "+j.Model+" is no longer in the catalog")
return
return none, refused(http.StatusConflict, "%s", "the model "+j.Model+" is no longer in the catalog")
}
for _, ad := range m.Adapters {
if ad.Name == j.Name {
writeError(w, http.StatusConflict, j.Name+" is already "+m.Name+"'s adapter; train under another name")
return
return none, refused(http.StatusConflict, "%s", j.Name+" is already "+m.Name+"'s adapter; train under another name")
}
}
llama := slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" })
source := "trained by llmgw job " + j.ID + " on " + j.Node + " from " + j.BaseRepo
var url string
if llama {
b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "gguf")
b, err := a.train.Store.TrainArtifact(ctx, j.ID, "gguf")
if err != nil {
writeError(w, http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf")
return
return none, refused(http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf")
}
res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, j.Name+".gguf", b, source)
res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, j.Name+".gguf", b, source)
if err != nil {
writeError(w, http.StatusBadGateway, "keeping it on the mirror: "+err.Error())
return
return none, refused(http.StatusBadGateway, "%s", "keeping it on the mirror: "+err.Error())
}
url = res.URL
} else {
b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "peft")
b, err := a.train.Store.TrainArtifact(ctx, j.ID, "peft")
if err != nil {
writeError(w, http.StatusConflict, "the job made no PEFT adapter")
return
return none, refused(http.StatusConflict, "the job made no PEFT adapter")
}
files, err := untarAll(b)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
return none, err
}
for name, data := range files {
res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, name, data, source)
res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, name, data, source)
if err != nil {
writeError(w, http.StatusBadGateway, "keeping "+name+" on the mirror: "+err.Error())
return
return none, refused(http.StatusBadGateway, "%s", "keeping "+name+" on the mirror: "+err.Error())
}
url = res.URL[:strings.LastIndex(res.URL, "/")+1]
}
}
ad := catalog.Adapter{Name: j.Name, Source: url, Owner: j.Owner}
if err := checkAdapters(r.Context(), m, []catalog.Adapter{ad}); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
if err := checkAdapters(ctx, m, []catalog.Adapter{ad}); err != nil {
return none, refused(http.StatusBadRequest, "%v", err)
}
if _, err := a.models.SetAdapters(r.Context(), m.Name, append(slices.Clone(m.Adapters), ad), actorOf(r)); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
if _, err := a.models.SetAdapters(ctx, m.Name, append(slices.Clone(m.Adapters), ad), actor); err != nil {
return none, refused(http.StatusBadRequest, "%v", err)
}
v := TrainPublishView{Adapter: j.Name, Model: m.Name, Source: url, Owner: j.Owner}
if llama {
@@ -528,7 +762,48 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) {
} else {
v.Note = "ask for model " + j.Name + "; a replica loads it on the first request"
}
writeJSON(w, http.StatusOK, v)
return v, nil
}
func (a *API) trainLog() *slog.Logger {
if a.train.Log == nil {
return slog.New(slog.DiscardHandler)
}
return a.train.Log
}
// TrainDone publishes what a job made, when it asked for that: on the
// mirror, as its model's adapter, and for a llama.cpp model with the
// model's replicas replaced, one at a time beside themselves, to load it.
func (a *API) TrainDone(j store.TrainJob) {
if !j.Publish || a.train == nil {
return
}
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
defer cancel()
note := "published: ask for model " + j.Name
v, err := a.publishTrained(ctx, j, "training:"+j.ID)
if err != nil {
note = "trained, but publishing it failed: " + err.Error()
a.trainLog().Warn("a trained adapter could not be published", "job", j.ID, "adapter", j.Name, "err", err)
} else if strings.HasPrefix(v.Note, "llama.cpp") {
var ids []string
for _, rep := range a.ctl.Reconciler().ReadyReplicas() {
if rep.Model == v.Model {
ids = append(ids, rep.ID)
}
}
if len(ids) > 0 {
if err := a.ctl.Reconciler().RestartPaced(reconciler.RestartPace{}, false, ids...); err != nil {
note = "published, but " + v.Model + "'s replicas could not be restarted to load it: " + err.Error()
} else {
note = "published: " + v.Model + "'s replicas are restarting to load it; then ask for model " + j.Name
}
}
}
if err := a.train.Store.SetTrainNote(ctx, j.ID, note); err != nil {
a.trainLog().Warn("could not record a trained adapter's publishing", "job", j.ID, "err", err)
}
}
// untarAll reads a gzipped tar of a PEFT directory's files.
+114
View File
@@ -0,0 +1,114 @@
package adminapi
import (
"context"
"errors"
"net/http"
"slices"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
// Training from the portal's "customise a model" page: the page decides who
// may do what (members of the job's project), and these do it.
// TrainableModels are the catalog's models an adapter can be trained for
// from the portal: chat models llama.cpp or vLLM serve whose weights came
// from the registry, so the weights to train on can be found.
func (a *API) TrainableModels() []string {
var out []string
for _, m := range a.ctl.Catalog().Models() {
if m.IsGuard() || m.IsExternal() || m.Capabilities.Embeddings || m.MakesImages() || m.MakesMeshes() ||
m.Origin == nil || m.Origin.Repo == "" {
continue
}
if slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" || p.Engine == "vllm" }) {
out = append(out, m.Name)
}
}
slices.Sort(out)
return out
}
// Trainings are the jobs started in projects, or by createdBy outside any.
func (a *API) Trainings(ctx context.Context, projects []string, createdBy string) ([]store.TrainJob, error) {
if a.train == nil {
return nil, nil
}
return a.train.Store.TrainJobsFor(ctx, projects, createdBy, 100)
}
// Training is one job.
func (a *API) Training(ctx context.Context, id string) (store.TrainJob, error) {
if a.train == nil {
return store.TrainJob{}, store.ErrNotFound
}
return a.train.Store.TrainJob(ctx, id)
}
// Published is whether a job's adapter is its model's in the catalog.
func (a *API) Published(j store.TrainJob) bool {
m, ok := a.ctl.Catalog().Resolve(j.Model)
return ok && slices.ContainsFunc(m.Adapters, func(ad catalog.Adapter) bool { return ad.Name == j.Name })
}
// ShareTraining changes who a job's adapter belongs to — owner as scope:ID,
// or "" for every caller its model allows — on the job and, once
// published, in the catalog. Refused wider than the knowledge bases its
// pairs came from may be read.
func (a *API) ShareTraining(ctx context.Context, id, owner, actor string) error {
j, err := a.Training(ctx, id)
if err != nil {
return err
}
_, colls, err := countPairs(string(j.Data))
if err != nil {
return err
}
resolved, err := a.trainOwner(ctx, TrainRequest{Owner: owner, Public: owner == ""}, colls)
if err != nil {
return refused(http.StatusBadRequest, "%v", err)
}
if err := a.train.Store.SetTrainOwner(ctx, id, resolved); err != nil {
return err
}
m, ok := a.ctl.Catalog().Resolve(j.Model)
if !ok || a.models == nil {
return nil
}
ads := slices.Clone(m.Adapters)
i := slices.IndexFunc(ads, func(ad catalog.Adapter) bool { return ad.Name == j.Name })
if i < 0 {
return nil
}
ads[i].Owner = resolved
_, err = a.models.SetAdapters(ctx, m.Name, ads, actor)
return err
}
// DeleteTraining stops a job if it trains, takes its adapter out of the
// catalog if it was published, and forgets it.
func (a *API) DeleteTraining(ctx context.Context, id, actor string) error {
j, err := a.Training(ctx, id)
if err != nil {
return err
}
if !store.TrainOver(j.State) {
if err := a.train.Service.Cancel(ctx, id); err != nil {
return err
}
}
if m, ok := a.ctl.Catalog().Resolve(j.Model); ok && a.models != nil {
ads := slices.DeleteFunc(slices.Clone(m.Adapters), func(ad catalog.Adapter) bool { return ad.Name == j.Name })
if len(ads) != len(m.Adapters) {
if _, err := a.models.SetAdapters(ctx, m.Name, ads, actor); err != nil {
return err
}
}
}
if err := a.train.Store.DeleteTrainJob(ctx, id); err != nil && !errors.Is(err, store.ErrNotFound) {
return err
}
return nil
}
@@ -0,0 +1,137 @@
package adminapi
import (
"context"
"errors"
"path/filepath"
"strings"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
type fakeRegistry struct {
repos map[string]hf.Repo
asked int
}
func (f *fakeRegistry) RepoAt(_ context.Context, id, _ string) (hf.Repo, error) {
f.asked++
r, ok := f.repos[id]
if !ok {
return hf.Repo{}, errors.New("404")
}
return r, nil
}
func TestAPortalTrainingFindsItsWeightsAndMemoryAndOneTrainsPerProject(t *testing.T) {
ctx := context.Background()
models := append(testModels(),
catalog.Model{Name: "small", Source: "/s", Origin: &catalog.Origin{Repo: "bartowski/Small-GGUF"},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
catalog.Model{Name: "gated", Source: "/g", Origin: &catalog.Origin{Repo: "meta/Gated"},
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
catalog.Model{Name: "local", Source: "/l", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
a, _ := setup(t, models, gpuNode("cuda01", 1))
s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db"))
if err != nil {
t.Fatal(err)
}
defer s.Close()
reg := &fakeRegistry{repos: map[string]hf.Repo{
"bartowski/Small-GGUF": {ID: "bartowski/Small-GGUF", Revision: "g1", BaseModel: []string{"Org/Small"},
Files: []hf.File{{Path: "small-Q4_K_M.gguf", Bytes: 400 << 20}}},
"Org/Small": {ID: "Org/Small", Revision: "abc", Files: []hf.File{{Path: "model.safetensors", Bytes: 1000 << 20},
{Path: "config.json", Bytes: 1 << 10}}},
"meta/Gated": {ID: "meta/Gated", Gated: hf.Gated{Reason: "manual"},
Files: []hf.File{{Path: "model.safetensors", Bytes: 1 << 30}}},
}}
a.SetTrain(&TrainAdmin{Store: s, Service: train.New(s, noAgents{}, nil)})
a.SetTrainMirror(nil, reg)
if got := a.TrainableModels(); strings.Join(got, ",") != "gated,small" {
t.Errorf("trainable: %v", got)
}
data := train.NDJSON([]train.Pair{{Q: "a", A: "b"}})
req := TrainRequest{Name: "tone", Model: "small", Data: data, Owner: "project:p1", Project: "p1", Publish: true}
j, err := a.StartTraining(ctx, req, "ayse@x")
if err != nil {
t.Fatal(err)
}
if j.BaseRepo != "Org/Small" || j.BaseRevision != "abc" || j.Node != "" || j.GPUMemoryMiB != TrainMemoryMiB(1000<<20) ||
!j.Publish || j.Project != "p1" || j.CreatedBy != "ayse@x" || !j.GGUF {
t.Errorf("queued %+v", j)
}
if j.GPUMemoryMiB != 3072 {
t.Errorf("1000 MiB of weights take %d MiB to train", j.GPUMemoryMiB)
}
req.Name = "tone2"
if _, err := a.StartTraining(ctx, req, "mehmet@x"); err == nil || !strings.Contains(err.Error(), "already has a model training") {
t.Errorf("a second in the project: %v", err)
}
req.Name, req.Project, req.Owner = "tone", "p2", "project:p2"
if _, err := a.StartTraining(ctx, req, "x"); err == nil || !strings.Contains(err.Error(), "already being trained") {
t.Errorf("the same name twice: %v", err)
}
req.Name = "tone3"
asked := reg.asked
if _, err := a.StartTraining(ctx, req, "x"); err != nil {
t.Errorf("another project: %v", err)
}
if reg.asked != asked {
t.Errorf("the registry was asked again: %d", reg.asked-asked)
}
for model, says := range map[string]string{"gated": "gated", "local": "did not come from the model registry"} {
req.Model, req.Name, req.Project = model, "x-"+model, "p-"+model
_, err := a.StartTraining(ctx, req, "x")
if err == nil || !strings.Contains(err.Error(), says) {
t.Errorf("%s: %v", model, err)
}
if msg, theirs := TrainRefusal(err); !theirs || msg == "" {
t.Errorf("%s is not the request's fault: %q", model, msg)
}
}
jobs, err := a.Trainings(ctx, []string{"p1"}, "nobody")
if err != nil || len(jobs) != 1 || jobs[0].Name != "tone" {
t.Errorf("p1's: %+v %v", jobs, err)
}
if err := a.ShareTraining(ctx, j.ID, "", "ayse@x"); err != nil {
t.Fatal(err)
}
if got, _ := a.Training(ctx, j.ID); got.Owner != "" {
t.Errorf("shared with everyone: %q", got.Owner)
}
if err := a.DeleteTraining(ctx, j.ID, "ayse@x"); err != nil {
t.Fatal(err)
}
if _, err := a.Training(ctx, j.ID); !errors.Is(err, store.ErrNotFound) {
t.Errorf("deleted: %v", err)
}
}
func TestATrainingGoesToTheNodeWithTheMostFree(t *testing.T) {
busy := gpuNode("busy", 1)
busy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 23000}}}
roomy := gpuNode("roomy", 1)
roomy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 4000}}}
cpu := inventory.Node{ID: "cpu", State: inventory.Ready}
small := gpuNode("small", 1)
small.GPUs = []discovery.GPU{{Index: 0, MemoryTotalMB: 8000}}
a, _ := setup(t, testModels(), busy, roomy, cpu, small)
if got := a.PickTrainNode(3000, nil); got != "roomy" {
t.Errorf("picked %q", got)
}
if got := a.PickTrainNode(3000, map[string]bool{"roomy": true}); got != "small" {
t.Errorf("with roomy busy, picked %q", got)
}
if got := a.PickTrainNode(30000, nil); got != "" {
t.Errorf("none has 30 GB, picked %q", got)
}
}