package adminapi import ( "archive/tar" "bufio" "bytes" "compress/gzip" "context" "encoding/json" "errors" "fmt" "io" "log/slog" "net/http" "regexp" "slices" "strconv" "strings" "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) // Training LoRA adapters (BACKLOG F4), administered: a job names the model // the adapter is for, the weights to train on, the node and how much of a // card it needs, and the pairs; the node trains when it can, and what it // made is kept here to download or publish. // TrainStore is where jobs are kept. type TrainStore interface { CreateTrainJob(ctx context.Context, j store.TrainJob) error TrainJobs(ctx context.Context, limit int) ([]store.TrainJob, error) TrainJob(ctx context.Context, id string) (store.TrainJob, error) TrainArtifact(ctx context.Context, job, kind string) ([]byte, error) TrainArtifactKinds(ctx context.Context, job string) ([]string, error) DeleteTrainJob(ctx context.Context, id string) error QueuedTrainJobs(ctx context.Context) ([]store.TrainJob, error) ActiveTrainJobs(ctx context.Context) ([]store.TrainJob, error) TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]store.TrainJob, error) SetTrainOwner(ctx context.Context, id, owner string) error SetTrainNote(ctx context.Context, id, message string) error } // TrainAdmin is what the training endpoints need. type TrainAdmin struct { Store TrainStore Service *train.Service // Mirror keeps what a job made where the nodes fetch weights; nil // leaves publishing off. Mirror TrainMirror // Registry finds the weights to train a catalog model on, from where // its own came; nil has every job name them. Registry TrainRegistry // Log says what publishing by itself could not do. Log *slog.Logger mu sync.Mutex bases map[string]trainBaseInfo // by the model's origin repository } // TrainRegistry reads a repository of the model registry. type TrainRegistry interface { RepoAt(ctx context.Context, id, rev string) (hf.Repo, error) } // trainBaseInfo is the safetensors weights a model is trained on. type trainBaseInfo struct { repo, revision string bytes int64 err error at time.Time } // TrainMirror is the model mirror, as publishing needs it. type TrainMirror interface { Upload(ctx context.Context, model, version, file string, body []byte, source string) (mirror.FetchResponse, error) } // TrainPublishView is an adapter published: where the nodes fetch it from, // and what to do for replicas to load it. type TrainPublishView struct { Adapter string `json:"adapter"` Model string `json:"model"` Source string `json:"source"` // Owner is who sees it; empty is every caller of the model. Owner string `json:"owner,omitempty"` Note string `json:"note,omitempty"` } // SetTrain enables the training endpoints. func (a *API) SetTrain(t *TrainAdmin) { a.train = t } // SetTrainMirror gives publishing the mirror, and finding a model's // weights the registry, which are set up after the training endpoints are. func (a *API) SetTrainMirror(m TrainMirror, reg TrainRegistry) { if a.train != nil { a.train.Mirror, a.train.Registry = m, reg } } // TrainRequest queues a job. type TrainRequest struct { // Name is the adapter's: what requests will ask for it by. Name string `json:"name"` // Model is the catalog model the adapter is for. Model string `json:"model"` // BaseRepo and BaseRevision are the weights trained on: the model's own // on its registry, in safetensors (Hugging Face), not a GGUF. Empty // finds them from where the model's came: the repository itself, or the // one its card says it was made from. BaseRepo string `json:"base_repo,omitempty"` BaseRevision string `json:"base_revision,omitempty"` // Node trains it; empty is the first with a card that has enough free. // GPUMemoryMiB is what it needs of a card, which can be what an engine // there leaves free; 0 is worked out from the weights' size. Node string `json:"node,omitempty"` GPUMemoryMiB int `json:"gpu_memory_mib,omitempty"` // Project is the project the job is started in: its members see and // manage it, and only one of its jobs trains at a time. Project string `json:"project,omitempty"` // Publish puts what the job makes on the mirror and makes it the // model's adapter by itself when the job is done, restarting a llama.cpp // model's replicas to load it. Publish bool `json:"publish,omitempty"` // Owner is who the adapter will belong to: org:, team:, project: (by // path or ID), user: or key:. Only callers there see it. When the // pairs come from one knowledge base, it defaults to that collection's // owner; it must be able to read every collection they come from. Owner string `json:"owner,omitempty"` // Public makes the adapter every caller's its model allows. Refused for // pairs from a knowledge base. Public bool `json:"public,omitempty"` // Data is the pairs, NDJSON of {"q", "a"} (llmgw ctl kb dataset). Data string `json:"data"` // Params tune the trainer: rank, alpha, dropout, epochs, lr, batch, // max_len, seed. Params map[string]any `json:"params,omitempty"` // GGUF also makes the adapter in llama.cpp's format; when absent, it // does when the model is served by llama.cpp. GGUF *bool `json:"gguf,omitempty"` } // TrainJobView is a job as listed. type TrainJobView struct { ID string `json:"id"` Name string `json:"name"` Model string `json:"model"` BaseRepo string `json:"base_repo"` BaseRevision string `json:"base_revision,omitempty"` Node string `json:"node"` Owner string `json:"owner,omitempty"` GPUMemoryMiB int `json:"gpu_memory_mib"` Params string `json:"params,omitempty"` GGUF bool `json:"gguf"` Pairs int `json:"pairs"` State string `json:"state"` Step int `json:"step"` Steps int `json:"steps"` Loss float64 `json:"loss"` Message string `json:"message,omitempty"` PeakMiB int `json:"peak_mib"` CreatedAt time.Time `json:"created_at"` StartedAt time.Time `json:"started_at,omitzero"` FinishedAt time.Time `json:"finished_at,omitzero"` CreatedBy string `json:"created_by,omitempty"` Project string `json:"project,omitempty"` Publish bool `json:"publish,omitempty"` // Artifacts are what it made: peft (a gzipped tar), gguf. Artifacts []string `json:"artifacts,omitempty"` } func trainView(j store.TrainJob) TrainJobView { return TrainJobView{ID: j.ID, Name: j.Name, Model: j.Model, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision, Node: j.Node, Owner: j.Owner, GPUMemoryMiB: j.GPUMemoryMiB, Params: j.Params, GGUF: j.GGUF, Pairs: j.Pairs, State: j.State, Step: j.Step, Steps: j.Steps, Loss: j.Loss, Message: j.Message, PeakMiB: j.PeakMiB, CreatedAt: j.CreatedAt, StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy, Project: j.Project, Publish: j.Publish} } func (a *API) registerTrain(mux Mux) { mux.HandleFunc("POST /api/v1/train/jobs", a.trainCreate) mux.HandleFunc("GET /api/v1/train/jobs", a.trainList) mux.HandleFunc("GET /api/v1/train/jobs/{id}", a.trainGet) mux.HandleFunc("POST /api/v1/train/jobs/{id}/cancel", a.trainCancel) mux.HandleFunc("DELETE /api/v1/train/jobs/{id}", a.trainDelete) mux.HandleFunc("GET /api/v1/train/jobs/{id}/artifacts/{kind}", a.trainArtifact) mux.HandleFunc("POST /api/v1/train/jobs/{id}/publish", a.trainPublish) } func (a *API) trainReady(w http.ResponseWriter) bool { if a.train == nil { writeError(w, http.StatusNotImplemented, "training needs the store and the nodes' agents, which this control plane runs without") return false } return true } // trainMaxData bounds a job's pairs. const trainMaxData = 64 << 20 var adapterName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$`) // trainParams are the trainer's settings, with what each may be. var trainParams = map[string][2]float64{ "rank": {1, 256}, "alpha": {1, 1024}, "dropout": {0, 0.9}, "epochs": {1, 100}, "lr": {1e-7, 1e-2}, "batch": {1, 128}, "max_len": {16, 32768}, "seed": {0, 1 << 31}, } func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) { if !a.trainReady(w) { return } var in TrainRequest if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, trainMaxData+1<<20)).Decode(&in); err != nil { writeError(w, http.StatusBadRequest, err.Error()) return } j, err := a.StartTraining(r.Context(), in, actorOf(r)) if err != nil { writeTrainErr(w, err) return } writeJSON(w, http.StatusCreated, trainView(j)) } // trainErr is a refusal, with the status it is answered with. type trainErr struct { code int msg string } func (e *trainErr) Error() string { return e.msg } func refused(code int, format string, args ...any) error { return &trainErr{code: code, msg: fmt.Sprintf(format, args...)} } func writeTrainErr(w http.ResponseWriter, err error) { var te *trainErr if errors.As(err, &te) { writeError(w, te.code, te.msg) return } writeError(w, http.StatusInternalServerError, err.Error()) } // TrainRefusal is the reason a training request was refused, for someone // to read, and whether it was the request's fault rather than the // cluster's. func TrainRefusal(err error) (string, bool) { if err == nil { return "", false } var te *trainErr if errors.As(err, &te) { return te.msg, te.code < 500 } return err.Error(), false } // StartTraining checks a job and queues it, as actor. func (a *API) StartTraining(ctx context.Context, in TrainRequest, actor string) (store.TrainJob, error) { if a.train == nil { return store.TrainJob{}, refused(http.StatusNotImplemented, "training needs the store and the nodes' agents, which this control plane runs without") } bad := func(format string, args ...any) (store.TrainJob, error) { return store.TrainJob{}, refused(http.StatusBadRequest, format, args...) } if !adapterName.MatchString(in.Name) { return bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name) } m, ok := a.ctl.Catalog().Resolve(in.Model) if !ok { return bad("no model %s in the catalog", in.Model) } if _, _, _, taken := a.ctl.Catalog().ResolveAdapter(in.Name); taken || in.Name == m.Name { return bad("%s is already a model's or an adapter's name", in.Name) } if _, taken := a.ctl.Catalog().Resolve(in.Name); taken { return bad("%s is already a model's or an adapter's name", in.Name) } queued, err := a.train.Store.QueuedTrainJobs(ctx) if err != nil { return store.TrainJob{}, err } active, err := a.train.Store.ActiveTrainJobs(ctx) if err != nil { return store.TrainJob{}, err } for _, j := range append(queued, active...) { if j.Name == in.Name { return bad("%s is already being trained (job %s)", in.Name, j.ID) } if in.Project != "" && j.Project == in.Project { return store.TrainJob{}, refused(http.StatusConflict, "this project already has a model training (%s): one at a time, so that one project does not hold every card", j.Name) } } var weights int64 if in.BaseRepo == "" { b := a.trainBase(ctx, m) if b.err != nil { return store.TrainJob{}, b.err } in.BaseRepo, in.BaseRevision, weights = b.repo, b.revision, b.bytes } else if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") { return bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo) } if in.Node != "" { if _, ok := a.ctl.Inventory().Get(in.Node); !ok { return bad("no node %s", in.Node) } } if in.GPUMemoryMiB <= 0 { if weights <= 0 { return bad("gpu_memory_mib is what the job needs of a card, in MiB") } in.GPUMemoryMiB = TrainMemoryMiB(weights) } pairs, colls, err := countPairs(in.Data) if err != nil { return bad("%v", err) } owner, err := a.trainOwner(ctx, in, colls) if err != nil { return bad("%v", err) } for k, v := range in.Params { lim, known := trainParams[k] f, isNum := v.(float64) if !known || !isNum || f < lim[0] || f > lim[1] { return bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k) } } params := "" if len(in.Params) > 0 { b, _ := json.Marshal(in.Params) params = string(b) } gguf := false for _, p := range m.Profiles { if p.Engine == "llamacpp" { gguf = true } } if in.GGUF != nil { gguf = *in.GGUF } j := store.TrainJob{ID: store.NewID("train-"), Name: in.Name, Model: m.Name, BaseRepo: in.BaseRepo, BaseRevision: in.BaseRevision, Node: in.Node, Owner: owner, GPUMemoryMiB: in.GPUMemoryMiB, Params: params, GGUF: gguf, Data: []byte(in.Data), Pairs: pairs, CreatedBy: actor, Project: in.Project, Publish: in.Publish} if err := a.train.Store.CreateTrainJob(ctx, j); err != nil { return store.TrainJob{}, err } // At once rather than at the next look, when the node is here. go a.train.Service.Dispatch(context.WithoutCancel(ctx)) j.State = store.TrainQueued return j, nil } // TrainMemoryMiB is what training a LoRA adapter on weights of this size // takes of a card: the weights in 16 bits, as much again for what passes // through them, and the CUDA context. Measured at 2.3 GB for a 0.5B model. func TrainMemoryMiB(weightBytes int64) int { mib := int(weightBytes>>20)*2 + 1024 return (mib + 255) / 256 * 256 } // trainBaseFor is how long what the registry said of a model's weights is // kept: it changes when the model is updated, which is rare. const trainBaseFor = time.Hour // trainBase is the safetensors weights to train m on: its own origin's, // when they are safetensors, or those its card says it was made from — a // GGUF's base model. func (a *API) trainBase(ctx context.Context, m catalog.Model) trainBaseInfo { if m.Origin == nil || m.Origin.Repo == "" { return trainBaseInfo{err: refused(http.StatusBadRequest, "%s's weights did not come from the model registry, so "+ "what to train it on is not known: an administrator can name them (llmgw ctl train start --base-repo)", m.Name)} } if a.train.Registry == nil { return trainBaseInfo{err: refused(http.StatusNotImplemented, "the model registry is not reachable from here "+ "(mirror.registry): the weights to train %s on cannot be found", m.Name)} } a.train.mu.Lock() b, ok := a.train.bases[m.Origin.Repo] a.train.mu.Unlock() if ok && time.Since(b.at) < trainBaseFor { return b } b = a.findTrainBase(ctx, m) if b.err != nil { var te *trainErr if !errors.As(b.err, &te) { return b // the registry did not answer: asked again next time } } b.at = time.Now() a.train.mu.Lock() if a.train.bases == nil { a.train.bases = map[string]trainBaseInfo{} } a.train.bases[m.Origin.Repo] = b a.train.mu.Unlock() return b } func (a *API) findTrainBase(ctx context.Context, m catalog.Model) trainBaseInfo { ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() repo, err := a.train.Registry.RepoAt(ctx, m.Origin.Repo, "") if err != nil { return trainBaseInfo{err: fmt.Errorf("asking the model registry about %s: %w", m.Origin.Repo, err)} } candidates := []hf.Repo{repo} for _, id := range repo.BaseModel { if id == repo.ID || strings.Count(id, "/") != 1 { continue } r, err := a.train.Registry.RepoAt(ctx, id, "") if err != nil { continue } candidates = append(candidates, r) } gated := "" for _, r := range candidates { var n int64 for _, f := range r.Files { if strings.HasSuffix(strings.ToLower(f.Path), ".safetensors") { n += f.Bytes } } if n == 0 { continue } if r.Gated.Is() { gated = r.ID continue } return trainBaseInfo{repo: r.ID, revision: r.Revision, bytes: n} } if gated != "" { return trainBaseInfo{err: refused(http.StatusBadRequest, "the weights to train %s on, %s, are gated on the registry: "+ "the nodes cannot fetch them without an account that accepted their terms", m.Name, gated)} } return trainBaseInfo{err: refused(http.StatusBadRequest, "neither %s, where %s's weights came from, nor the model its card "+ "names has safetensors weights to train on", m.Origin.Repo, m.Name)} } // PickTrainNode is the node whose freest card has the most memory free, // if that is need, among those ready and not busy. func (a *API) PickTrainNode(need int, busy map[string]bool) string { best, bestFree := "", 0 for _, n := range a.ctl.Inventory().Nodes() { if busy[n.ID] || n.State != inventory.Ready { continue } free := 0 if n.Load != nil { for _, g := range n.Load.GPUs { if g.MemoryTotalMB > 0 { free = max(free, int(g.MemoryTotalMB-g.MemoryUsedMB)) } } } if free == 0 { for _, g := range n.GPUs { free = max(free, g.MemoryTotalMB-g.MemoryUsedMB) } } if free >= need && free > bestFree { best, bestFree = n.ID, free } } return best } // trainOwner is who a job's adapter will belong to, as scope:ID: one that // may read every knowledge base the pairs come from. Pairs from a knowledge // base never make an adapter everyone's. func (a *API) trainOwner(ctx context.Context, in TrainRequest, colls []string) (string, error) { if in.Public { if len(colls) > 0 { return "", errors.New("these pairs come from a knowledge base, and an adapter trained on them " + "carries some of what it says: it belongs to a part of the tree that may read it, not to everyone") } if in.Owner != "" { return "", errors.New("an adapter is public or has an owner, not both") } return "", nil } if len(colls) > 0 && a.kb == nil { return "", errors.New("the pairs name knowledge bases, which this control plane cannot read") } var found []store.KBCollection for _, id := range colls { c, err := a.kb.Store.Collection(ctx, id) if err != nil { return "", fmt.Errorf("the pairs come from collection %s, which is not here: %w", id, err) } found = append(found, c) } if in.Owner == "" { if len(found) == 1 { return string(found[0].OwnerScope) + ":" + found[0].OwnerID, nil } return "", errors.New("name the adapter's owner (org:, team:, project:, user: or key:), or make it public " + "when its pairs are not a knowledge base's") } scope, id, err := a.kbTarget(ctx, in.Owner) if err != nil { return "", err } for _, c := range found { e, err := a.kb.Store.EffectiveFor(ctx, scope, id, c.OrgID) if err != nil { return "", fmt.Errorf("%s: %w", in.Owner, err) } readable, err := a.kb.Store.CollectionsFor(ctx, e, false) if err != nil { return "", err } if !slices.ContainsFunc(readable, func(x store.KBCollection) bool { return x.ID == c.ID }) { return "", fmt.Errorf("%s may not read collection %s, which the pairs come from: "+ "an adapter trained on it would tell its callers what it says", in.Owner, c.Name) } } return string(scope) + ":" + id, nil } // countPairs checks a job's data: NDJSON of {"q", "a"}, at least one, and // says which knowledge bases the pairs come from. func countPairs(data string) (int, []string, error) { if len(data) > trainMaxData { return 0, nil, fmt.Errorf("the data is over %d MiB", trainMaxData>>20) } seen := map[string]bool{} var colls []string n := 0 sc := bufio.NewScanner(strings.NewReader(data)) sc.Buffer(make([]byte, 64<<10), 1<<20) line := 0 for sc.Scan() { line++ b := bytes.TrimSpace(sc.Bytes()) if len(b) == 0 { continue } var p struct{ Q, A, Collection string } if err := json.Unmarshal(b, &p); err != nil { return 0, nil, fmt.Errorf("data line %d is not JSON: %v", line, err) } if strings.TrimSpace(p.Q) == "" || strings.TrimSpace(p.A) == "" { return 0, nil, fmt.Errorf("data line %d has no q or no a", line) } if p.Collection != "" && !seen[p.Collection] { seen[p.Collection] = true colls = append(colls, p.Collection) } n++ } if err := sc.Err(); err != nil { return 0, nil, err } if n == 0 { return 0, nil, errors.New("the data holds no pairs: NDJSON of {\"q\", \"a\"}, as llmgw ctl kb dataset writes") } return n, colls, nil } func (a *API) trainList(w http.ResponseWriter, r *http.Request) { if !a.trainReady(w) { return } limit := 50 if n, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && n > 0 && n <= 500 { limit = n } jobs, err := a.train.Store.TrainJobs(r.Context(), limit) if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } out := []TrainJobView{} for _, j := range jobs { out = append(out, trainView(j)) } writeJSON(w, http.StatusOK, out) } func (a *API) trainJob(w http.ResponseWriter, r *http.Request) (store.TrainJob, bool) { if !a.trainReady(w) { return store.TrainJob{}, false } j, err := a.train.Store.TrainJob(r.Context(), r.PathValue("id")) if errors.Is(err, store.ErrNotFound) { writeError(w, http.StatusNotFound, "no training job "+r.PathValue("id")) return j, false } if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return j, false } return j, true } func (a *API) trainGet(w http.ResponseWriter, r *http.Request) { j, ok := a.trainJob(w, r) if !ok { return } v := trainView(j) v.Artifacts, _ = a.train.Store.TrainArtifactKinds(r.Context(), j.ID) writeJSON(w, http.StatusOK, v) } func (a *API) trainCancel(w http.ResponseWriter, r *http.Request) { j, ok := a.trainJob(w, r) if !ok { return } if err := a.train.Service.Cancel(r.Context(), j.ID); err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } j, _ = a.train.Store.TrainJob(r.Context(), j.ID) writeJSON(w, http.StatusOK, trainView(j)) } func (a *API) trainDelete(w http.ResponseWriter, r *http.Request) { j, ok := a.trainJob(w, r) if !ok { return } if !store.TrainOver(j.State) { writeError(w, http.StatusConflict, "the job has not ended: cancel it first") return } if err := a.train.Store.DeleteTrainJob(r.Context(), j.ID); err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } writeJSON(w, http.StatusOK, map[string]any{"id": j.ID, "deleted": true}) } func (a *API) trainArtifact(w http.ResponseWriter, r *http.Request) { j, ok := a.trainJob(w, r) if !ok { return } kind := r.PathValue("kind") b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, kind) if errors.Is(err, store.ErrNotFound) { writeError(w, http.StatusNotFound, "the job made no "+kind) return } if err != nil { writeError(w, http.StatusInternalServerError, err.Error()) return } name := j.Name + ".tar.gz" w.Header().Set("Content-Type", "application/gzip") if kind == "gguf" { name = j.Name + ".gguf" w.Header().Set("Content-Type", "application/octet-stream") } w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`) w.Header().Set("Content-Length", strconv.Itoa(len(b))) _, _ = w.Write(b) } // trainPublish puts what a job made on the mirror and makes it the model's // adapter: the GGUF for a model llama.cpp serves, the PEFT directory for // one vLLM serves. func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) { j, ok := a.trainJob(w, r) if !ok { return } v, err := a.publishTrained(r.Context(), j, actorOf(r)) if err != nil { writeTrainErr(w, err) return } writeJSON(w, http.StatusOK, v) } // publishTrained puts what a job made on the mirror and makes it the // model's adapter: the GGUF for a model llama.cpp serves, the PEFT // directory for one vLLM serves. func (a *API) publishTrained(ctx context.Context, j store.TrainJob, actor string) (TrainPublishView, error) { var none TrainPublishView if a.train.Mirror == nil { return none, refused(http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+ "llmgw ctl train download, then llmgw mirror import there, does it by hand") } if a.models == nil { return none, refused(http.StatusNotImplemented, "this control plane has no store for the catalog") } if j.State != store.TrainDone { return none, refused(http.StatusConflict, "%s", "the job has not finished: it is "+j.State) } m, found := a.ctl.Catalog().Resolve(j.Model) if !found { return none, refused(http.StatusConflict, "%s", "the model "+j.Model+" is no longer in the catalog") } for _, ad := range m.Adapters { if ad.Name == j.Name { return none, refused(http.StatusConflict, "%s", j.Name+" is already "+m.Name+"'s adapter; train under another name") } } llama := slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" }) source := "trained by llmgw job " + j.ID + " on " + j.Node + " from " + j.BaseRepo var url string if llama { b, err := a.train.Store.TrainArtifact(ctx, j.ID, "gguf") if err != nil { return none, refused(http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf") } res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, j.Name+".gguf", b, source) if err != nil { return none, refused(http.StatusBadGateway, "%s", "keeping it on the mirror: "+err.Error()) } url = res.URL } else { b, err := a.train.Store.TrainArtifact(ctx, j.ID, "peft") if err != nil { return none, refused(http.StatusConflict, "the job made no PEFT adapter") } files, err := untarAll(b) if err != nil { return none, err } for name, data := range files { res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, name, data, source) if err != nil { return none, refused(http.StatusBadGateway, "%s", "keeping "+name+" on the mirror: "+err.Error()) } url = res.URL[:strings.LastIndex(res.URL, "/")+1] } } ad := catalog.Adapter{Name: j.Name, Source: url, Owner: j.Owner} if err := checkAdapters(ctx, m, []catalog.Adapter{ad}); err != nil { return none, refused(http.StatusBadRequest, "%v", err) } if _, err := a.models.SetAdapters(ctx, m.Name, append(slices.Clone(m.Adapters), ad), actor); err != nil { return none, refused(http.StatusBadRequest, "%v", err) } v := TrainPublishView{Adapter: j.Name, Model: m.Name, Source: url, Owner: j.Owner} if llama { v.Note = "llama.cpp loads adapters when a replica starts: restart " + m.Name + "'s replicas (llmgw ctl restart " + m.Name + ") and ask for model " + j.Name } else { v.Note = "ask for model " + j.Name + "; a replica loads it on the first request" } return v, nil } func (a *API) trainLog() *slog.Logger { if a.train.Log == nil { return slog.New(slog.DiscardHandler) } return a.train.Log } // TrainDone publishes what a job made, when it asked for that: on the // mirror, as its model's adapter, and for a llama.cpp model with the // model's replicas replaced, one at a time beside themselves, to load it. func (a *API) TrainDone(j store.TrainJob) { if !j.Publish || a.train == nil { return } ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) defer cancel() note := "published: ask for model " + j.Name v, err := a.publishTrained(ctx, j, "training:"+j.ID) if err != nil { note = "trained, but publishing it failed: " + err.Error() a.trainLog().Warn("a trained adapter could not be published", "job", j.ID, "adapter", j.Name, "err", err) } else if strings.HasPrefix(v.Note, "llama.cpp") { var ids []string for _, rep := range a.ctl.Reconciler().ReadyReplicas() { if rep.Model == v.Model { ids = append(ids, rep.ID) } } if len(ids) > 0 { if err := a.ctl.Reconciler().RestartPaced(reconciler.RestartPace{}, false, ids...); err != nil { note = "published, but " + v.Model + "'s replicas could not be restarted to load it: " + err.Error() } else { note = "published: " + v.Model + "'s replicas are restarting to load it; then ask for model " + j.Name } } } if err := a.train.Store.SetTrainNote(ctx, j.ID, note); err != nil { a.trainLog().Warn("could not record a trained adapter's publishing", "job", j.ID, "err", err) } } // untarAll reads a gzipped tar of a PEFT directory's files. func untarAll(b []byte) (map[string][]byte, error) { gz, err := gzip.NewReader(bytes.NewReader(b)) if err != nil { return nil, err } tr := tar.NewReader(gz) out := map[string][]byte{} for { h, err := tr.Next() if errors.Is(err, io.EOF) { return out, nil } if err != nil { return nil, err } if h.Typeflag != tar.TypeReg || strings.ContainsAny(h.Name, "/\\") { continue } data, err := io.ReadAll(io.LimitReader(tr, 4<<30)) if err != nil { return nil, err } out[h.Name] = data } }