Customise a model from the portal: a file, a model, a button
A page for anyone in a project, built for the one who knows least: an Excel, CSV or JSONL file of questions and answers, previewed; a model; a name; who may use it; and a button. The weights to train on are found from where the model's came (a GGUF repository's base model), the card's memory from their size, the node as the one with the most free. When it is done the adapter is published by itself (a llama.cpp model's replicas replaced to load it) and is a model of its own in the chat and the API. The project's by default; its members share it wider (confirmed, and never wider than the knowledge bases its pairs came from) or delete it. One trains at a time per project. ctl train start needs only --model and --data now, and --publish publishes when done. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fcda4d3f85
commit
a53b2f4be9
33 files changed
+2199
-134
No files matched your search
@@ -10,15 +10,20 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
@@ -36,6 +41,11 @@ type TrainStore interface {
|
||||
TrainArtifact(ctx context.Context, job, kind string) ([]byte, error)
|
||||
TrainArtifactKinds(ctx context.Context, job string) ([]string, error)
|
||||
DeleteTrainJob(ctx context.Context, id string) error
|
||||
QueuedTrainJobs(ctx context.Context) ([]store.TrainJob, error)
|
||||
ActiveTrainJobs(ctx context.Context) ([]store.TrainJob, error)
|
||||
TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]store.TrainJob, error)
|
||||
SetTrainOwner(ctx context.Context, id, owner string) error
|
||||
SetTrainNote(ctx context.Context, id, message string) error
|
||||
}
|
||||
|
||||
// TrainAdmin is what the training endpoints need.
|
||||
@@ -45,6 +55,27 @@ type TrainAdmin struct {
|
||||
// Mirror keeps what a job made where the nodes fetch weights; nil
|
||||
// leaves publishing off.
|
||||
Mirror TrainMirror
|
||||
// Registry finds the weights to train a catalog model on, from where
|
||||
// its own came; nil has every job name them.
|
||||
Registry TrainRegistry
|
||||
// Log says what publishing by itself could not do.
|
||||
Log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
bases map[string]trainBaseInfo // by the model's origin repository
|
||||
}
|
||||
|
||||
// TrainRegistry reads a repository of the model registry.
|
||||
type TrainRegistry interface {
|
||||
RepoAt(ctx context.Context, id, rev string) (hf.Repo, error)
|
||||
}
|
||||
|
||||
// trainBaseInfo is the safetensors weights a model is trained on.
|
||||
type trainBaseInfo struct {
|
||||
repo, revision string
|
||||
bytes int64
|
||||
err error
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// TrainMirror is the model mirror, as publishing needs it.
|
||||
@@ -66,11 +97,11 @@ type TrainPublishView struct {
|
||||
// SetTrain enables the training endpoints.
|
||||
func (a *API) SetTrain(t *TrainAdmin) { a.train = t }
|
||||
|
||||
// SetTrainMirror gives publishing the mirror, which is set up after the
|
||||
// training endpoints are.
|
||||
func (a *API) SetTrainMirror(m TrainMirror) {
|
||||
// SetTrainMirror gives publishing the mirror, and finding a model's
|
||||
// weights the registry, which are set up after the training endpoints are.
|
||||
func (a *API) SetTrainMirror(m TrainMirror, reg TrainRegistry) {
|
||||
if a.train != nil {
|
||||
a.train.Mirror = m
|
||||
a.train.Mirror, a.train.Registry = m, reg
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,13 +112,23 @@ type TrainRequest struct {
|
||||
// Model is the catalog model the adapter is for.
|
||||
Model string `json:"model"`
|
||||
// BaseRepo and BaseRevision are the weights trained on: the model's own
|
||||
// on its registry, in safetensors (Hugging Face), not a GGUF.
|
||||
BaseRepo string `json:"base_repo"`
|
||||
// on its registry, in safetensors (Hugging Face), not a GGUF. Empty
|
||||
// finds them from where the model's came: the repository itself, or the
|
||||
// one its card says it was made from.
|
||||
BaseRepo string `json:"base_repo,omitempty"`
|
||||
BaseRevision string `json:"base_revision,omitempty"`
|
||||
// Node trains it; GPUMemoryMiB is what it needs of a card, which can be
|
||||
// what an engine there leaves free.
|
||||
Node string `json:"node"`
|
||||
GPUMemoryMiB int `json:"gpu_memory_mib"`
|
||||
// Node trains it; empty is the first with a card that has enough free.
|
||||
// GPUMemoryMiB is what it needs of a card, which can be what an engine
|
||||
// there leaves free; 0 is worked out from the weights' size.
|
||||
Node string `json:"node,omitempty"`
|
||||
GPUMemoryMiB int `json:"gpu_memory_mib,omitempty"`
|
||||
// Project is the project the job is started in: its members see and
|
||||
// manage it, and only one of its jobs trains at a time.
|
||||
Project string `json:"project,omitempty"`
|
||||
// Publish puts what the job makes on the mirror and makes it the
|
||||
// model's adapter by itself when the job is done, restarting a llama.cpp
|
||||
// model's replicas to load it.
|
||||
Publish bool `json:"publish,omitempty"`
|
||||
// Owner is who the adapter will belong to: org:, team:, project: (by
|
||||
// path or ID), user: or key:. Only callers there see it. When the
|
||||
// pairs come from one knowledge base, it defaults to that collection's
|
||||
@@ -129,6 +170,8 @@ type TrainJobView struct {
|
||||
StartedAt time.Time `json:"started_at,omitzero"`
|
||||
FinishedAt time.Time `json:"finished_at,omitzero"`
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
Project string `json:"project,omitempty"`
|
||||
Publish bool `json:"publish,omitempty"`
|
||||
// Artifacts are what it made: peft (a gzipped tar), gguf.
|
||||
Artifacts []string `json:"artifacts,omitempty"`
|
||||
}
|
||||
@@ -137,7 +180,8 @@ func trainView(j store.TrainJob) TrainJobView {
|
||||
return TrainJobView{ID: j.ID, Name: j.Name, Model: j.Model, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision,
|
||||
Node: j.Node, Owner: j.Owner, GPUMemoryMiB: j.GPUMemoryMiB, Params: j.Params, GGUF: j.GGUF, Pairs: j.Pairs, State: j.State,
|
||||
Step: j.Step, Steps: j.Steps, Loss: j.Loss, Message: j.Message, PeakMiB: j.PeakMiB, CreatedAt: j.CreatedAt,
|
||||
StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy}
|
||||
StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy,
|
||||
Project: j.Project, Publish: j.Publish}
|
||||
}
|
||||
|
||||
func (a *API) registerTrain(mux Mux) {
|
||||
@@ -178,50 +222,122 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
bad := func(format string, args ...any) {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf(format, args...))
|
||||
j, err := a.StartTraining(r.Context(), in, actorOf(r))
|
||||
if err != nil {
|
||||
writeTrainErr(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, trainView(j))
|
||||
}
|
||||
|
||||
// trainErr is a refusal, with the status it is answered with.
|
||||
type trainErr struct {
|
||||
code int
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *trainErr) Error() string { return e.msg }
|
||||
|
||||
func refused(code int, format string, args ...any) error {
|
||||
return &trainErr{code: code, msg: fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
func writeTrainErr(w http.ResponseWriter, err error) {
|
||||
var te *trainErr
|
||||
if errors.As(err, &te) {
|
||||
writeError(w, te.code, te.msg)
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
}
|
||||
|
||||
// TrainRefusal is the reason a training request was refused, for someone
|
||||
// to read, and whether it was the request's fault rather than the
|
||||
// cluster's.
|
||||
func TrainRefusal(err error) (string, bool) {
|
||||
if err == nil {
|
||||
return "", false
|
||||
}
|
||||
var te *trainErr
|
||||
if errors.As(err, &te) {
|
||||
return te.msg, te.code < 500
|
||||
}
|
||||
return err.Error(), false
|
||||
}
|
||||
|
||||
// StartTraining checks a job and queues it, as actor.
|
||||
func (a *API) StartTraining(ctx context.Context, in TrainRequest, actor string) (store.TrainJob, error) {
|
||||
if a.train == nil {
|
||||
return store.TrainJob{}, refused(http.StatusNotImplemented,
|
||||
"training needs the store and the nodes' agents, which this control plane runs without")
|
||||
}
|
||||
bad := func(format string, args ...any) (store.TrainJob, error) {
|
||||
return store.TrainJob{}, refused(http.StatusBadRequest, format, args...)
|
||||
}
|
||||
if !adapterName.MatchString(in.Name) {
|
||||
bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name)
|
||||
return
|
||||
return bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name)
|
||||
}
|
||||
m, ok := a.ctl.Catalog().Resolve(in.Model)
|
||||
if !ok {
|
||||
bad("no model %s in the catalog", in.Model)
|
||||
return
|
||||
return bad("no model %s in the catalog", in.Model)
|
||||
}
|
||||
if _, _, _, taken := a.ctl.Catalog().ResolveAdapter(in.Name); taken || in.Name == m.Name {
|
||||
bad("%s is already a model's or an adapter's name", in.Name)
|
||||
return
|
||||
return bad("%s is already a model's or an adapter's name", in.Name)
|
||||
}
|
||||
if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") {
|
||||
bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo)
|
||||
return
|
||||
if _, taken := a.ctl.Catalog().Resolve(in.Name); taken {
|
||||
return bad("%s is already a model's or an adapter's name", in.Name)
|
||||
}
|
||||
if _, ok := a.ctl.Inventory().Get(in.Node); !ok {
|
||||
bad("no node %s", in.Node)
|
||||
return
|
||||
queued, err := a.train.Store.QueuedTrainJobs(ctx)
|
||||
if err != nil {
|
||||
return store.TrainJob{}, err
|
||||
}
|
||||
active, err := a.train.Store.ActiveTrainJobs(ctx)
|
||||
if err != nil {
|
||||
return store.TrainJob{}, err
|
||||
}
|
||||
for _, j := range append(queued, active...) {
|
||||
if j.Name == in.Name {
|
||||
return bad("%s is already being trained (job %s)", in.Name, j.ID)
|
||||
}
|
||||
if in.Project != "" && j.Project == in.Project {
|
||||
return store.TrainJob{}, refused(http.StatusConflict,
|
||||
"this project already has a model training (%s): one at a time, so that one project does not hold every card", j.Name)
|
||||
}
|
||||
}
|
||||
var weights int64
|
||||
if in.BaseRepo == "" {
|
||||
b := a.trainBase(ctx, m)
|
||||
if b.err != nil {
|
||||
return store.TrainJob{}, b.err
|
||||
}
|
||||
in.BaseRepo, in.BaseRevision, weights = b.repo, b.revision, b.bytes
|
||||
} else if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") {
|
||||
return bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo)
|
||||
}
|
||||
if in.Node != "" {
|
||||
if _, ok := a.ctl.Inventory().Get(in.Node); !ok {
|
||||
return bad("no node %s", in.Node)
|
||||
}
|
||||
}
|
||||
if in.GPUMemoryMiB <= 0 {
|
||||
bad("gpu_memory_mib is what the job needs of a card, in MiB")
|
||||
return
|
||||
if weights <= 0 {
|
||||
return bad("gpu_memory_mib is what the job needs of a card, in MiB")
|
||||
}
|
||||
in.GPUMemoryMiB = TrainMemoryMiB(weights)
|
||||
}
|
||||
pairs, colls, err := countPairs(in.Data)
|
||||
if err != nil {
|
||||
bad("%v", err)
|
||||
return
|
||||
return bad("%v", err)
|
||||
}
|
||||
owner, err := a.trainOwner(r.Context(), in, colls)
|
||||
owner, err := a.trainOwner(ctx, in, colls)
|
||||
if err != nil {
|
||||
bad("%v", err)
|
||||
return
|
||||
return bad("%v", err)
|
||||
}
|
||||
for k, v := range in.Params {
|
||||
lim, known := trainParams[k]
|
||||
f, isNum := v.(float64)
|
||||
if !known || !isNum || f < lim[0] || f > lim[1] {
|
||||
bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k)
|
||||
return
|
||||
return bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k)
|
||||
}
|
||||
}
|
||||
params := ""
|
||||
@@ -240,15 +356,132 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
j := store.TrainJob{ID: store.NewID("train-"), Name: in.Name, Model: m.Name, BaseRepo: in.BaseRepo,
|
||||
BaseRevision: in.BaseRevision, Node: in.Node, Owner: owner, GPUMemoryMiB: in.GPUMemoryMiB, Params: params, GGUF: gguf,
|
||||
Data: []byte(in.Data), Pairs: pairs, CreatedBy: actorOf(r)}
|
||||
if err := a.train.Store.CreateTrainJob(r.Context(), j); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
Data: []byte(in.Data), Pairs: pairs, CreatedBy: actor, Project: in.Project, Publish: in.Publish}
|
||||
if err := a.train.Store.CreateTrainJob(ctx, j); err != nil {
|
||||
return store.TrainJob{}, err
|
||||
}
|
||||
// At once rather than at the next look, when the node is here.
|
||||
go a.train.Service.Dispatch(context.WithoutCancel(r.Context()))
|
||||
go a.train.Service.Dispatch(context.WithoutCancel(ctx))
|
||||
j.State = store.TrainQueued
|
||||
writeJSON(w, http.StatusCreated, trainView(j))
|
||||
return j, nil
|
||||
}
|
||||
|
||||
// TrainMemoryMiB is what training a LoRA adapter on weights of this size
|
||||
// takes of a card: the weights in 16 bits, as much again for what passes
|
||||
// through them, and the CUDA context. Measured at 2.3 GB for a 0.5B model.
|
||||
func TrainMemoryMiB(weightBytes int64) int {
|
||||
mib := int(weightBytes>>20)*2 + 1024
|
||||
return (mib + 255) / 256 * 256
|
||||
}
|
||||
|
||||
// trainBaseFor is how long what the registry said of a model's weights is
|
||||
// kept: it changes when the model is updated, which is rare.
|
||||
const trainBaseFor = time.Hour
|
||||
|
||||
// trainBase is the safetensors weights to train m on: its own origin's,
|
||||
// when they are safetensors, or those its card says it was made from — a
|
||||
// GGUF's base model.
|
||||
func (a *API) trainBase(ctx context.Context, m catalog.Model) trainBaseInfo {
|
||||
if m.Origin == nil || m.Origin.Repo == "" {
|
||||
return trainBaseInfo{err: refused(http.StatusBadRequest, "%s's weights did not come from the model registry, so "+
|
||||
"what to train it on is not known: an administrator can name them (llmgw ctl train start --base-repo)", m.Name)}
|
||||
}
|
||||
if a.train.Registry == nil {
|
||||
return trainBaseInfo{err: refused(http.StatusNotImplemented, "the model registry is not reachable from here "+
|
||||
"(mirror.registry): the weights to train %s on cannot be found", m.Name)}
|
||||
}
|
||||
a.train.mu.Lock()
|
||||
b, ok := a.train.bases[m.Origin.Repo]
|
||||
a.train.mu.Unlock()
|
||||
if ok && time.Since(b.at) < trainBaseFor {
|
||||
return b
|
||||
}
|
||||
b = a.findTrainBase(ctx, m)
|
||||
if b.err != nil {
|
||||
var te *trainErr
|
||||
if !errors.As(b.err, &te) {
|
||||
return b // the registry did not answer: asked again next time
|
||||
}
|
||||
}
|
||||
b.at = time.Now()
|
||||
a.train.mu.Lock()
|
||||
if a.train.bases == nil {
|
||||
a.train.bases = map[string]trainBaseInfo{}
|
||||
}
|
||||
a.train.bases[m.Origin.Repo] = b
|
||||
a.train.mu.Unlock()
|
||||
return b
|
||||
}
|
||||
|
||||
func (a *API) findTrainBase(ctx context.Context, m catalog.Model) trainBaseInfo {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
repo, err := a.train.Registry.RepoAt(ctx, m.Origin.Repo, "")
|
||||
if err != nil {
|
||||
return trainBaseInfo{err: fmt.Errorf("asking the model registry about %s: %w", m.Origin.Repo, err)}
|
||||
}
|
||||
candidates := []hf.Repo{repo}
|
||||
for _, id := range repo.BaseModel {
|
||||
if id == repo.ID || strings.Count(id, "/") != 1 {
|
||||
continue
|
||||
}
|
||||
r, err := a.train.Registry.RepoAt(ctx, id, "")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, r)
|
||||
}
|
||||
gated := ""
|
||||
for _, r := range candidates {
|
||||
var n int64
|
||||
for _, f := range r.Files {
|
||||
if strings.HasSuffix(strings.ToLower(f.Path), ".safetensors") {
|
||||
n += f.Bytes
|
||||
}
|
||||
}
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
if r.Gated.Is() {
|
||||
gated = r.ID
|
||||
continue
|
||||
}
|
||||
return trainBaseInfo{repo: r.ID, revision: r.Revision, bytes: n}
|
||||
}
|
||||
if gated != "" {
|
||||
return trainBaseInfo{err: refused(http.StatusBadRequest, "the weights to train %s on, %s, are gated on the registry: "+
|
||||
"the nodes cannot fetch them without an account that accepted their terms", m.Name, gated)}
|
||||
}
|
||||
return trainBaseInfo{err: refused(http.StatusBadRequest, "neither %s, where %s's weights came from, nor the model its card "+
|
||||
"names has safetensors weights to train on", m.Origin.Repo, m.Name)}
|
||||
}
|
||||
|
||||
// PickTrainNode is the node whose freest card has the most memory free,
|
||||
// if that is need, among those ready and not busy.
|
||||
func (a *API) PickTrainNode(need int, busy map[string]bool) string {
|
||||
best, bestFree := "", 0
|
||||
for _, n := range a.ctl.Inventory().Nodes() {
|
||||
if busy[n.ID] || n.State != inventory.Ready {
|
||||
continue
|
||||
}
|
||||
free := 0
|
||||
if n.Load != nil {
|
||||
for _, g := range n.Load.GPUs {
|
||||
if g.MemoryTotalMB > 0 {
|
||||
free = max(free, int(g.MemoryTotalMB-g.MemoryUsedMB))
|
||||
}
|
||||
}
|
||||
}
|
||||
if free == 0 {
|
||||
for _, g := range n.GPUs {
|
||||
free = max(free, g.MemoryTotalMB-g.MemoryUsedMB)
|
||||
}
|
||||
}
|
||||
if free >= need && free > bestFree {
|
||||
best, bestFree = n.ID, free
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// trainOwner is who a job's adapter will belong to, as scope:ID: one that
|
||||
@@ -453,73 +686,74 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if a.train.Mirror == nil {
|
||||
writeError(w, http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+
|
||||
"llmgw ctl train download, then llmgw mirror import there, does it by hand")
|
||||
v, err := a.publishTrained(r.Context(), j, actorOf(r))
|
||||
if err != nil {
|
||||
writeTrainErr(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
// publishTrained puts what a job made on the mirror and makes it the
|
||||
// model's adapter: the GGUF for a model llama.cpp serves, the PEFT
|
||||
// directory for one vLLM serves.
|
||||
func (a *API) publishTrained(ctx context.Context, j store.TrainJob, actor string) (TrainPublishView, error) {
|
||||
var none TrainPublishView
|
||||
if a.train.Mirror == nil {
|
||||
return none, refused(http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+
|
||||
"llmgw ctl train download, then llmgw mirror import there, does it by hand")
|
||||
}
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store for the catalog")
|
||||
return
|
||||
return none, refused(http.StatusNotImplemented, "this control plane has no store for the catalog")
|
||||
}
|
||||
if j.State != store.TrainDone {
|
||||
writeError(w, http.StatusConflict, "the job has not finished: it is "+j.State)
|
||||
return
|
||||
return none, refused(http.StatusConflict, "%s", "the job has not finished: it is "+j.State)
|
||||
}
|
||||
m, found := a.ctl.Catalog().Resolve(j.Model)
|
||||
if !found {
|
||||
writeError(w, http.StatusConflict, "the model "+j.Model+" is no longer in the catalog")
|
||||
return
|
||||
return none, refused(http.StatusConflict, "%s", "the model "+j.Model+" is no longer in the catalog")
|
||||
}
|
||||
for _, ad := range m.Adapters {
|
||||
if ad.Name == j.Name {
|
||||
writeError(w, http.StatusConflict, j.Name+" is already "+m.Name+"'s adapter; train under another name")
|
||||
return
|
||||
return none, refused(http.StatusConflict, "%s", j.Name+" is already "+m.Name+"'s adapter; train under another name")
|
||||
}
|
||||
}
|
||||
llama := slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" })
|
||||
source := "trained by llmgw job " + j.ID + " on " + j.Node + " from " + j.BaseRepo
|
||||
var url string
|
||||
if llama {
|
||||
b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "gguf")
|
||||
b, err := a.train.Store.TrainArtifact(ctx, j.ID, "gguf")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf")
|
||||
return
|
||||
return none, refused(http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf")
|
||||
}
|
||||
res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, j.Name+".gguf", b, source)
|
||||
res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, j.Name+".gguf", b, source)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, "keeping it on the mirror: "+err.Error())
|
||||
return
|
||||
return none, refused(http.StatusBadGateway, "%s", "keeping it on the mirror: "+err.Error())
|
||||
}
|
||||
url = res.URL
|
||||
} else {
|
||||
b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "peft")
|
||||
b, err := a.train.Store.TrainArtifact(ctx, j.ID, "peft")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusConflict, "the job made no PEFT adapter")
|
||||
return
|
||||
return none, refused(http.StatusConflict, "the job made no PEFT adapter")
|
||||
}
|
||||
files, err := untarAll(b)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
return none, err
|
||||
}
|
||||
for name, data := range files {
|
||||
res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, name, data, source)
|
||||
res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, name, data, source)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, "keeping "+name+" on the mirror: "+err.Error())
|
||||
return
|
||||
return none, refused(http.StatusBadGateway, "%s", "keeping "+name+" on the mirror: "+err.Error())
|
||||
}
|
||||
url = res.URL[:strings.LastIndex(res.URL, "/")+1]
|
||||
}
|
||||
}
|
||||
ad := catalog.Adapter{Name: j.Name, Source: url, Owner: j.Owner}
|
||||
if err := checkAdapters(r.Context(), m, []catalog.Adapter{ad}); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
if err := checkAdapters(ctx, m, []catalog.Adapter{ad}); err != nil {
|
||||
return none, refused(http.StatusBadRequest, "%v", err)
|
||||
}
|
||||
if _, err := a.models.SetAdapters(r.Context(), m.Name, append(slices.Clone(m.Adapters), ad), actorOf(r)); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
if _, err := a.models.SetAdapters(ctx, m.Name, append(slices.Clone(m.Adapters), ad), actor); err != nil {
|
||||
return none, refused(http.StatusBadRequest, "%v", err)
|
||||
}
|
||||
v := TrainPublishView{Adapter: j.Name, Model: m.Name, Source: url, Owner: j.Owner}
|
||||
if llama {
|
||||
@@ -528,7 +762,48 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) {
|
||||
} else {
|
||||
v.Note = "ask for model " + j.Name + "; a replica loads it on the first request"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (a *API) trainLog() *slog.Logger {
|
||||
if a.train.Log == nil {
|
||||
return slog.New(slog.DiscardHandler)
|
||||
}
|
||||
return a.train.Log
|
||||
}
|
||||
|
||||
// TrainDone publishes what a job made, when it asked for that: on the
|
||||
// mirror, as its model's adapter, and for a llama.cpp model with the
|
||||
// model's replicas replaced, one at a time beside themselves, to load it.
|
||||
func (a *API) TrainDone(j store.TrainJob) {
|
||||
if !j.Publish || a.train == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
note := "published: ask for model " + j.Name
|
||||
v, err := a.publishTrained(ctx, j, "training:"+j.ID)
|
||||
if err != nil {
|
||||
note = "trained, but publishing it failed: " + err.Error()
|
||||
a.trainLog().Warn("a trained adapter could not be published", "job", j.ID, "adapter", j.Name, "err", err)
|
||||
} else if strings.HasPrefix(v.Note, "llama.cpp") {
|
||||
var ids []string
|
||||
for _, rep := range a.ctl.Reconciler().ReadyReplicas() {
|
||||
if rep.Model == v.Model {
|
||||
ids = append(ids, rep.ID)
|
||||
}
|
||||
}
|
||||
if len(ids) > 0 {
|
||||
if err := a.ctl.Reconciler().RestartPaced(reconciler.RestartPace{}, false, ids...); err != nil {
|
||||
note = "published, but " + v.Model + "'s replicas could not be restarted to load it: " + err.Error()
|
||||
} else {
|
||||
note = "published: " + v.Model + "'s replicas are restarting to load it; then ask for model " + j.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := a.train.Store.SetTrainNote(ctx, j.ID, note); err != nil {
|
||||
a.trainLog().Warn("could not record a trained adapter's publishing", "job", j.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// untarAll reads a gzipped tar of a PEFT directory's files.
|
||||
|
||||
Reference in new issue
Block a user