Customise a model from the portal: a file, a model, a button
A page for anyone in a project, built for the one who knows least: an Excel, CSV or JSONL file of questions and answers, previewed; a model; a name; who may use it; and a button. The weights to train on are found from where the model's came (a GGUF repository's base model), the card's memory from their size, the node as the one with the most free. When it is done the adapter is published by itself (a llama.cpp model's replicas replaced to load it) and is a model of its own in the chat and the API. The project's by default; its members share it wider (confirmed, and never wider than the knowledge bases its pairs came from) or delete it. One trains at a time per project. ctl train start needs only --model and --data now, and --publish publishes when done. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
1 parent
fcda4d3f85
commit
a53b2f4be9
33 files changed
+2199
-134
No files matched your search
@@ -10,15 +10,20 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log/slog"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
@@ -36,6 +41,11 @@ type TrainStore interface {
|
||||
TrainArtifact(ctx context.Context, job, kind string) ([]byte, error)
|
||||
TrainArtifactKinds(ctx context.Context, job string) ([]string, error)
|
||||
DeleteTrainJob(ctx context.Context, id string) error
|
||||
QueuedTrainJobs(ctx context.Context) ([]store.TrainJob, error)
|
||||
ActiveTrainJobs(ctx context.Context) ([]store.TrainJob, error)
|
||||
TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]store.TrainJob, error)
|
||||
SetTrainOwner(ctx context.Context, id, owner string) error
|
||||
SetTrainNote(ctx context.Context, id, message string) error
|
||||
}
|
||||
|
||||
// TrainAdmin is what the training endpoints need.
|
||||
@@ -45,6 +55,27 @@ type TrainAdmin struct {
|
||||
// Mirror keeps what a job made where the nodes fetch weights; nil
|
||||
// leaves publishing off.
|
||||
Mirror TrainMirror
|
||||
// Registry finds the weights to train a catalog model on, from where
|
||||
// its own came; nil has every job name them.
|
||||
Registry TrainRegistry
|
||||
// Log says what publishing by itself could not do.
|
||||
Log *slog.Logger
|
||||
|
||||
mu sync.Mutex
|
||||
bases map[string]trainBaseInfo // by the model's origin repository
|
||||
}
|
||||
|
||||
// TrainRegistry reads a repository of the model registry.
|
||||
type TrainRegistry interface {
|
||||
RepoAt(ctx context.Context, id, rev string) (hf.Repo, error)
|
||||
}
|
||||
|
||||
// trainBaseInfo is the safetensors weights a model is trained on.
|
||||
type trainBaseInfo struct {
|
||||
repo, revision string
|
||||
bytes int64
|
||||
err error
|
||||
at time.Time
|
||||
}
|
||||
|
||||
// TrainMirror is the model mirror, as publishing needs it.
|
||||
@@ -66,11 +97,11 @@ type TrainPublishView struct {
|
||||
// SetTrain enables the training endpoints.
|
||||
func (a *API) SetTrain(t *TrainAdmin) { a.train = t }
|
||||
|
||||
// SetTrainMirror gives publishing the mirror, which is set up after the
|
||||
// training endpoints are.
|
||||
func (a *API) SetTrainMirror(m TrainMirror) {
|
||||
// SetTrainMirror gives publishing the mirror, and finding a model's
|
||||
// weights the registry, which are set up after the training endpoints are.
|
||||
func (a *API) SetTrainMirror(m TrainMirror, reg TrainRegistry) {
|
||||
if a.train != nil {
|
||||
a.train.Mirror = m
|
||||
a.train.Mirror, a.train.Registry = m, reg
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,13 +112,23 @@ type TrainRequest struct {
|
||||
// Model is the catalog model the adapter is for.
|
||||
Model string `json:"model"`
|
||||
// BaseRepo and BaseRevision are the weights trained on: the model's own
|
||||
// on its registry, in safetensors (Hugging Face), not a GGUF.
|
||||
BaseRepo string `json:"base_repo"`
|
||||
// on its registry, in safetensors (Hugging Face), not a GGUF. Empty
|
||||
// finds them from where the model's came: the repository itself, or the
|
||||
// one its card says it was made from.
|
||||
BaseRepo string `json:"base_repo,omitempty"`
|
||||
BaseRevision string `json:"base_revision,omitempty"`
|
||||
// Node trains it; GPUMemoryMiB is what it needs of a card, which can be
|
||||
// what an engine there leaves free.
|
||||
Node string `json:"node"`
|
||||
GPUMemoryMiB int `json:"gpu_memory_mib"`
|
||||
// Node trains it; empty is the first with a card that has enough free.
|
||||
// GPUMemoryMiB is what it needs of a card, which can be what an engine
|
||||
// there leaves free; 0 is worked out from the weights' size.
|
||||
Node string `json:"node,omitempty"`
|
||||
GPUMemoryMiB int `json:"gpu_memory_mib,omitempty"`
|
||||
// Project is the project the job is started in: its members see and
|
||||
// manage it, and only one of its jobs trains at a time.
|
||||
Project string `json:"project,omitempty"`
|
||||
// Publish puts what the job makes on the mirror and makes it the
|
||||
// model's adapter by itself when the job is done, restarting a llama.cpp
|
||||
// model's replicas to load it.
|
||||
Publish bool `json:"publish,omitempty"`
|
||||
// Owner is who the adapter will belong to: org:, team:, project: (by
|
||||
// path or ID), user: or key:. Only callers there see it. When the
|
||||
// pairs come from one knowledge base, it defaults to that collection's
|
||||
@@ -129,6 +170,8 @@ type TrainJobView struct {
|
||||
StartedAt time.Time `json:"started_at,omitzero"`
|
||||
FinishedAt time.Time `json:"finished_at,omitzero"`
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
Project string `json:"project,omitempty"`
|
||||
Publish bool `json:"publish,omitempty"`
|
||||
// Artifacts are what it made: peft (a gzipped tar), gguf.
|
||||
Artifacts []string `json:"artifacts,omitempty"`
|
||||
}
|
||||
@@ -137,7 +180,8 @@ func trainView(j store.TrainJob) TrainJobView {
|
||||
return TrainJobView{ID: j.ID, Name: j.Name, Model: j.Model, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision,
|
||||
Node: j.Node, Owner: j.Owner, GPUMemoryMiB: j.GPUMemoryMiB, Params: j.Params, GGUF: j.GGUF, Pairs: j.Pairs, State: j.State,
|
||||
Step: j.Step, Steps: j.Steps, Loss: j.Loss, Message: j.Message, PeakMiB: j.PeakMiB, CreatedAt: j.CreatedAt,
|
||||
StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy}
|
||||
StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy,
|
||||
Project: j.Project, Publish: j.Publish}
|
||||
}
|
||||
|
||||
func (a *API) registerTrain(mux Mux) {
|
||||
@@ -178,50 +222,122 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
bad := func(format string, args ...any) {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf(format, args...))
|
||||
j, err := a.StartTraining(r.Context(), in, actorOf(r))
|
||||
if err != nil {
|
||||
writeTrainErr(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusCreated, trainView(j))
|
||||
}
|
||||
|
||||
// trainErr is a refusal, with the status it is answered with.
|
||||
type trainErr struct {
|
||||
code int
|
||||
msg string
|
||||
}
|
||||
|
||||
func (e *trainErr) Error() string { return e.msg }
|
||||
|
||||
func refused(code int, format string, args ...any) error {
|
||||
return &trainErr{code: code, msg: fmt.Sprintf(format, args...)}
|
||||
}
|
||||
|
||||
func writeTrainErr(w http.ResponseWriter, err error) {
|
||||
var te *trainErr
|
||||
if errors.As(err, &te) {
|
||||
writeError(w, te.code, te.msg)
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
}
|
||||
|
||||
// TrainRefusal is the reason a training request was refused, for someone
|
||||
// to read, and whether it was the request's fault rather than the
|
||||
// cluster's.
|
||||
func TrainRefusal(err error) (string, bool) {
|
||||
if err == nil {
|
||||
return "", false
|
||||
}
|
||||
var te *trainErr
|
||||
if errors.As(err, &te) {
|
||||
return te.msg, te.code < 500
|
||||
}
|
||||
return err.Error(), false
|
||||
}
|
||||
|
||||
// StartTraining checks a job and queues it, as actor.
|
||||
func (a *API) StartTraining(ctx context.Context, in TrainRequest, actor string) (store.TrainJob, error) {
|
||||
if a.train == nil {
|
||||
return store.TrainJob{}, refused(http.StatusNotImplemented,
|
||||
"training needs the store and the nodes' agents, which this control plane runs without")
|
||||
}
|
||||
bad := func(format string, args ...any) (store.TrainJob, error) {
|
||||
return store.TrainJob{}, refused(http.StatusBadRequest, format, args...)
|
||||
}
|
||||
if !adapterName.MatchString(in.Name) {
|
||||
bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name)
|
||||
return
|
||||
return bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name)
|
||||
}
|
||||
m, ok := a.ctl.Catalog().Resolve(in.Model)
|
||||
if !ok {
|
||||
bad("no model %s in the catalog", in.Model)
|
||||
return
|
||||
return bad("no model %s in the catalog", in.Model)
|
||||
}
|
||||
if _, _, _, taken := a.ctl.Catalog().ResolveAdapter(in.Name); taken || in.Name == m.Name {
|
||||
bad("%s is already a model's or an adapter's name", in.Name)
|
||||
return
|
||||
return bad("%s is already a model's or an adapter's name", in.Name)
|
||||
}
|
||||
if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") {
|
||||
bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo)
|
||||
return
|
||||
if _, taken := a.ctl.Catalog().Resolve(in.Name); taken {
|
||||
return bad("%s is already a model's or an adapter's name", in.Name)
|
||||
}
|
||||
if _, ok := a.ctl.Inventory().Get(in.Node); !ok {
|
||||
bad("no node %s", in.Node)
|
||||
return
|
||||
queued, err := a.train.Store.QueuedTrainJobs(ctx)
|
||||
if err != nil {
|
||||
return store.TrainJob{}, err
|
||||
}
|
||||
active, err := a.train.Store.ActiveTrainJobs(ctx)
|
||||
if err != nil {
|
||||
return store.TrainJob{}, err
|
||||
}
|
||||
for _, j := range append(queued, active...) {
|
||||
if j.Name == in.Name {
|
||||
return bad("%s is already being trained (job %s)", in.Name, j.ID)
|
||||
}
|
||||
if in.Project != "" && j.Project == in.Project {
|
||||
return store.TrainJob{}, refused(http.StatusConflict,
|
||||
"this project already has a model training (%s): one at a time, so that one project does not hold every card", j.Name)
|
||||
}
|
||||
}
|
||||
var weights int64
|
||||
if in.BaseRepo == "" {
|
||||
b := a.trainBase(ctx, m)
|
||||
if b.err != nil {
|
||||
return store.TrainJob{}, b.err
|
||||
}
|
||||
in.BaseRepo, in.BaseRevision, weights = b.repo, b.revision, b.bytes
|
||||
} else if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") {
|
||||
return bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo)
|
||||
}
|
||||
if in.Node != "" {
|
||||
if _, ok := a.ctl.Inventory().Get(in.Node); !ok {
|
||||
return bad("no node %s", in.Node)
|
||||
}
|
||||
}
|
||||
if in.GPUMemoryMiB <= 0 {
|
||||
bad("gpu_memory_mib is what the job needs of a card, in MiB")
|
||||
return
|
||||
if weights <= 0 {
|
||||
return bad("gpu_memory_mib is what the job needs of a card, in MiB")
|
||||
}
|
||||
in.GPUMemoryMiB = TrainMemoryMiB(weights)
|
||||
}
|
||||
pairs, colls, err := countPairs(in.Data)
|
||||
if err != nil {
|
||||
bad("%v", err)
|
||||
return
|
||||
return bad("%v", err)
|
||||
}
|
||||
owner, err := a.trainOwner(r.Context(), in, colls)
|
||||
owner, err := a.trainOwner(ctx, in, colls)
|
||||
if err != nil {
|
||||
bad("%v", err)
|
||||
return
|
||||
return bad("%v", err)
|
||||
}
|
||||
for k, v := range in.Params {
|
||||
lim, known := trainParams[k]
|
||||
f, isNum := v.(float64)
|
||||
if !known || !isNum || f < lim[0] || f > lim[1] {
|
||||
bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k)
|
||||
return
|
||||
return bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k)
|
||||
}
|
||||
}
|
||||
params := ""
|
||||
@@ -240,15 +356,132 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
j := store.TrainJob{ID: store.NewID("train-"), Name: in.Name, Model: m.Name, BaseRepo: in.BaseRepo,
|
||||
BaseRevision: in.BaseRevision, Node: in.Node, Owner: owner, GPUMemoryMiB: in.GPUMemoryMiB, Params: params, GGUF: gguf,
|
||||
Data: []byte(in.Data), Pairs: pairs, CreatedBy: actorOf(r)}
|
||||
if err := a.train.Store.CreateTrainJob(r.Context(), j); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
Data: []byte(in.Data), Pairs: pairs, CreatedBy: actor, Project: in.Project, Publish: in.Publish}
|
||||
if err := a.train.Store.CreateTrainJob(ctx, j); err != nil {
|
||||
return store.TrainJob{}, err
|
||||
}
|
||||
// At once rather than at the next look, when the node is here.
|
||||
go a.train.Service.Dispatch(context.WithoutCancel(r.Context()))
|
||||
go a.train.Service.Dispatch(context.WithoutCancel(ctx))
|
||||
j.State = store.TrainQueued
|
||||
writeJSON(w, http.StatusCreated, trainView(j))
|
||||
return j, nil
|
||||
}
|
||||
|
||||
// TrainMemoryMiB is what training a LoRA adapter on weights of this size
|
||||
// takes of a card: the weights in 16 bits, as much again for what passes
|
||||
// through them, and the CUDA context. Measured at 2.3 GB for a 0.5B model.
|
||||
func TrainMemoryMiB(weightBytes int64) int {
|
||||
mib := int(weightBytes>>20)*2 + 1024
|
||||
return (mib + 255) / 256 * 256
|
||||
}
|
||||
|
||||
// trainBaseFor is how long what the registry said of a model's weights is
|
||||
// kept: it changes when the model is updated, which is rare.
|
||||
const trainBaseFor = time.Hour
|
||||
|
||||
// trainBase is the safetensors weights to train m on: its own origin's,
|
||||
// when they are safetensors, or those its card says it was made from — a
|
||||
// GGUF's base model.
|
||||
func (a *API) trainBase(ctx context.Context, m catalog.Model) trainBaseInfo {
|
||||
if m.Origin == nil || m.Origin.Repo == "" {
|
||||
return trainBaseInfo{err: refused(http.StatusBadRequest, "%s's weights did not come from the model registry, so "+
|
||||
"what to train it on is not known: an administrator can name them (llmgw ctl train start --base-repo)", m.Name)}
|
||||
}
|
||||
if a.train.Registry == nil {
|
||||
return trainBaseInfo{err: refused(http.StatusNotImplemented, "the model registry is not reachable from here "+
|
||||
"(mirror.registry): the weights to train %s on cannot be found", m.Name)}
|
||||
}
|
||||
a.train.mu.Lock()
|
||||
b, ok := a.train.bases[m.Origin.Repo]
|
||||
a.train.mu.Unlock()
|
||||
if ok && time.Since(b.at) < trainBaseFor {
|
||||
return b
|
||||
}
|
||||
b = a.findTrainBase(ctx, m)
|
||||
if b.err != nil {
|
||||
var te *trainErr
|
||||
if !errors.As(b.err, &te) {
|
||||
return b // the registry did not answer: asked again next time
|
||||
}
|
||||
}
|
||||
b.at = time.Now()
|
||||
a.train.mu.Lock()
|
||||
if a.train.bases == nil {
|
||||
a.train.bases = map[string]trainBaseInfo{}
|
||||
}
|
||||
a.train.bases[m.Origin.Repo] = b
|
||||
a.train.mu.Unlock()
|
||||
return b
|
||||
}
|
||||
|
||||
func (a *API) findTrainBase(ctx context.Context, m catalog.Model) trainBaseInfo {
|
||||
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
|
||||
defer cancel()
|
||||
repo, err := a.train.Registry.RepoAt(ctx, m.Origin.Repo, "")
|
||||
if err != nil {
|
||||
return trainBaseInfo{err: fmt.Errorf("asking the model registry about %s: %w", m.Origin.Repo, err)}
|
||||
}
|
||||
candidates := []hf.Repo{repo}
|
||||
for _, id := range repo.BaseModel {
|
||||
if id == repo.ID || strings.Count(id, "/") != 1 {
|
||||
continue
|
||||
}
|
||||
r, err := a.train.Registry.RepoAt(ctx, id, "")
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
candidates = append(candidates, r)
|
||||
}
|
||||
gated := ""
|
||||
for _, r := range candidates {
|
||||
var n int64
|
||||
for _, f := range r.Files {
|
||||
if strings.HasSuffix(strings.ToLower(f.Path), ".safetensors") {
|
||||
n += f.Bytes
|
||||
}
|
||||
}
|
||||
if n == 0 {
|
||||
continue
|
||||
}
|
||||
if r.Gated.Is() {
|
||||
gated = r.ID
|
||||
continue
|
||||
}
|
||||
return trainBaseInfo{repo: r.ID, revision: r.Revision, bytes: n}
|
||||
}
|
||||
if gated != "" {
|
||||
return trainBaseInfo{err: refused(http.StatusBadRequest, "the weights to train %s on, %s, are gated on the registry: "+
|
||||
"the nodes cannot fetch them without an account that accepted their terms", m.Name, gated)}
|
||||
}
|
||||
return trainBaseInfo{err: refused(http.StatusBadRequest, "neither %s, where %s's weights came from, nor the model its card "+
|
||||
"names has safetensors weights to train on", m.Origin.Repo, m.Name)}
|
||||
}
|
||||
|
||||
// PickTrainNode is the node whose freest card has the most memory free,
|
||||
// if that is need, among those ready and not busy.
|
||||
func (a *API) PickTrainNode(need int, busy map[string]bool) string {
|
||||
best, bestFree := "", 0
|
||||
for _, n := range a.ctl.Inventory().Nodes() {
|
||||
if busy[n.ID] || n.State != inventory.Ready {
|
||||
continue
|
||||
}
|
||||
free := 0
|
||||
if n.Load != nil {
|
||||
for _, g := range n.Load.GPUs {
|
||||
if g.MemoryTotalMB > 0 {
|
||||
free = max(free, int(g.MemoryTotalMB-g.MemoryUsedMB))
|
||||
}
|
||||
}
|
||||
}
|
||||
if free == 0 {
|
||||
for _, g := range n.GPUs {
|
||||
free = max(free, g.MemoryTotalMB-g.MemoryUsedMB)
|
||||
}
|
||||
}
|
||||
if free >= need && free > bestFree {
|
||||
best, bestFree = n.ID, free
|
||||
}
|
||||
}
|
||||
return best
|
||||
}
|
||||
|
||||
// trainOwner is who a job's adapter will belong to, as scope:ID: one that
|
||||
@@ -453,73 +686,74 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) {
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if a.train.Mirror == nil {
|
||||
writeError(w, http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+
|
||||
"llmgw ctl train download, then llmgw mirror import there, does it by hand")
|
||||
v, err := a.publishTrained(r.Context(), j, actorOf(r))
|
||||
if err != nil {
|
||||
writeTrainErr(w, err)
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
// publishTrained puts what a job made on the mirror and makes it the
|
||||
// model's adapter: the GGUF for a model llama.cpp serves, the PEFT
|
||||
// directory for one vLLM serves.
|
||||
func (a *API) publishTrained(ctx context.Context, j store.TrainJob, actor string) (TrainPublishView, error) {
|
||||
var none TrainPublishView
|
||||
if a.train.Mirror == nil {
|
||||
return none, refused(http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+
|
||||
"llmgw ctl train download, then llmgw mirror import there, does it by hand")
|
||||
}
|
||||
if a.models == nil {
|
||||
writeError(w, http.StatusNotImplemented, "this control plane has no store for the catalog")
|
||||
return
|
||||
return none, refused(http.StatusNotImplemented, "this control plane has no store for the catalog")
|
||||
}
|
||||
if j.State != store.TrainDone {
|
||||
writeError(w, http.StatusConflict, "the job has not finished: it is "+j.State)
|
||||
return
|
||||
return none, refused(http.StatusConflict, "%s", "the job has not finished: it is "+j.State)
|
||||
}
|
||||
m, found := a.ctl.Catalog().Resolve(j.Model)
|
||||
if !found {
|
||||
writeError(w, http.StatusConflict, "the model "+j.Model+" is no longer in the catalog")
|
||||
return
|
||||
return none, refused(http.StatusConflict, "%s", "the model "+j.Model+" is no longer in the catalog")
|
||||
}
|
||||
for _, ad := range m.Adapters {
|
||||
if ad.Name == j.Name {
|
||||
writeError(w, http.StatusConflict, j.Name+" is already "+m.Name+"'s adapter; train under another name")
|
||||
return
|
||||
return none, refused(http.StatusConflict, "%s", j.Name+" is already "+m.Name+"'s adapter; train under another name")
|
||||
}
|
||||
}
|
||||
llama := slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" })
|
||||
source := "trained by llmgw job " + j.ID + " on " + j.Node + " from " + j.BaseRepo
|
||||
var url string
|
||||
if llama {
|
||||
b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "gguf")
|
||||
b, err := a.train.Store.TrainArtifact(ctx, j.ID, "gguf")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf")
|
||||
return
|
||||
return none, refused(http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf")
|
||||
}
|
||||
res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, j.Name+".gguf", b, source)
|
||||
res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, j.Name+".gguf", b, source)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, "keeping it on the mirror: "+err.Error())
|
||||
return
|
||||
return none, refused(http.StatusBadGateway, "%s", "keeping it on the mirror: "+err.Error())
|
||||
}
|
||||
url = res.URL
|
||||
} else {
|
||||
b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "peft")
|
||||
b, err := a.train.Store.TrainArtifact(ctx, j.ID, "peft")
|
||||
if err != nil {
|
||||
writeError(w, http.StatusConflict, "the job made no PEFT adapter")
|
||||
return
|
||||
return none, refused(http.StatusConflict, "the job made no PEFT adapter")
|
||||
}
|
||||
files, err := untarAll(b)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
return none, err
|
||||
}
|
||||
for name, data := range files {
|
||||
res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, name, data, source)
|
||||
res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, name, data, source)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadGateway, "keeping "+name+" on the mirror: "+err.Error())
|
||||
return
|
||||
return none, refused(http.StatusBadGateway, "%s", "keeping "+name+" on the mirror: "+err.Error())
|
||||
}
|
||||
url = res.URL[:strings.LastIndex(res.URL, "/")+1]
|
||||
}
|
||||
}
|
||||
ad := catalog.Adapter{Name: j.Name, Source: url, Owner: j.Owner}
|
||||
if err := checkAdapters(r.Context(), m, []catalog.Adapter{ad}); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
if err := checkAdapters(ctx, m, []catalog.Adapter{ad}); err != nil {
|
||||
return none, refused(http.StatusBadRequest, "%v", err)
|
||||
}
|
||||
if _, err := a.models.SetAdapters(r.Context(), m.Name, append(slices.Clone(m.Adapters), ad), actorOf(r)); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
if _, err := a.models.SetAdapters(ctx, m.Name, append(slices.Clone(m.Adapters), ad), actor); err != nil {
|
||||
return none, refused(http.StatusBadRequest, "%v", err)
|
||||
}
|
||||
v := TrainPublishView{Adapter: j.Name, Model: m.Name, Source: url, Owner: j.Owner}
|
||||
if llama {
|
||||
@@ -528,7 +762,48 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) {
|
||||
} else {
|
||||
v.Note = "ask for model " + j.Name + "; a replica loads it on the first request"
|
||||
}
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
return v, nil
|
||||
}
|
||||
|
||||
func (a *API) trainLog() *slog.Logger {
|
||||
if a.train.Log == nil {
|
||||
return slog.New(slog.DiscardHandler)
|
||||
}
|
||||
return a.train.Log
|
||||
}
|
||||
|
||||
// TrainDone publishes what a job made, when it asked for that: on the
|
||||
// mirror, as its model's adapter, and for a llama.cpp model with the
|
||||
// model's replicas replaced, one at a time beside themselves, to load it.
|
||||
func (a *API) TrainDone(j store.TrainJob) {
|
||||
if !j.Publish || a.train == nil {
|
||||
return
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute)
|
||||
defer cancel()
|
||||
note := "published: ask for model " + j.Name
|
||||
v, err := a.publishTrained(ctx, j, "training:"+j.ID)
|
||||
if err != nil {
|
||||
note = "trained, but publishing it failed: " + err.Error()
|
||||
a.trainLog().Warn("a trained adapter could not be published", "job", j.ID, "adapter", j.Name, "err", err)
|
||||
} else if strings.HasPrefix(v.Note, "llama.cpp") {
|
||||
var ids []string
|
||||
for _, rep := range a.ctl.Reconciler().ReadyReplicas() {
|
||||
if rep.Model == v.Model {
|
||||
ids = append(ids, rep.ID)
|
||||
}
|
||||
}
|
||||
if len(ids) > 0 {
|
||||
if err := a.ctl.Reconciler().RestartPaced(reconciler.RestartPace{}, false, ids...); err != nil {
|
||||
note = "published, but " + v.Model + "'s replicas could not be restarted to load it: " + err.Error()
|
||||
} else {
|
||||
note = "published: " + v.Model + "'s replicas are restarting to load it; then ask for model " + j.Name
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := a.train.Store.SetTrainNote(ctx, j.ID, note); err != nil {
|
||||
a.trainLog().Warn("could not record a trained adapter's publishing", "job", j.ID, "err", err)
|
||||
}
|
||||
}
|
||||
|
||||
// untarAll reads a gzipped tar of a PEFT directory's files.
|
||||
|
||||
@@ -0,0 +1,114 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"slices"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Training from the portal's "customise a model" page: the page decides who
|
||||
// may do what (members of the job's project), and these do it.
|
||||
|
||||
// TrainableModels are the catalog's models an adapter can be trained for
|
||||
// from the portal: chat models llama.cpp or vLLM serve whose weights came
|
||||
// from the registry, so the weights to train on can be found.
|
||||
func (a *API) TrainableModels() []string {
|
||||
var out []string
|
||||
for _, m := range a.ctl.Catalog().Models() {
|
||||
if m.IsGuard() || m.IsExternal() || m.Capabilities.Embeddings || m.MakesImages() || m.MakesMeshes() ||
|
||||
m.Origin == nil || m.Origin.Repo == "" {
|
||||
continue
|
||||
}
|
||||
if slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" || p.Engine == "vllm" }) {
|
||||
out = append(out, m.Name)
|
||||
}
|
||||
}
|
||||
slices.Sort(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// Trainings are the jobs started in projects, or by createdBy outside any.
|
||||
func (a *API) Trainings(ctx context.Context, projects []string, createdBy string) ([]store.TrainJob, error) {
|
||||
if a.train == nil {
|
||||
return nil, nil
|
||||
}
|
||||
return a.train.Store.TrainJobsFor(ctx, projects, createdBy, 100)
|
||||
}
|
||||
|
||||
// Training is one job.
|
||||
func (a *API) Training(ctx context.Context, id string) (store.TrainJob, error) {
|
||||
if a.train == nil {
|
||||
return store.TrainJob{}, store.ErrNotFound
|
||||
}
|
||||
return a.train.Store.TrainJob(ctx, id)
|
||||
}
|
||||
|
||||
// Published is whether a job's adapter is its model's in the catalog.
|
||||
func (a *API) Published(j store.TrainJob) bool {
|
||||
m, ok := a.ctl.Catalog().Resolve(j.Model)
|
||||
return ok && slices.ContainsFunc(m.Adapters, func(ad catalog.Adapter) bool { return ad.Name == j.Name })
|
||||
}
|
||||
|
||||
// ShareTraining changes who a job's adapter belongs to — owner as scope:ID,
|
||||
// or "" for every caller its model allows — on the job and, once
|
||||
// published, in the catalog. Refused wider than the knowledge bases its
|
||||
// pairs came from may be read.
|
||||
func (a *API) ShareTraining(ctx context.Context, id, owner, actor string) error {
|
||||
j, err := a.Training(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_, colls, err := countPairs(string(j.Data))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
resolved, err := a.trainOwner(ctx, TrainRequest{Owner: owner, Public: owner == ""}, colls)
|
||||
if err != nil {
|
||||
return refused(http.StatusBadRequest, "%v", err)
|
||||
}
|
||||
if err := a.train.Store.SetTrainOwner(ctx, id, resolved); err != nil {
|
||||
return err
|
||||
}
|
||||
m, ok := a.ctl.Catalog().Resolve(j.Model)
|
||||
if !ok || a.models == nil {
|
||||
return nil
|
||||
}
|
||||
ads := slices.Clone(m.Adapters)
|
||||
i := slices.IndexFunc(ads, func(ad catalog.Adapter) bool { return ad.Name == j.Name })
|
||||
if i < 0 {
|
||||
return nil
|
||||
}
|
||||
ads[i].Owner = resolved
|
||||
_, err = a.models.SetAdapters(ctx, m.Name, ads, actor)
|
||||
return err
|
||||
}
|
||||
|
||||
// DeleteTraining stops a job if it trains, takes its adapter out of the
|
||||
// catalog if it was published, and forgets it.
|
||||
func (a *API) DeleteTraining(ctx context.Context, id, actor string) error {
|
||||
j, err := a.Training(ctx, id)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !store.TrainOver(j.State) {
|
||||
if err := a.train.Service.Cancel(ctx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if m, ok := a.ctl.Catalog().Resolve(j.Model); ok && a.models != nil {
|
||||
ads := slices.DeleteFunc(slices.Clone(m.Adapters), func(ad catalog.Adapter) bool { return ad.Name == j.Name })
|
||||
if len(ads) != len(m.Adapters) {
|
||||
if _, err := a.models.SetAdapters(ctx, m.Name, ads, actor); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
if err := a.train.Store.DeleteTrainJob(ctx, id); err != nil && !errors.Is(err, store.ErrNotFound) {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,137 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
type fakeRegistry struct {
|
||||
repos map[string]hf.Repo
|
||||
asked int
|
||||
}
|
||||
|
||||
func (f *fakeRegistry) RepoAt(_ context.Context, id, _ string) (hf.Repo, error) {
|
||||
f.asked++
|
||||
r, ok := f.repos[id]
|
||||
if !ok {
|
||||
return hf.Repo{}, errors.New("404")
|
||||
}
|
||||
return r, nil
|
||||
}
|
||||
|
||||
func TestAPortalTrainingFindsItsWeightsAndMemoryAndOneTrainsPerProject(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
models := append(testModels(),
|
||||
catalog.Model{Name: "small", Source: "/s", Origin: &catalog.Origin{Repo: "bartowski/Small-GGUF"},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
|
||||
catalog.Model{Name: "gated", Source: "/g", Origin: &catalog.Origin{Repo: "meta/Gated"},
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}},
|
||||
catalog.Model{Name: "local", Source: "/l", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
|
||||
a, _ := setup(t, models, gpuNode("cuda01", 1))
|
||||
s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
reg := &fakeRegistry{repos: map[string]hf.Repo{
|
||||
"bartowski/Small-GGUF": {ID: "bartowski/Small-GGUF", Revision: "g1", BaseModel: []string{"Org/Small"},
|
||||
Files: []hf.File{{Path: "small-Q4_K_M.gguf", Bytes: 400 << 20}}},
|
||||
"Org/Small": {ID: "Org/Small", Revision: "abc", Files: []hf.File{{Path: "model.safetensors", Bytes: 1000 << 20},
|
||||
{Path: "config.json", Bytes: 1 << 10}}},
|
||||
"meta/Gated": {ID: "meta/Gated", Gated: hf.Gated{Reason: "manual"},
|
||||
Files: []hf.File{{Path: "model.safetensors", Bytes: 1 << 30}}},
|
||||
}}
|
||||
a.SetTrain(&TrainAdmin{Store: s, Service: train.New(s, noAgents{}, nil)})
|
||||
a.SetTrainMirror(nil, reg)
|
||||
|
||||
if got := a.TrainableModels(); strings.Join(got, ",") != "gated,small" {
|
||||
t.Errorf("trainable: %v", got)
|
||||
}
|
||||
data := train.NDJSON([]train.Pair{{Q: "a", A: "b"}})
|
||||
req := TrainRequest{Name: "tone", Model: "small", Data: data, Owner: "project:p1", Project: "p1", Publish: true}
|
||||
j, err := a.StartTraining(ctx, req, "ayse@x")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j.BaseRepo != "Org/Small" || j.BaseRevision != "abc" || j.Node != "" || j.GPUMemoryMiB != TrainMemoryMiB(1000<<20) ||
|
||||
!j.Publish || j.Project != "p1" || j.CreatedBy != "ayse@x" || !j.GGUF {
|
||||
t.Errorf("queued %+v", j)
|
||||
}
|
||||
if j.GPUMemoryMiB != 3072 {
|
||||
t.Errorf("1000 MiB of weights take %d MiB to train", j.GPUMemoryMiB)
|
||||
}
|
||||
|
||||
req.Name = "tone2"
|
||||
if _, err := a.StartTraining(ctx, req, "mehmet@x"); err == nil || !strings.Contains(err.Error(), "already has a model training") {
|
||||
t.Errorf("a second in the project: %v", err)
|
||||
}
|
||||
req.Name, req.Project, req.Owner = "tone", "p2", "project:p2"
|
||||
if _, err := a.StartTraining(ctx, req, "x"); err == nil || !strings.Contains(err.Error(), "already being trained") {
|
||||
t.Errorf("the same name twice: %v", err)
|
||||
}
|
||||
req.Name = "tone3"
|
||||
asked := reg.asked
|
||||
if _, err := a.StartTraining(ctx, req, "x"); err != nil {
|
||||
t.Errorf("another project: %v", err)
|
||||
}
|
||||
if reg.asked != asked {
|
||||
t.Errorf("the registry was asked again: %d", reg.asked-asked)
|
||||
}
|
||||
for model, says := range map[string]string{"gated": "gated", "local": "did not come from the model registry"} {
|
||||
req.Model, req.Name, req.Project = model, "x-"+model, "p-"+model
|
||||
_, err := a.StartTraining(ctx, req, "x")
|
||||
if err == nil || !strings.Contains(err.Error(), says) {
|
||||
t.Errorf("%s: %v", model, err)
|
||||
}
|
||||
if msg, theirs := TrainRefusal(err); !theirs || msg == "" {
|
||||
t.Errorf("%s is not the request's fault: %q", model, msg)
|
||||
}
|
||||
}
|
||||
|
||||
jobs, err := a.Trainings(ctx, []string{"p1"}, "nobody")
|
||||
if err != nil || len(jobs) != 1 || jobs[0].Name != "tone" {
|
||||
t.Errorf("p1's: %+v %v", jobs, err)
|
||||
}
|
||||
if err := a.ShareTraining(ctx, j.ID, "", "ayse@x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got, _ := a.Training(ctx, j.ID); got.Owner != "" {
|
||||
t.Errorf("shared with everyone: %q", got.Owner)
|
||||
}
|
||||
if err := a.DeleteTraining(ctx, j.ID, "ayse@x"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := a.Training(ctx, j.ID); !errors.Is(err, store.ErrNotFound) {
|
||||
t.Errorf("deleted: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestATrainingGoesToTheNodeWithTheMostFree(t *testing.T) {
|
||||
busy := gpuNode("busy", 1)
|
||||
busy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 23000}}}
|
||||
roomy := gpuNode("roomy", 1)
|
||||
roomy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 4000}}}
|
||||
cpu := inventory.Node{ID: "cpu", State: inventory.Ready}
|
||||
small := gpuNode("small", 1)
|
||||
small.GPUs = []discovery.GPU{{Index: 0, MemoryTotalMB: 8000}}
|
||||
a, _ := setup(t, testModels(), busy, roomy, cpu, small)
|
||||
if got := a.PickTrainNode(3000, nil); got != "roomy" {
|
||||
t.Errorf("picked %q", got)
|
||||
}
|
||||
if got := a.PickTrainNode(3000, map[string]bool{"roomy": true}); got != "small" {
|
||||
t.Errorf("with roomy busy, picked %q", got)
|
||||
}
|
||||
if got := a.PickTrainNode(30000, nil); got != "" {
|
||||
t.Errorf("none has 30 GB, picked %q", got)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,205 @@
|
||||
package train
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/csv"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"path"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kb/extract"
|
||||
)
|
||||
|
||||
// Pairs from a file a person uploads: a spreadsheet of questions and
|
||||
// answers (Excel, CSV, tab-separated) or llmgw's own NDJSON of {"q", "a"}.
|
||||
// The columns are found by their header — Soru/Cevap, Question/Answer and
|
||||
// the like — or are the first two when the first row is not a header.
|
||||
|
||||
// Pair is one question and the answer to learn for it.
|
||||
type Pair struct {
|
||||
Q string `json:"q"`
|
||||
A string `json:"a"`
|
||||
}
|
||||
|
||||
// Read is what a file gave.
|
||||
type Read struct {
|
||||
Pairs []Pair
|
||||
// Skipped counts rows with a question and no answer, or the other way.
|
||||
Skipped int
|
||||
// Columns names the columns read, as the header has them; empty for
|
||||
// NDJSON and for a table with no header.
|
||||
Columns [2]string
|
||||
}
|
||||
|
||||
// MaxPairs bounds one file: past it a job trains for hours on a page that
|
||||
// was meant to take minutes.
|
||||
const MaxPairs = 20000
|
||||
|
||||
var (
|
||||
questionNames = []string{"soru", "sorular", "question", "questions", "q", "prompt", "instruction", "input", "girdi"}
|
||||
answerNames = []string{"cevap", "cevaplar", "yanıt", "yanit", "answer", "answers", "a", "response", "output", "completion", "çıktı", "cikti"}
|
||||
)
|
||||
|
||||
// ReadPairs reads a file's pairs; name says what kind it is.
|
||||
func ReadPairs(name string, b []byte) (Read, error) {
|
||||
var rows [][]string
|
||||
switch strings.ToLower(path.Ext(name)) {
|
||||
case ".xlsx", ".xlsm":
|
||||
r, err := extract.XLSXRows(b)
|
||||
if err != nil {
|
||||
return Read{}, fmt.Errorf("the Excel file could not be read: %w", err)
|
||||
}
|
||||
rows = r
|
||||
case ".jsonl", ".ndjson", ".json":
|
||||
return readNDJSON(b)
|
||||
case ".csv", ".tsv", ".txt":
|
||||
r, err := readDelimited(b)
|
||||
if err != nil {
|
||||
return Read{}, err
|
||||
}
|
||||
rows = r
|
||||
case ".xls":
|
||||
return Read{}, errors.New("an old Excel file (.xls): save it as .xlsx and choose it again")
|
||||
default:
|
||||
return Read{}, errors.New("choose an Excel (.xlsx), CSV or JSONL file")
|
||||
}
|
||||
return fromRows(rows)
|
||||
}
|
||||
|
||||
func readDelimited(b []byte) ([][]string, error) {
|
||||
b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf"))
|
||||
if !utf8.Valid(b) {
|
||||
return nil, errors.New("the file is not UTF-8 text: save it as \"CSV UTF-8\"")
|
||||
}
|
||||
first, _, _ := strings.Cut(string(b), "\n")
|
||||
// Whichever separator the header line has most of: Excel writes ";"
|
||||
// where the decimal mark is a comma, as it is in Turkish.
|
||||
sep, most := ',', strings.Count(first, ",")
|
||||
for _, c := range []rune{';', '\t'} {
|
||||
if n := strings.Count(first, string(c)); n > most {
|
||||
sep, most = c, n
|
||||
}
|
||||
}
|
||||
r := csv.NewReader(bytes.NewReader(b))
|
||||
r.Comma, r.FieldsPerRecord, r.LazyQuotes = sep, -1, true
|
||||
rows, err := r.ReadAll()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the file could not be read as a table: %w", err)
|
||||
}
|
||||
return rows, nil
|
||||
}
|
||||
|
||||
func fromRows(rows [][]string) (Read, error) {
|
||||
var out Read
|
||||
qc, ac := 0, 1
|
||||
if len(rows) > 0 {
|
||||
q, a := -1, -1
|
||||
for i, c := range rows[0] {
|
||||
n := strings.ToLower(strings.TrimSpace(c))
|
||||
switch {
|
||||
case q < 0 && contains(questionNames, n):
|
||||
q = i
|
||||
case a < 0 && contains(answerNames, n):
|
||||
a = i
|
||||
}
|
||||
}
|
||||
if q >= 0 && a >= 0 {
|
||||
qc, ac = q, a
|
||||
out.Columns = [2]string{strings.TrimSpace(rows[0][q]), strings.TrimSpace(rows[0][a])}
|
||||
rows = rows[1:]
|
||||
}
|
||||
}
|
||||
for _, r := range rows {
|
||||
var q, a string
|
||||
if qc < len(r) {
|
||||
q = strings.TrimSpace(r[qc])
|
||||
}
|
||||
if ac < len(r) {
|
||||
a = strings.TrimSpace(r[ac])
|
||||
}
|
||||
switch {
|
||||
case q == "" && a == "":
|
||||
case q == "" || a == "":
|
||||
out.Skipped++
|
||||
default:
|
||||
out.Pairs = append(out.Pairs, Pair{Q: q, A: a})
|
||||
}
|
||||
}
|
||||
return out, check(out)
|
||||
}
|
||||
|
||||
func contains(names []string, n string) bool {
|
||||
for _, x := range names {
|
||||
if x == n {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func readNDJSON(b []byte) (Read, error) {
|
||||
var out Read
|
||||
sc := bufio.NewScanner(bytes.NewReader(b))
|
||||
sc.Buffer(make([]byte, 64<<10), 1<<20)
|
||||
line := 0
|
||||
for sc.Scan() {
|
||||
line++
|
||||
l := bytes.TrimSpace(sc.Bytes())
|
||||
if len(l) == 0 {
|
||||
continue
|
||||
}
|
||||
var p struct {
|
||||
Q, A string
|
||||
Question, Answer string
|
||||
Prompt, Completion string
|
||||
}
|
||||
if err := json.Unmarshal(l, &p); err != nil {
|
||||
return Read{}, fmt.Errorf("line %d is not JSON: %v", line, err)
|
||||
}
|
||||
q, a := first(p.Q, p.Question, p.Prompt), first(p.A, p.Answer, p.Completion)
|
||||
if q == "" || a == "" {
|
||||
out.Skipped++
|
||||
continue
|
||||
}
|
||||
out.Pairs = append(out.Pairs, Pair{Q: q, A: a})
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return Read{}, err
|
||||
}
|
||||
return out, check(out)
|
||||
}
|
||||
|
||||
func first(s ...string) string {
|
||||
for _, x := range s {
|
||||
if x = strings.TrimSpace(x); x != "" {
|
||||
return x
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func check(r Read) error {
|
||||
switch {
|
||||
case len(r.Pairs) == 0:
|
||||
return errors.New("no question and answer found: the first column is the question and the second the answer, " +
|
||||
"or columns headed Soru and Cevap")
|
||||
case len(r.Pairs) > MaxPairs:
|
||||
return fmt.Errorf("%d pairs, more than the %d one training takes", len(r.Pairs), MaxPairs)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NDJSON is the pairs as a job's data.
|
||||
func NDJSON(pairs []Pair) string {
|
||||
var b strings.Builder
|
||||
enc := json.NewEncoder(&b)
|
||||
enc.SetEscapeHTML(false)
|
||||
for _, p := range pairs {
|
||||
_ = enc.Encode(p)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package train
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestPairsFromATableByItsHeaderOrItsFirstColumns(t *testing.T) {
|
||||
for _, c := range []struct {
|
||||
name, file string
|
||||
want []Pair
|
||||
skipped int
|
||||
cols [2]string
|
||||
}{
|
||||
{"headed.csv", "No;Cevap;Soru\n1;Yirmi gün.;İzin kaç gün?\n2;;Boş cevap\n3;Evet;Uzaktan çalışabilir miyim?\n",
|
||||
[]Pair{{"İzin kaç gün?", "Yirmi gün."}, {"Uzaktan çalışabilir miyim?", "Evet"}}, 1, [2]string{"Soru", "Cevap"}},
|
||||
{"plain.csv", "\ufeffİzin kaç gün?,\"Yirmi gün, yılda.\"\n\nMasraf?,Formla.\n",
|
||||
[]Pair{{"İzin kaç gün?", "Yirmi gün, yılda."}, {"Masraf?", "Formla."}}, 0, [2]string{}},
|
||||
{"tabs.tsv", "Question\tAnswer\nA?\tB.\n", []Pair{{"A?", "B."}}, 0, [2]string{"Question", "Answer"}},
|
||||
{"d.jsonl", `{"q":"A?","a":"B."}` + "\n" + `{"question":"C?","answer":"D."}` + "\n" + `{"q":"E?"}` + "\n",
|
||||
[]Pair{{"A?", "B."}, {"C?", "D."}}, 1, [2]string{}},
|
||||
} {
|
||||
got, err := ReadPairs(c.name, []byte(c.file))
|
||||
if err != nil {
|
||||
t.Errorf("%s: %v", c.name, err)
|
||||
continue
|
||||
}
|
||||
if len(got.Pairs) != len(c.want) || got.Skipped != c.skipped || got.Columns != c.cols {
|
||||
t.Errorf("%s: %+v", c.name, got)
|
||||
continue
|
||||
}
|
||||
for i := range c.want {
|
||||
if got.Pairs[i] != c.want[i] {
|
||||
t.Errorf("%s: pair %d = %+v, want %+v", c.name, i, got.Pairs[i], c.want[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestPairsRefusedWithAReason(t *testing.T) {
|
||||
for name, file := range map[string]string{
|
||||
"a.csv": "only one column\n",
|
||||
"a.xls": "x",
|
||||
"a.pdf": "x",
|
||||
"a.json": "not json\n",
|
||||
} {
|
||||
if _, err := ReadPairs(name, []byte(file)); err == nil {
|
||||
t.Errorf("%s read", name)
|
||||
}
|
||||
}
|
||||
var b strings.Builder
|
||||
for range MaxPairs + 1 {
|
||||
b.WriteString("x,y\n")
|
||||
}
|
||||
if _, err := ReadPairs("big.csv", []byte(b.String())); err == nil || !strings.Contains(err.Error(), "more than") {
|
||||
t.Errorf("too many: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNDJSONIsWhatAJobReads(t *testing.T) {
|
||||
got := NDJSON([]Pair{{"<b>?", "a & b"}})
|
||||
if got != `{"q":"<b>?","a":"a & b"}`+"\n" {
|
||||
t.Errorf("%q", got)
|
||||
}
|
||||
}
|
||||
@@ -35,11 +35,26 @@ type Service struct {
|
||||
agents Agents
|
||||
log *slog.Logger
|
||||
|
||||
// pick chooses the node for a job that names none, among those not
|
||||
// busy; "" is none can take it now. Nil leaves such jobs queued.
|
||||
pick func(needMiB int, busy map[string]bool) string
|
||||
// done is told of a job that finished, after what it made is kept.
|
||||
done func(j store.TrainJob)
|
||||
|
||||
mu sync.Mutex
|
||||
// pieces are artifacts arriving, by job and kind.
|
||||
pieces map[[2]string][]byte
|
||||
}
|
||||
|
||||
// SetPicker has jobs that name no node sent where pick says.
|
||||
func (s *Service) SetPicker(pick func(needMiB int, busy map[string]bool) string) { s.pick = pick }
|
||||
|
||||
// SetOnDone has done told of each job that finishes.
|
||||
func (s *Service) SetOnDone(done func(j store.TrainJob)) { s.done = done }
|
||||
|
||||
// Waiting is what a job that names no node says while no node can take it.
|
||||
const Waiting = "waiting for a card with enough memory free"
|
||||
|
||||
// New is a service.
|
||||
func New(s Store, a Agents, log *slog.Logger) *Service {
|
||||
if log == nil {
|
||||
@@ -81,13 +96,27 @@ func (s *Service) Dispatch(ctx context.Context) {
|
||||
busy[j.Node] = true
|
||||
}
|
||||
for _, q := range queued {
|
||||
if busy[q.Node] || !s.agents.Connected(q.Node) {
|
||||
node := q.Node
|
||||
if node == "" {
|
||||
if s.pick != nil {
|
||||
node = s.pick(q.GPUMemoryMiB, busy)
|
||||
}
|
||||
if node == "" {
|
||||
if q.Message != Waiting {
|
||||
q.Message = Waiting
|
||||
_ = s.store.UpdateTrainJob(ctx, q)
|
||||
}
|
||||
continue
|
||||
}
|
||||
}
|
||||
if busy[node] || !s.agents.Connected(node) {
|
||||
continue
|
||||
}
|
||||
j, err := s.store.TrainJob(ctx, q.ID)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
j.Node = node
|
||||
err = s.agents.Send(j.Node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_TrainStart{TrainStart: &llmgwv1.TrainStart{
|
||||
JobId: j.ID, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision, Data: j.Data, Params: j.Params,
|
||||
GpuMemoryMib: int32(j.GPUMemoryMiB), Gguf: j.GGUF}}}) //nolint:gosec // MiB
|
||||
@@ -135,6 +164,11 @@ func (s *Service) OnTrainStatus(node string, st *llmgwv1.TrainStatus) {
|
||||
}
|
||||
if err := s.store.UpdateTrainJob(ctx, j); err != nil {
|
||||
s.log.Warn("could not record a training job's progress", "job", j.ID, "err", err)
|
||||
return
|
||||
}
|
||||
// What it made came before it said it was done, on the same stream.
|
||||
if j.State == store.TrainDone && s.done != nil {
|
||||
go s.done(j)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -122,3 +122,41 @@ func TestCancellingAJob(t *testing.T) {
|
||||
t.Errorf("the node was not told: %+v", a.sent)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAJobNamingNoNodeGoesWhereThePickerSaysAndIsToldWhenDone(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
svc, s, a := setup(t)
|
||||
job(t, s, "j1", "")
|
||||
asked := 0
|
||||
free := ""
|
||||
svc.SetPicker(func(need int, busy map[string]bool) string {
|
||||
asked++
|
||||
if need != 2000 {
|
||||
t.Errorf("asked for %d MiB", need)
|
||||
}
|
||||
return free
|
||||
})
|
||||
done := make(chan store.TrainJob, 1)
|
||||
svc.SetOnDone(func(j store.TrainJob) { done <- j })
|
||||
|
||||
svc.Dispatch(ctx)
|
||||
if j, _ := s.TrainJob(ctx, "j1"); j.State != store.TrainQueued || j.Message != Waiting || len(a.sent) != 0 {
|
||||
t.Fatalf("with no card free: %+v, sent %d", j, len(a.sent))
|
||||
}
|
||||
free = "gpu2"
|
||||
a.connected["gpu2"] = true
|
||||
svc.Dispatch(ctx)
|
||||
j, _ := s.TrainJob(ctx, "j1")
|
||||
if len(a.sent) != 1 || j.Node != "gpu2" || j.State != store.TrainStarting {
|
||||
t.Fatalf("picked: %+v, sent %d", j, len(a.sent))
|
||||
}
|
||||
// Its node is the one it was sent to, which alone reports for it.
|
||||
svc.OnTrainStatus("cuda01", &llmgwv1.TrainStatus{JobId: "j1", State: "done"})
|
||||
if j, _ := s.TrainJob(ctx, "j1"); j.State == store.TrainDone {
|
||||
t.Fatal("another node finished it")
|
||||
}
|
||||
svc.OnTrainStatus("gpu2", &llmgwv1.TrainStatus{JobId: "j1", State: "done"})
|
||||
if got := <-done; got.ID != "j1" || got.Node != "gpu2" {
|
||||
t.Errorf("told of %+v", got)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user