A page for anyone in a project, built for the one who knows least: an Excel, CSV or JSONL file of questions and answers, previewed; a model; a name; who may use it; and a button. The weights to train on are found from where the model's came (a GGUF repository's base model), the card's memory from their size, the node as the one with the most free. When it is done the adapter is published by itself (a llama.cpp model's replicas replaced to load it) and is a model of its own in the chat and the API. The project's by default; its members share it wider (confirmed, and never wider than the knowledge bases its pairs came from) or delete it. One trains at a time per project. ctl train start needs only --model and --data now, and --publish publishes when done. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
242 lines
8.4 KiB
Go
242 lines
8.4 KiB
Go
package store
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"strings"
|
|
"time"
|
|
)
|
|
|
|
// TrainJob is a LoRA adapter trained on a node (BACKLOG F4).
|
|
type TrainJob struct {
|
|
ID string
|
|
Name string
|
|
Model string
|
|
BaseRepo string
|
|
BaseRevision string
|
|
Node string
|
|
// Owner is who the adapter belongs to, scope:ID; empty is everyone.
|
|
Owner string
|
|
GPUMemoryMiB int
|
|
Params string
|
|
GGUF bool
|
|
// Data is the pairs, NDJSON; Pairs how many.
|
|
Data []byte
|
|
Pairs int
|
|
|
|
State string
|
|
Step int
|
|
Steps int
|
|
Loss float64
|
|
Message string
|
|
PeakMiB int
|
|
CreatedAt time.Time
|
|
StartedAt time.Time
|
|
FinishedAt time.Time
|
|
CreatedBy string
|
|
// Project is the project a job was started in from the portal: its
|
|
// members see and manage it, and it pays. Empty for an administrator's.
|
|
Project string
|
|
// Publish has what the job makes published when it is done.
|
|
Publish bool
|
|
}
|
|
|
|
// The states of a training job.
|
|
const (
|
|
TrainQueued = "queued"
|
|
TrainStarting = "starting"
|
|
TrainDone = "done"
|
|
TrainFailed = "failed"
|
|
TrainCancelled = "cancelled"
|
|
)
|
|
|
|
// TrainOver is whether a state is one a job ends in.
|
|
func TrainOver(state string) bool {
|
|
return state == TrainDone || state == TrainFailed || state == TrainCancelled
|
|
}
|
|
|
|
// CreateTrainJob queues a job.
|
|
func (s *DB) CreateTrainJob(ctx context.Context, j TrainJob) error {
|
|
if j.CreatedAt.IsZero() {
|
|
j.CreatedAt = time.Now()
|
|
}
|
|
_, err := s.db.ExecContext(ctx, `INSERT INTO train_jobs (id, name, model, base_repo, base_revision, node, owner, gpu_memory_mib,
|
|
params, gguf, data, pairs, state, created_at, created_by, project_id, publish)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
|
j.ID, j.Name, j.Model, j.BaseRepo, j.BaseRevision, j.Node, j.Owner, j.GPUMemoryMiB, j.Params, boolInt(j.GGUF), j.Data, j.Pairs,
|
|
TrainQueued, millis(j.CreatedAt), j.CreatedBy, j.Project, boolInt(j.Publish))
|
|
return err
|
|
}
|
|
|
|
const trainCols = `id, name, model, base_repo, base_revision, node, owner, gpu_memory_mib, params, gguf, pairs, state, step, steps,
|
|
loss, message, peak_mib, created_at, started_at, finished_at, created_by, project_id, publish`
|
|
|
|
func scanTrainJob(sc scanner) (TrainJob, error) {
|
|
var j TrainJob
|
|
var gguf, publish int
|
|
var created, started, finished int64
|
|
err := sc.Scan(&j.ID, &j.Name, &j.Model, &j.BaseRepo, &j.BaseRevision, &j.Node, &j.Owner, &j.GPUMemoryMiB, &j.Params, &gguf,
|
|
&j.Pairs, &j.State, &j.Step, &j.Steps, &j.Loss, &j.Message, &j.PeakMiB, &created, &started, &finished, &j.CreatedBy,
|
|
&j.Project, &publish)
|
|
j.GGUF, j.Publish = gguf != 0, publish != 0
|
|
j.CreatedAt = fromMillis(created)
|
|
if started > 0 {
|
|
j.StartedAt = fromMillis(started)
|
|
}
|
|
if finished > 0 {
|
|
j.FinishedAt = fromMillis(finished)
|
|
}
|
|
return j, err
|
|
}
|
|
|
|
// TrainJobs lists jobs, newest first.
|
|
func (s *DB) TrainJobs(ctx context.Context, limit int) ([]TrainJob, error) {
|
|
return queryAll(ctx, s.db, `SELECT `+trainCols+` FROM train_jobs ORDER BY created_at DESC LIMIT ?`, []any{limit},
|
|
func(r *sql.Rows) (TrainJob, error) { return scanTrainJob(r) })
|
|
}
|
|
|
|
// TrainJob reads one job, with its data.
|
|
func (s *DB) TrainJob(ctx context.Context, id string) (TrainJob, error) {
|
|
j, err := scanTrainJob(s.db.QueryRowContext(ctx, `SELECT `+trainCols+` FROM train_jobs WHERE id = ?`, id))
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return TrainJob{}, ErrNotFound
|
|
}
|
|
if err != nil {
|
|
return j, err
|
|
}
|
|
err = s.db.QueryRowContext(ctx, `SELECT data FROM train_jobs WHERE id = ?`, id).Scan(&j.Data)
|
|
return j, err
|
|
}
|
|
|
|
// QueuedTrainJobs are the jobs waiting for their node, oldest first.
|
|
func (s *DB) QueuedTrainJobs(ctx context.Context) ([]TrainJob, error) {
|
|
return queryAll(ctx, s.db, `SELECT `+trainCols+` FROM train_jobs WHERE state = ? ORDER BY created_at`, []any{TrainQueued},
|
|
func(r *sql.Rows) (TrainJob, error) { return scanTrainJob(r) })
|
|
}
|
|
|
|
// ActiveTrainJobs are the jobs sent to a node and not ended.
|
|
func (s *DB) ActiveTrainJobs(ctx context.Context) ([]TrainJob, error) {
|
|
return queryAll(ctx, s.db, `SELECT `+trainCols+` FROM train_jobs WHERE state NOT IN (?, ?, ?, ?)`,
|
|
[]any{TrainQueued, TrainDone, TrainFailed, TrainCancelled},
|
|
func(r *sql.Rows) (TrainJob, error) { return scanTrainJob(r) })
|
|
}
|
|
|
|
// UpdateTrainJob records how a job goes; a job that has ended is not
|
|
// changed again, so a late message cannot bring one back.
|
|
func (s *DB) UpdateTrainJob(ctx context.Context, j TrainJob) error {
|
|
var started, finished int64
|
|
if !j.StartedAt.IsZero() {
|
|
started = millis(j.StartedAt)
|
|
}
|
|
if !j.FinishedAt.IsZero() {
|
|
finished = millis(j.FinishedAt)
|
|
}
|
|
_, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET node = ?, state = ?, step = ?, steps = ?, loss = ?, message = ?,
|
|
peak_mib = ?, started_at = ?, finished_at = ? WHERE id = ? AND state NOT IN (?, ?, ?)`,
|
|
j.Node, j.State, j.Step, j.Steps, j.Loss, j.Message, j.PeakMiB, started, finished, j.ID,
|
|
TrainDone, TrainFailed, TrainCancelled)
|
|
return err
|
|
}
|
|
|
|
// TrainJobsFor lists the jobs started in projects, or by a person outside
|
|
// any project, newest first.
|
|
func (s *DB) TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]TrainJob, error) {
|
|
q := `SELECT ` + trainCols + ` FROM train_jobs WHERE (project_id = '' AND publish = 1 AND created_by = ?)`
|
|
args := []any{createdBy}
|
|
if len(projects) > 0 {
|
|
q += ` OR project_id IN (?` + strings.Repeat(", ?", len(projects)-1) + `)`
|
|
for _, p := range projects {
|
|
args = append(args, p)
|
|
}
|
|
}
|
|
return queryAll(ctx, s.db, q+` ORDER BY created_at DESC LIMIT ?`, append(args, limit),
|
|
func(r *sql.Rows) (TrainJob, error) { return scanTrainJob(r) })
|
|
}
|
|
|
|
// SetTrainNote changes what a job says of itself, ended or not: what
|
|
// happened when what it made was published.
|
|
func (s *DB) SetTrainNote(ctx context.Context, id, message string) error {
|
|
_, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET message = ? WHERE id = ?`, message, id)
|
|
return err
|
|
}
|
|
|
|
// SetTrainOwner changes who a job's adapter belongs to.
|
|
func (s *DB) SetTrainOwner(ctx context.Context, id, owner string) error {
|
|
res, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET owner = ? WHERE id = ?`, owner, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// PutTrainArtifact keeps what a job made.
|
|
func (s *DB) PutTrainArtifact(ctx context.Context, job, kind string, data []byte) error {
|
|
_, err := s.db.ExecContext(ctx, `INSERT INTO train_artifacts (job_id, kind, data) VALUES (?, ?, ?)
|
|
ON CONFLICT (job_id, kind) DO UPDATE SET data = excluded.data`, job, kind, data)
|
|
return err
|
|
}
|
|
|
|
// TrainArtifact reads what a job made, of one kind.
|
|
func (s *DB) TrainArtifact(ctx context.Context, job, kind string) ([]byte, error) {
|
|
var b []byte
|
|
err := s.db.QueryRowContext(ctx, `SELECT data FROM train_artifacts WHERE job_id = ? AND kind = ?`, job, kind).Scan(&b)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return nil, ErrNotFound
|
|
}
|
|
return b, err
|
|
}
|
|
|
|
// TrainArtifactKinds lists what a job made.
|
|
func (s *DB) TrainArtifactKinds(ctx context.Context, job string) ([]string, error) {
|
|
return queryAll(ctx, s.db, `SELECT kind FROM train_artifacts WHERE job_id = ? ORDER BY kind`, []any{job},
|
|
func(r *sql.Rows) (string, error) {
|
|
var k string
|
|
return k, r.Scan(&k)
|
|
})
|
|
}
|
|
|
|
// DeleteTrainJob removes a job and what it made.
|
|
func (s *DB) DeleteTrainJob(ctx context.Context, id string) error {
|
|
return s.inTx(ctx, func(c conn) error {
|
|
if _, err := c.ExecContext(ctx, `DELETE FROM train_artifacts WHERE job_id = ?`, id); err != nil {
|
|
return err
|
|
}
|
|
res, err := c.ExecContext(ctx, `DELETE FROM train_jobs WHERE id = ?`, id)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if n, _ := res.RowsAffected(); n == 0 {
|
|
return ErrNotFound
|
|
}
|
|
return nil
|
|
})
|
|
}
|
|
|
|
// EffectiveFor is a scope's place in the tree, as a caller in it would have
|
|
// it, for asking what it may read: an organization, a team or a project
|
|
// with what is above it, a key as it resolves, a person in an organization.
|
|
func (s *DB) EffectiveFor(ctx context.Context, scope Scope, id, org string) (Effective, error) {
|
|
switch scope {
|
|
case ScopeOrg:
|
|
return Effective{OrgID: id}, nil
|
|
case ScopeTeam:
|
|
var o string
|
|
err := s.db.QueryRowContext(ctx, `SELECT org_id FROM teams WHERE id = ?`, id).Scan(&o)
|
|
if errors.Is(err, sql.ErrNoRows) {
|
|
return Effective{}, ErrNotFound
|
|
}
|
|
return Effective{TeamID: id, OrgID: o}, err
|
|
case ScopeProject:
|
|
return s.ResolveFor(ctx, "", "", id)
|
|
case ScopeKey:
|
|
return s.Resolve(ctx, id)
|
|
case ScopeUser:
|
|
return Effective{UserID: id, OrgID: org}, nil
|
|
}
|
|
return Effective{}, errors.New("no such scope " + string(scope))
|
|
}
|