A page for anyone in a project, built for the one who knows least: an Excel, CSV or JSONL file of questions and answers, previewed; a model; a name; who may use it; and a button. The weights to train on are found from where the model's came (a GGUF repository's base model), the card's memory from their size, the node as the one with the most free. When it is done the adapter is published by itself (a llama.cpp model's replicas replaced to load it) and is a model of its own in the chat and the API. The project's by default; its members share it wider (confirmed, and never wider than the knowledge bases its pairs came from) or delete it. One trains at a time per project. ctl train start needs only --model and --data now, and --publish publishes when done. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
303 lines
11 KiB
Go
303 lines
11 KiB
Go
package cli
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"os"
|
|
"strings"
|
|
"time"
|
|
|
|
"github.com/spf13/cobra"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
|
c := &cobra.Command{
|
|
Use: "train",
|
|
Short: "Train LoRA adapters on the nodes, on question/answer pairs",
|
|
Long: "A node trains an adapter for a catalog model on pairs (llmgw ctl kb dataset\n" +
|
|
"writes them from a knowledge base; read them through first). It trains with\n" +
|
|
"its installed vLLM's torch and a small overlay, as the engines' user, on a card\n" +
|
|
"with --gpu-memory free — beside an engine, if it leaves that much. Without\n" +
|
|
"--base-repo the weights are found from where the model's came, without\n" +
|
|
"--gpu-memory from their size, and without --node it goes to the node whose\n" +
|
|
"card has the most free. What it makes is kept by the control plane: llmgw ctl\n" +
|
|
"train download, or publish it as the model's adapter (--publish does that by\n" +
|
|
"itself when it is done). People in a project can do all this from the portal's\n" +
|
|
"\"Customise a model\" page.\n\n" +
|
|
"An adapter learns how answers sound — words, format, tone — far better than\n" +
|
|
"what they say. Facts belong to the knowledge base.",
|
|
}
|
|
|
|
var model, base, rev, node, data, owner string
|
|
var public bool
|
|
var gpuMem int
|
|
var epochs, rank int
|
|
var lr float64
|
|
var noGGUF, publishWhenDone bool
|
|
start := &cobra.Command{
|
|
Use: "start NAME --model MODEL --data FILE",
|
|
Short: "Queue an adapter's training on a node",
|
|
Example: " llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct --data policies.jsonl --public --publish\n" +
|
|
" llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct \\\n" +
|
|
" --base-repo Qwen/Qwen2.5-0.5B-Instruct --node cuda01 --gpu-memory 2300 --data policies.jsonl",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
b, err := os.ReadFile(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
req := adminapi.TrainRequest{Name: args[0], Model: model, BaseRepo: base, BaseRevision: rev, Node: node,
|
|
GPUMemoryMiB: gpuMem, Data: string(b), Params: map[string]any{}, Owner: owner, Public: public, Publish: publishWhenDone}
|
|
if cmd.Flags().Changed("epochs") {
|
|
req.Params["epochs"] = epochs
|
|
}
|
|
if cmd.Flags().Changed("rank") {
|
|
req.Params["rank"] = rank
|
|
}
|
|
if cmd.Flags().Changed("lr") {
|
|
req.Params["lr"] = lr
|
|
}
|
|
if noGGUF {
|
|
f := false
|
|
req.GGUF = &f
|
|
}
|
|
body, _ := json.Marshal(req)
|
|
var v adminapi.TrainJobView
|
|
if err := client().patient(2*time.Minute).do(cmd.Context(), "POST", "/api/v1/train/jobs", json.RawMessage(body), &v); err != nil {
|
|
return err
|
|
}
|
|
who := "every caller of " + v.Model
|
|
if v.Owner != "" {
|
|
who = v.Owner + " only"
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "queued %s: %s for %s on %s, %d pairs from %s, %d MiB of a card; it will be seen by %s\n",
|
|
v.ID, v.Name, v.Model, or(v.Node, "the node with a card free enough"), v.Pairs, v.BaseRepo, v.GPUMemoryMiB, who)
|
|
fmt.Fprintf(cmd.OutOrStdout(), "follow it with: llmgw ctl train show %s --follow\n", v.ID)
|
|
return nil
|
|
},
|
|
}
|
|
f := start.Flags()
|
|
f.StringVar(&model, "model", "", "the catalog model the adapter is for")
|
|
f.StringVar(&base, "base-repo", "", "the weights to train on, as safetensors on Hugging Face (owner/name)")
|
|
f.StringVar(&rev, "base-revision", "", "a revision of them (default: the newest)")
|
|
f.StringVar(&node, "node", "", "the node that trains (default: the one whose card has the most free)")
|
|
f.IntVar(&gpuMem, "gpu-memory", 0, "MiB of a card the job needs free (default: from the weights' size)")
|
|
f.BoolVar(&publishWhenDone, "publish", false, "publish it as the model's adapter when it is done, restarting a llama.cpp model's replicas")
|
|
f.StringVar(&data, "data", "", "the pairs, NDJSON of {\"q\", \"a\"}")
|
|
f.IntVar(&epochs, "epochs", 6, "passes over the pairs")
|
|
f.IntVar(&rank, "rank", 16, "the adapter's rank")
|
|
f.Float64Var(&lr, "lr", 2e-4, "the learning rate")
|
|
f.BoolVar(&noGGUF, "no-gguf", false, "skip the llama.cpp GGUF, even for a model llama.cpp serves")
|
|
f.StringVar(&owner, "owner", "", "who the adapter belongs to — org:, team:, project: (path or ID), user: or key: — and only they see it "+
|
|
"(default: the knowledge base's owner, when the pairs come from one)")
|
|
f.BoolVar(&public, "public", false, "every caller of the model sees it; refused for pairs from a knowledge base")
|
|
for _, req := range []string{"model", "data"} {
|
|
_ = start.MarkFlagRequired(req)
|
|
}
|
|
c.AddCommand(start)
|
|
|
|
c.AddCommand(&cobra.Command{
|
|
Use: "list",
|
|
Short: "Training jobs, newest first",
|
|
Args: cobra.NoArgs,
|
|
RunE: func(cmd *cobra.Command, _ []string) error {
|
|
var jobs []adminapi.TrainJobView
|
|
if err := client().get(cmd.Context(), "/api/v1/train/jobs", &jobs); err != nil {
|
|
return err
|
|
}
|
|
if *asJSON {
|
|
return json.NewEncoder(cmd.OutOrStdout()).Encode(jobs)
|
|
}
|
|
if len(jobs) == 0 {
|
|
fmt.Fprintln(cmd.OutOrStdout(), "No training jobs (llmgw ctl train start).")
|
|
return nil
|
|
}
|
|
w := tab()
|
|
fmt.Fprintln(w, "JOB\tADAPTER\tMODEL\tOWNER\tNODE\tSTATE\tPROGRESS\tLOSS\tPEAK\tCREATED")
|
|
for _, j := range jobs {
|
|
fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", j.ID, j.Name, j.Model, or(j.Owner, "everyone"), j.Node, j.State,
|
|
trainProgress(j), trainLoss(j), trainPeak(j), j.CreatedAt.Local().Format("01-02 15:04"))
|
|
}
|
|
return w.Flush()
|
|
},
|
|
})
|
|
|
|
var follow bool
|
|
show := &cobra.Command{
|
|
Use: "show JOB",
|
|
Short: "How a training job goes, and what it made",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
out := cmd.OutOrStdout()
|
|
last := ""
|
|
for {
|
|
var j adminapi.TrainJobView
|
|
if err := client().get(cmd.Context(), "/api/v1/train/jobs/"+args[0], &j); err != nil {
|
|
return err
|
|
}
|
|
if *asJSON {
|
|
return json.NewEncoder(out).Encode(j)
|
|
}
|
|
line := fmt.Sprintf("%s %s %s loss %s peak %s %s", j.State, trainProgress(j), j.Node, trainLoss(j), trainPeak(j), j.Message)
|
|
if !follow {
|
|
fmt.Fprintf(out, "%s: adapter %s for %s, owned by %s, on %s (base %s), %d pairs, %d MiB of a card\n", j.ID, j.Name, j.Model,
|
|
or(j.Owner, "everyone"), or(j.Node, "the node whose card has the most free"), j.BaseRepo, j.Pairs, j.GPUMemoryMiB)
|
|
if j.Project != "" {
|
|
fmt.Fprintf(out, "started from the portal in project %s by %s; published when done: %t\n", j.Project, j.CreatedBy, j.Publish)
|
|
}
|
|
fmt.Fprintln(out, line)
|
|
if len(j.Artifacts) > 0 {
|
|
fmt.Fprintf(out, "made: %s (llmgw ctl train download %s --kind KIND)\n", strings.Join(j.Artifacts, ", "), j.ID)
|
|
}
|
|
return nil
|
|
}
|
|
if line != last {
|
|
fmt.Fprintf(out, "%s %s\n", time.Now().Format("15:04:05"), line)
|
|
last = line
|
|
}
|
|
if store.TrainOver(j.State) {
|
|
if j.State != store.TrainDone {
|
|
return fmt.Errorf("%s %s", j.ID, j.State)
|
|
}
|
|
return nil
|
|
}
|
|
select {
|
|
case <-cmd.Context().Done():
|
|
return nil
|
|
case <-time.After(3 * time.Second):
|
|
}
|
|
}
|
|
},
|
|
}
|
|
show.Flags().BoolVarP(&follow, "follow", "f", false, "follow it until it ends")
|
|
c.AddCommand(show)
|
|
|
|
c.AddCommand(&cobra.Command{
|
|
Use: "cancel JOB",
|
|
Short: "Stop a training job",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
if err := client().do(cmd.Context(), "POST", "/api/v1/train/jobs/"+args[0]+"/cancel", nil, nil); err != nil {
|
|
return err
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%s cancelled\n", args[0])
|
|
return nil
|
|
},
|
|
})
|
|
c.AddCommand(&cobra.Command{
|
|
Use: "rm JOB",
|
|
Short: "Remove an ended training job and what it made",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
if err := client().do(cmd.Context(), "DELETE", "/api/v1/train/jobs/"+args[0], nil, nil); err != nil {
|
|
return err
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%s removed\n", args[0])
|
|
return nil
|
|
},
|
|
})
|
|
|
|
var restart bool
|
|
publish := &cobra.Command{
|
|
Use: "publish JOB",
|
|
Short: "Make what a finished job made its model's adapter, kept on the mirror",
|
|
Long: "The GGUF for a model llama.cpp serves, the PEFT files for one vLLM serves, are kept\n" +
|
|
"on the mirror and added to the model's adapters; a request then asks for the\n" +
|
|
"adapter by its name, as a model. llama.cpp loads adapters when a replica starts:\n" +
|
|
"--restart replaces the model's replicas, one at a time, to load it.",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
var v adminapi.TrainPublishView
|
|
if err := client().patient(15*time.Minute).do(cmd.Context(), "POST", "/api/v1/train/jobs/"+args[0]+"/publish", nil, &v); err != nil {
|
|
return err
|
|
}
|
|
out := cmd.OutOrStdout()
|
|
fmt.Fprintf(out, "%s is %s's adapter, from %s\n", v.Adapter, v.Model, v.Source)
|
|
if v.Owner != "" {
|
|
fmt.Fprintf(out, "only %s sees it; to anyone else it is a model that does not exist\n", v.Owner)
|
|
}
|
|
if restart && strings.Contains(v.Note, "restart") {
|
|
body, _ := json.Marshal(adminapi.RestartRequest{Models: []string{v.Model}})
|
|
var res adminapi.RestartResult
|
|
if err := client().post(cmd.Context(), "/api/v1/restart", body, &res); err != nil {
|
|
return err
|
|
}
|
|
fmt.Fprintf(out, "%d replica(s) of %s restarting to load it; then ask for model %s\n", len(res.Queued), v.Model, v.Adapter)
|
|
return nil
|
|
}
|
|
fmt.Fprintln(out, v.Note)
|
|
return nil
|
|
},
|
|
}
|
|
publish.Flags().BoolVar(&restart, "restart", false, "restart the model's replicas to load it (llama.cpp)")
|
|
c.AddCommand(publish)
|
|
|
|
var kind, outFile string
|
|
download := &cobra.Command{
|
|
Use: "download JOB",
|
|
Short: "Save what a training job made: the PEFT adapter (peft) or the GGUF (gguf)",
|
|
Args: cobra.ExactArgs(1),
|
|
RunE: func(cmd *cobra.Command, args []string) error {
|
|
if outFile == "" {
|
|
outFile = args[0] + ".tar.gz"
|
|
if kind == "gguf" {
|
|
outFile = args[0] + ".gguf"
|
|
}
|
|
}
|
|
f, err := os.Create(outFile)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
defer f.Close()
|
|
cw := &countWriter{w: f}
|
|
if err := client().patient(10*time.Minute).do(cmd.Context(), "GET",
|
|
"/api/v1/train/jobs/"+args[0]+"/artifacts/"+kind, nil, io.Writer(cw)); err != nil {
|
|
_ = os.Remove(outFile)
|
|
return err
|
|
}
|
|
fmt.Fprintf(cmd.OutOrStdout(), "%s: %s\n", outFile, humanBytes(cw.n))
|
|
return nil
|
|
},
|
|
}
|
|
download.Flags().StringVar(&kind, "kind", "peft", "peft (a gzipped tar of the PEFT adapter) or gguf")
|
|
download.Flags().StringVarP(&outFile, "out", "o", "", "the file to write")
|
|
c.AddCommand(download)
|
|
return c
|
|
}
|
|
|
|
type countWriter struct {
|
|
w io.Writer
|
|
n int64
|
|
}
|
|
|
|
func (c *countWriter) Write(p []byte) (int, error) {
|
|
n, err := c.w.Write(p)
|
|
c.n += int64(n)
|
|
return n, err
|
|
}
|
|
|
|
func trainProgress(j adminapi.TrainJobView) string {
|
|
if j.Steps == 0 {
|
|
return "-"
|
|
}
|
|
return fmt.Sprintf("%d/%d", j.Step, j.Steps)
|
|
}
|
|
|
|
func trainLoss(j adminapi.TrainJobView) string {
|
|
if j.Loss == 0 {
|
|
return "-"
|
|
}
|
|
return fmt.Sprintf("%.3f", j.Loss)
|
|
}
|
|
|
|
func trainPeak(j adminapi.TrainJobView) string {
|
|
if j.PeakMiB == 0 {
|
|
return "-"
|
|
}
|
|
return fmt.Sprintf("%d MiB", j.PeakMiB)
|
|
}
|