From a53b2f4be901f6eccf82ff1dc0a5583adc172921 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 19 Nov 2026 11:31:12 +0100 Subject: [PATCH] Customise a model from the portal: a file, a model, a button A page for anyone in a project, built for the one who knows least: an Excel, CSV or JSONL file of questions and answers, previewed; a model; a name; who may use it; and a button. The weights to train on are found from where the model's came (a GGUF repository's base model), the card's memory from their size, the node as the one with the most free. When it is done the adapter is published by itself (a llama.cpp model's replicas replaced to load it) and is a model of its own in the chat and the API. The project's by default; its members share it wider (confirmed, and never wider than the knowledge bases its pairs came from) or delete it. One trains at a time per project. ctl train start needs only --model and --data now, and --publish publishes when done. Co-Authored-By: Claude Opus 5.5 --- BACKLOG.md | 9 + CHANGELOG.md | 16 + docs/knowledge.md | 17 +- internal/cli/server.go | 9 +- internal/cli/train.go | 37 +- internal/control/adminapi/train.go | 419 +++++++++++++++--- internal/control/adminapi/train_portal.go | 114 +++++ .../control/adminapi/train_portal_test.go | 137 ++++++ internal/control/train/pairs.go | 205 +++++++++ internal/control/train/pairs_test.go | 65 +++ internal/control/train/service.go | 36 +- internal/control/train/service_test.go | 38 ++ internal/kb/extract/extract_test.go | 23 + internal/kb/extract/ooxml.go | 137 +++++- internal/router/chat.go | 1 + internal/router/handlers.go | 2 +- internal/router/pages_auth_test.go | 2 +- internal/router/router.go | 3 + internal/router/train.go | 331 ++++++++++++++ internal/router/train_test.go | 165 +++++++ .../migrations/postgres/0055_train_portal.sql | 5 + .../migrations/sqlite/0055_train_portal.sql | 5 + internal/store/train.go | 64 ++- internal/web/account.go | 1 + internal/web/locales/en/nav.json | 3 +- internal/web/locales/en/train.json | 55 +++ internal/web/locales/tr/nav.json | 3 +- internal/web/locales/tr/train.json | 55 +++ internal/web/nav.go | 3 + internal/web/templates/nav.html | 3 + internal/web/templates/train.html | 287 ++++++++++++ internal/web/train.go | 34 ++ internal/web/train_test.go | 49 ++ 33 files changed, 2199 insertions(+), 134 deletions(-) create mode 100644 internal/control/adminapi/train_portal.go create mode 100644 internal/control/adminapi/train_portal_test.go create mode 100644 internal/control/train/pairs.go create mode 100644 internal/control/train/pairs_test.go create mode 100644 internal/router/train.go create mode 100644 internal/router/train_test.go create mode 100644 internal/store/migrations/postgres/0055_train_portal.sql create mode 100644 internal/store/migrations/sqlite/0055_train_portal.sql create mode 100644 internal/web/locales/en/train.json create mode 100644 internal/web/locales/tr/train.json create mode 100644 internal/web/templates/train.html create mode 100644 internal/web/train.go create mode 100644 internal/web/train_test.go diff --git a/BACKLOG.md b/BACKLOG.md index 4122a16..8e5ff50 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2107,6 +2107,15 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t controller ~20 s later, sbatch ran on n1 as ada. Disabled in AD: suspended in 56 s, the Linux user gone in 89 s, ssh refused, home kept. Enabled: back in 32 s, same UID, key kept. Taken out of ml-team: out of the team in 11 s; put back: in again in 56 s. +- P5, replaced 2026-11-17 by the portal (decided 2026-11-15: users of every level, build for the one who + knows least first; any project member may start, no approval, one per project, the project's by default): + **Customise a model** (`/train`): an xlsx/CSV/JSONL of questions and answers, checked and previewed; base + weights from the model's origin (a GGUF repo's card base_model), memory from their size (2x + 1 GiB), + the node with the most free; published by itself when done (llama.cpp replicas replaced to load it); + shared me/project/team/org/everyone (wider than the project only confirmed, never wider than its + knowledge bases), deleted from the page. Left: pairs generated from a knowledge base on the page, a + base-vs-new comparison, the training's GPU time charged to the project, big jobs through Slurm, the + server's own error messages in Turkish. - P6, done 2026-11-15 but for budgets: each project a Slurm account (org and team accounts above it, ids keyed by kind: the first install's org, team and project share the id "default"), its members and its team's and org's on it; AccountingStorageEnforce=associations; the controller's agent keeps them with diff --git a/CHANGELOG.md b/CHANGELOG.md index d1fb8ed..0723f1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,22 @@ APIs change between minors without a deprecation period. ### Added +- **Customise a model from the portal.** A new page, **Customise a model** + (`/train`), lets anyone in a project teach a model how to answer without a + terminal: an Excel, CSV or JSONL file of questions and answers (columns + headed Soru/Cevap or Question/Answer, else the first two), checked and + shown before anything trains; a model; a name; who may use it; and a + button. The weights to train on are found from where the model's came (a + GGUF repository's base model on Hugging Face), the card's memory from their + size, the node as the one with the most free; when it is done the adapter + is published by itself and appears as a model of its own in the chat and + the API, a llama.cpp model's replicas replaced one at a time to load it. + It is the project's by default; its members may share it with the person, + the team, the organization or everyone (wider than the project only once + they confirm the file holds nothing that group must not see, and never + wider than the knowledge bases its pairs came from), and delete it. One + trains at a time per project. `llmgw ctl train start` needs only `--model` + and `--data` now, and `--publish` publishes when done. - **An organization's admin.** Besides member and manager, a membership of an organization may now be `admin` (`llmgw users join … --org Acme --role admin`, the role chosen on a membership on the people page, the console, diff --git a/docs/knowledge.md b/docs/knowledge.md index 500a81e..edb8f9b 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -385,18 +385,25 @@ right and mixed facts up on the rest, while the same questions through the knowledge base got 9 of 9. Use both: the knowledge base for what answers say, an adapter for how they sound. +Anyone in a project can do it from the portal: **Customise a model** +(`/train`) takes an Excel, CSV or JSONL file of questions and answers, a +model, a name and who may use it, and publishes the adapter by itself when it +is trained. The weights, the node and the card's memory are worked out. From +the command line: + ```sh # 1. Questions and answers from the collection, written by a model of the # cluster. Read the file through and correct it. llmgw ctl kb dataset Acme/policies --model qwen2.5-coder-14b-instruct --out policies.jsonl -# 2. Train on a node. --gpu-memory is what the job needs of a card; it -# runs beside an engine when the engine leaves that much free. -llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct \ - --base-repo Qwen/Qwen2.5-0.5B-Instruct --node cuda01 --gpu-memory 2300 --data policies.jsonl +# 2. Train. The weights to train on are found from where the model's came +# (--base-repo names others), the card's memory from their size +# (--gpu-memory), the node as the one whose card has the most free +# (--node); it runs beside an engine when the engine leaves that much. +llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct --data policies.jsonl llmgw ctl train show JOB --follow -# 3. Make it the model's adapter, kept on the mirror. +# 3. Make it the model's adapter, kept on the mirror (or --publish in step 2). llmgw ctl train publish JOB --restart ``` diff --git a/internal/cli/server.go b/internal/cli/server.go index faa5059..bcaa68c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -465,7 +465,12 @@ func newServerCmd() *cobra.Command { trainer := train.New(st, agentSvc, log) agentSvc.SetTrain(trainer) go trainer.Run(ctx) - api.SetTrain(&adminapi.TrainAdmin{Store: st, Service: trainer}) + api.SetTrain(&adminapi.TrainAdmin{Store: st, Service: trainer, Log: log}) + // From the portal: a node with a card free enough, and + // what was made published when it is done. + trainer.SetPicker(api.PickTrainNode) + trainer.SetOnDone(api.TrainDone) + rt.SetTraining(api) // Slurm on the nodes the settings name, its munge key // made from a cluster secret. slurmSvc = slurmsvc.New(func() config.SlurmSettings { return current.Load().Slurm }, @@ -661,7 +666,7 @@ func newServerCmd() *cobra.Command { // takes longer than the mirror's usual answers. if adminAPI != nil { if up, err := mirror.NewFailoverClient(mirrors, cfg.Mirror.TLS, 10*time.Minute); err == nil { - adminAPI.SetTrainMirror(up) + adminAPI.SetTrainMirror(up, hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)) } } // Browsing the registry is what turns "paste a resolve URL" diff --git a/internal/cli/train.go b/internal/cli/train.go index a51ebef..3de82f7 100644 --- a/internal/cli/train.go +++ b/internal/cli/train.go @@ -21,9 +21,13 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { Long: "A node trains an adapter for a catalog model on pairs (llmgw ctl kb dataset\n" + "writes them from a knowledge base; read them through first). It trains with\n" + "its installed vLLM's torch and a small overlay, as the engines' user, on a card\n" + - "with --gpu-memory free — beside an engine, if it leaves that much. What it\n" + - "makes is kept by the control plane: llmgw ctl train download, or publish it\n" + - "as the model's adapter.\n\n" + + "with --gpu-memory free — beside an engine, if it leaves that much. Without\n" + + "--base-repo the weights are found from where the model's came, without\n" + + "--gpu-memory from their size, and without --node it goes to the node whose\n" + + "card has the most free. What it makes is kept by the control plane: llmgw ctl\n" + + "train download, or publish it as the model's adapter (--publish does that by\n" + + "itself when it is done). People in a project can do all this from the portal's\n" + + "\"Customise a model\" page.\n\n" + "An adapter learns how answers sound — words, format, tone — far better than\n" + "what they say. Facts belong to the knowledge base.", } @@ -33,11 +37,12 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { var gpuMem int var epochs, rank int var lr float64 - var noGGUF bool + var noGGUF, publishWhenDone bool start := &cobra.Command{ - Use: "start NAME --model MODEL --base-repo OWNER/NAME --node NODE --data FILE", + Use: "start NAME --model MODEL --data FILE", Short: "Queue an adapter's training on a node", - Example: " llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct \\\n" + + Example: " llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct --data policies.jsonl --public --publish\n" + + " llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct \\\n" + " --base-repo Qwen/Qwen2.5-0.5B-Instruct --node cuda01 --gpu-memory 2300 --data policies.jsonl", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -46,7 +51,7 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { return err } req := adminapi.TrainRequest{Name: args[0], Model: model, BaseRepo: base, BaseRevision: rev, Node: node, - GPUMemoryMiB: gpuMem, Data: string(b), Params: map[string]any{}, Owner: owner, Public: public} + GPUMemoryMiB: gpuMem, Data: string(b), Params: map[string]any{}, Owner: owner, Public: public, Publish: publishWhenDone} if cmd.Flags().Changed("epochs") { req.Params["epochs"] = epochs } @@ -69,8 +74,8 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { if v.Owner != "" { who = v.Owner + " only" } - fmt.Fprintf(cmd.OutOrStdout(), "queued %s: %s for %s on %s, %d pairs; it will be seen by %s\n", - v.ID, v.Name, v.Model, v.Node, v.Pairs, who) + fmt.Fprintf(cmd.OutOrStdout(), "queued %s: %s for %s on %s, %d pairs from %s, %d MiB of a card; it will be seen by %s\n", + v.ID, v.Name, v.Model, or(v.Node, "the node with a card free enough"), v.Pairs, v.BaseRepo, v.GPUMemoryMiB, who) fmt.Fprintf(cmd.OutOrStdout(), "follow it with: llmgw ctl train show %s --follow\n", v.ID) return nil }, @@ -79,8 +84,9 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { f.StringVar(&model, "model", "", "the catalog model the adapter is for") f.StringVar(&base, "base-repo", "", "the weights to train on, as safetensors on Hugging Face (owner/name)") f.StringVar(&rev, "base-revision", "", "a revision of them (default: the newest)") - f.StringVar(&node, "node", "", "the node that trains") - f.IntVar(&gpuMem, "gpu-memory", 4096, "MiB of a card the job needs free") + f.StringVar(&node, "node", "", "the node that trains (default: the one whose card has the most free)") + f.IntVar(&gpuMem, "gpu-memory", 0, "MiB of a card the job needs free (default: from the weights' size)") + f.BoolVar(&publishWhenDone, "publish", false, "publish it as the model's adapter when it is done, restarting a llama.cpp model's replicas") f.StringVar(&data, "data", "", "the pairs, NDJSON of {\"q\", \"a\"}") f.IntVar(&epochs, "epochs", 6, "passes over the pairs") f.IntVar(&rank, "rank", 16, "the adapter's rank") @@ -89,7 +95,7 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { f.StringVar(&owner, "owner", "", "who the adapter belongs to — org:, team:, project: (path or ID), user: or key: — and only they see it "+ "(default: the knowledge base's owner, when the pairs come from one)") f.BoolVar(&public, "public", false, "every caller of the model sees it; refused for pairs from a knowledge base") - for _, req := range []string{"model", "base-repo", "node", "data"} { + for _, req := range []string{"model", "data"} { _ = start.MarkFlagRequired(req) } c.AddCommand(start) @@ -138,8 +144,11 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } line := fmt.Sprintf("%s %s %s loss %s peak %s %s", j.State, trainProgress(j), j.Node, trainLoss(j), trainPeak(j), j.Message) if !follow { - fmt.Fprintf(out, "%s: adapter %s for %s, owned by %s, on %s (base %s), %d pairs\n", j.ID, j.Name, j.Model, - or(j.Owner, "everyone"), j.Node, j.BaseRepo, j.Pairs) + fmt.Fprintf(out, "%s: adapter %s for %s, owned by %s, on %s (base %s), %d pairs, %d MiB of a card\n", j.ID, j.Name, j.Model, + or(j.Owner, "everyone"), or(j.Node, "the node whose card has the most free"), j.BaseRepo, j.Pairs, j.GPUMemoryMiB) + if j.Project != "" { + fmt.Fprintf(out, "started from the portal in project %s by %s; published when done: %t\n", j.Project, j.CreatedBy, j.Publish) + } fmt.Fprintln(out, line) if len(j.Artifacts) > 0 { fmt.Fprintf(out, "made: %s (llmgw ctl train download %s --kind KIND)\n", strings.Join(j.Artifacts, ", "), j.ID) diff --git a/internal/control/adminapi/train.go b/internal/control/adminapi/train.go index b777799..e2545f5 100644 --- a/internal/control/adminapi/train.go +++ b/internal/control/adminapi/train.go @@ -10,15 +10,20 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" "regexp" "slices" "strconv" "strings" + "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -36,6 +41,11 @@ type TrainStore interface { TrainArtifact(ctx context.Context, job, kind string) ([]byte, error) TrainArtifactKinds(ctx context.Context, job string) ([]string, error) DeleteTrainJob(ctx context.Context, id string) error + QueuedTrainJobs(ctx context.Context) ([]store.TrainJob, error) + ActiveTrainJobs(ctx context.Context) ([]store.TrainJob, error) + TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]store.TrainJob, error) + SetTrainOwner(ctx context.Context, id, owner string) error + SetTrainNote(ctx context.Context, id, message string) error } // TrainAdmin is what the training endpoints need. @@ -45,6 +55,27 @@ type TrainAdmin struct { // Mirror keeps what a job made where the nodes fetch weights; nil // leaves publishing off. Mirror TrainMirror + // Registry finds the weights to train a catalog model on, from where + // its own came; nil has every job name them. + Registry TrainRegistry + // Log says what publishing by itself could not do. + Log *slog.Logger + + mu sync.Mutex + bases map[string]trainBaseInfo // by the model's origin repository +} + +// TrainRegistry reads a repository of the model registry. +type TrainRegistry interface { + RepoAt(ctx context.Context, id, rev string) (hf.Repo, error) +} + +// trainBaseInfo is the safetensors weights a model is trained on. +type trainBaseInfo struct { + repo, revision string + bytes int64 + err error + at time.Time } // TrainMirror is the model mirror, as publishing needs it. @@ -66,11 +97,11 @@ type TrainPublishView struct { // SetTrain enables the training endpoints. func (a *API) SetTrain(t *TrainAdmin) { a.train = t } -// SetTrainMirror gives publishing the mirror, which is set up after the -// training endpoints are. -func (a *API) SetTrainMirror(m TrainMirror) { +// SetTrainMirror gives publishing the mirror, and finding a model's +// weights the registry, which are set up after the training endpoints are. +func (a *API) SetTrainMirror(m TrainMirror, reg TrainRegistry) { if a.train != nil { - a.train.Mirror = m + a.train.Mirror, a.train.Registry = m, reg } } @@ -81,13 +112,23 @@ type TrainRequest struct { // Model is the catalog model the adapter is for. Model string `json:"model"` // BaseRepo and BaseRevision are the weights trained on: the model's own - // on its registry, in safetensors (Hugging Face), not a GGUF. - BaseRepo string `json:"base_repo"` + // on its registry, in safetensors (Hugging Face), not a GGUF. Empty + // finds them from where the model's came: the repository itself, or the + // one its card says it was made from. + BaseRepo string `json:"base_repo,omitempty"` BaseRevision string `json:"base_revision,omitempty"` - // Node trains it; GPUMemoryMiB is what it needs of a card, which can be - // what an engine there leaves free. - Node string `json:"node"` - GPUMemoryMiB int `json:"gpu_memory_mib"` + // Node trains it; empty is the first with a card that has enough free. + // GPUMemoryMiB is what it needs of a card, which can be what an engine + // there leaves free; 0 is worked out from the weights' size. + Node string `json:"node,omitempty"` + GPUMemoryMiB int `json:"gpu_memory_mib,omitempty"` + // Project is the project the job is started in: its members see and + // manage it, and only one of its jobs trains at a time. + Project string `json:"project,omitempty"` + // Publish puts what the job makes on the mirror and makes it the + // model's adapter by itself when the job is done, restarting a llama.cpp + // model's replicas to load it. + Publish bool `json:"publish,omitempty"` // Owner is who the adapter will belong to: org:, team:, project: (by // path or ID), user: or key:. Only callers there see it. When the // pairs come from one knowledge base, it defaults to that collection's @@ -129,6 +170,8 @@ type TrainJobView struct { StartedAt time.Time `json:"started_at,omitzero"` FinishedAt time.Time `json:"finished_at,omitzero"` CreatedBy string `json:"created_by,omitempty"` + Project string `json:"project,omitempty"` + Publish bool `json:"publish,omitempty"` // Artifacts are what it made: peft (a gzipped tar), gguf. Artifacts []string `json:"artifacts,omitempty"` } @@ -137,7 +180,8 @@ func trainView(j store.TrainJob) TrainJobView { return TrainJobView{ID: j.ID, Name: j.Name, Model: j.Model, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision, Node: j.Node, Owner: j.Owner, GPUMemoryMiB: j.GPUMemoryMiB, Params: j.Params, GGUF: j.GGUF, Pairs: j.Pairs, State: j.State, Step: j.Step, Steps: j.Steps, Loss: j.Loss, Message: j.Message, PeakMiB: j.PeakMiB, CreatedAt: j.CreatedAt, - StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy} + StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy, + Project: j.Project, Publish: j.Publish} } func (a *API) registerTrain(mux Mux) { @@ -178,50 +222,122 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, err.Error()) return } - bad := func(format string, args ...any) { - writeError(w, http.StatusBadRequest, fmt.Sprintf(format, args...)) + j, err := a.StartTraining(r.Context(), in, actorOf(r)) + if err != nil { + writeTrainErr(w, err) + return + } + writeJSON(w, http.StatusCreated, trainView(j)) +} + +// trainErr is a refusal, with the status it is answered with. +type trainErr struct { + code int + msg string +} + +func (e *trainErr) Error() string { return e.msg } + +func refused(code int, format string, args ...any) error { + return &trainErr{code: code, msg: fmt.Sprintf(format, args...)} +} + +func writeTrainErr(w http.ResponseWriter, err error) { + var te *trainErr + if errors.As(err, &te) { + writeError(w, te.code, te.msg) + return + } + writeError(w, http.StatusInternalServerError, err.Error()) +} + +// TrainRefusal is the reason a training request was refused, for someone +// to read, and whether it was the request's fault rather than the +// cluster's. +func TrainRefusal(err error) (string, bool) { + if err == nil { + return "", false + } + var te *trainErr + if errors.As(err, &te) { + return te.msg, te.code < 500 + } + return err.Error(), false +} + +// StartTraining checks a job and queues it, as actor. +func (a *API) StartTraining(ctx context.Context, in TrainRequest, actor string) (store.TrainJob, error) { + if a.train == nil { + return store.TrainJob{}, refused(http.StatusNotImplemented, + "training needs the store and the nodes' agents, which this control plane runs without") + } + bad := func(format string, args ...any) (store.TrainJob, error) { + return store.TrainJob{}, refused(http.StatusBadRequest, format, args...) } if !adapterName.MatchString(in.Name) { - bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name) - return + return bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name) } m, ok := a.ctl.Catalog().Resolve(in.Model) if !ok { - bad("no model %s in the catalog", in.Model) - return + return bad("no model %s in the catalog", in.Model) } if _, _, _, taken := a.ctl.Catalog().ResolveAdapter(in.Name); taken || in.Name == m.Name { - bad("%s is already a model's or an adapter's name", in.Name) - return + return bad("%s is already a model's or an adapter's name", in.Name) } - if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") { - bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo) - return + if _, taken := a.ctl.Catalog().Resolve(in.Name); taken { + return bad("%s is already a model's or an adapter's name", in.Name) } - if _, ok := a.ctl.Inventory().Get(in.Node); !ok { - bad("no node %s", in.Node) - return + queued, err := a.train.Store.QueuedTrainJobs(ctx) + if err != nil { + return store.TrainJob{}, err + } + active, err := a.train.Store.ActiveTrainJobs(ctx) + if err != nil { + return store.TrainJob{}, err + } + for _, j := range append(queued, active...) { + if j.Name == in.Name { + return bad("%s is already being trained (job %s)", in.Name, j.ID) + } + if in.Project != "" && j.Project == in.Project { + return store.TrainJob{}, refused(http.StatusConflict, + "this project already has a model training (%s): one at a time, so that one project does not hold every card", j.Name) + } + } + var weights int64 + if in.BaseRepo == "" { + b := a.trainBase(ctx, m) + if b.err != nil { + return store.TrainJob{}, b.err + } + in.BaseRepo, in.BaseRevision, weights = b.repo, b.revision, b.bytes + } else if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") { + return bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo) + } + if in.Node != "" { + if _, ok := a.ctl.Inventory().Get(in.Node); !ok { + return bad("no node %s", in.Node) + } } if in.GPUMemoryMiB <= 0 { - bad("gpu_memory_mib is what the job needs of a card, in MiB") - return + if weights <= 0 { + return bad("gpu_memory_mib is what the job needs of a card, in MiB") + } + in.GPUMemoryMiB = TrainMemoryMiB(weights) } pairs, colls, err := countPairs(in.Data) if err != nil { - bad("%v", err) - return + return bad("%v", err) } - owner, err := a.trainOwner(r.Context(), in, colls) + owner, err := a.trainOwner(ctx, in, colls) if err != nil { - bad("%v", err) - return + return bad("%v", err) } for k, v := range in.Params { lim, known := trainParams[k] f, isNum := v.(float64) if !known || !isNum || f < lim[0] || f > lim[1] { - bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k) - return + return bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k) } } params := "" @@ -240,15 +356,132 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) { } j := store.TrainJob{ID: store.NewID("train-"), Name: in.Name, Model: m.Name, BaseRepo: in.BaseRepo, BaseRevision: in.BaseRevision, Node: in.Node, Owner: owner, GPUMemoryMiB: in.GPUMemoryMiB, Params: params, GGUF: gguf, - Data: []byte(in.Data), Pairs: pairs, CreatedBy: actorOf(r)} - if err := a.train.Store.CreateTrainJob(r.Context(), j); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return + Data: []byte(in.Data), Pairs: pairs, CreatedBy: actor, Project: in.Project, Publish: in.Publish} + if err := a.train.Store.CreateTrainJob(ctx, j); err != nil { + return store.TrainJob{}, err } // At once rather than at the next look, when the node is here. - go a.train.Service.Dispatch(context.WithoutCancel(r.Context())) + go a.train.Service.Dispatch(context.WithoutCancel(ctx)) j.State = store.TrainQueued - writeJSON(w, http.StatusCreated, trainView(j)) + return j, nil +} + +// TrainMemoryMiB is what training a LoRA adapter on weights of this size +// takes of a card: the weights in 16 bits, as much again for what passes +// through them, and the CUDA context. Measured at 2.3 GB for a 0.5B model. +func TrainMemoryMiB(weightBytes int64) int { + mib := int(weightBytes>>20)*2 + 1024 + return (mib + 255) / 256 * 256 +} + +// trainBaseFor is how long what the registry said of a model's weights is +// kept: it changes when the model is updated, which is rare. +const trainBaseFor = time.Hour + +// trainBase is the safetensors weights to train m on: its own origin's, +// when they are safetensors, or those its card says it was made from — a +// GGUF's base model. +func (a *API) trainBase(ctx context.Context, m catalog.Model) trainBaseInfo { + if m.Origin == nil || m.Origin.Repo == "" { + return trainBaseInfo{err: refused(http.StatusBadRequest, "%s's weights did not come from the model registry, so "+ + "what to train it on is not known: an administrator can name them (llmgw ctl train start --base-repo)", m.Name)} + } + if a.train.Registry == nil { + return trainBaseInfo{err: refused(http.StatusNotImplemented, "the model registry is not reachable from here "+ + "(mirror.registry): the weights to train %s on cannot be found", m.Name)} + } + a.train.mu.Lock() + b, ok := a.train.bases[m.Origin.Repo] + a.train.mu.Unlock() + if ok && time.Since(b.at) < trainBaseFor { + return b + } + b = a.findTrainBase(ctx, m) + if b.err != nil { + var te *trainErr + if !errors.As(b.err, &te) { + return b // the registry did not answer: asked again next time + } + } + b.at = time.Now() + a.train.mu.Lock() + if a.train.bases == nil { + a.train.bases = map[string]trainBaseInfo{} + } + a.train.bases[m.Origin.Repo] = b + a.train.mu.Unlock() + return b +} + +func (a *API) findTrainBase(ctx context.Context, m catalog.Model) trainBaseInfo { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + repo, err := a.train.Registry.RepoAt(ctx, m.Origin.Repo, "") + if err != nil { + return trainBaseInfo{err: fmt.Errorf("asking the model registry about %s: %w", m.Origin.Repo, err)} + } + candidates := []hf.Repo{repo} + for _, id := range repo.BaseModel { + if id == repo.ID || strings.Count(id, "/") != 1 { + continue + } + r, err := a.train.Registry.RepoAt(ctx, id, "") + if err != nil { + continue + } + candidates = append(candidates, r) + } + gated := "" + for _, r := range candidates { + var n int64 + for _, f := range r.Files { + if strings.HasSuffix(strings.ToLower(f.Path), ".safetensors") { + n += f.Bytes + } + } + if n == 0 { + continue + } + if r.Gated.Is() { + gated = r.ID + continue + } + return trainBaseInfo{repo: r.ID, revision: r.Revision, bytes: n} + } + if gated != "" { + return trainBaseInfo{err: refused(http.StatusBadRequest, "the weights to train %s on, %s, are gated on the registry: "+ + "the nodes cannot fetch them without an account that accepted their terms", m.Name, gated)} + } + return trainBaseInfo{err: refused(http.StatusBadRequest, "neither %s, where %s's weights came from, nor the model its card "+ + "names has safetensors weights to train on", m.Origin.Repo, m.Name)} +} + +// PickTrainNode is the node whose freest card has the most memory free, +// if that is need, among those ready and not busy. +func (a *API) PickTrainNode(need int, busy map[string]bool) string { + best, bestFree := "", 0 + for _, n := range a.ctl.Inventory().Nodes() { + if busy[n.ID] || n.State != inventory.Ready { + continue + } + free := 0 + if n.Load != nil { + for _, g := range n.Load.GPUs { + if g.MemoryTotalMB > 0 { + free = max(free, int(g.MemoryTotalMB-g.MemoryUsedMB)) + } + } + } + if free == 0 { + for _, g := range n.GPUs { + free = max(free, g.MemoryTotalMB-g.MemoryUsedMB) + } + } + if free >= need && free > bestFree { + best, bestFree = n.ID, free + } + } + return best } // trainOwner is who a job's adapter will belong to, as scope:ID: one that @@ -453,73 +686,74 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) { if !ok { return } - if a.train.Mirror == nil { - writeError(w, http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+ - "llmgw ctl train download, then llmgw mirror import there, does it by hand") + v, err := a.publishTrained(r.Context(), j, actorOf(r)) + if err != nil { + writeTrainErr(w, err) return } + writeJSON(w, http.StatusOK, v) +} + +// publishTrained puts what a job made on the mirror and makes it the +// model's adapter: the GGUF for a model llama.cpp serves, the PEFT +// directory for one vLLM serves. +func (a *API) publishTrained(ctx context.Context, j store.TrainJob, actor string) (TrainPublishView, error) { + var none TrainPublishView + if a.train.Mirror == nil { + return none, refused(http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+ + "llmgw ctl train download, then llmgw mirror import there, does it by hand") + } if a.models == nil { - writeError(w, http.StatusNotImplemented, "this control plane has no store for the catalog") - return + return none, refused(http.StatusNotImplemented, "this control plane has no store for the catalog") } if j.State != store.TrainDone { - writeError(w, http.StatusConflict, "the job has not finished: it is "+j.State) - return + return none, refused(http.StatusConflict, "%s", "the job has not finished: it is "+j.State) } m, found := a.ctl.Catalog().Resolve(j.Model) if !found { - writeError(w, http.StatusConflict, "the model "+j.Model+" is no longer in the catalog") - return + return none, refused(http.StatusConflict, "%s", "the model "+j.Model+" is no longer in the catalog") } for _, ad := range m.Adapters { if ad.Name == j.Name { - writeError(w, http.StatusConflict, j.Name+" is already "+m.Name+"'s adapter; train under another name") - return + return none, refused(http.StatusConflict, "%s", j.Name+" is already "+m.Name+"'s adapter; train under another name") } } llama := slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" }) source := "trained by llmgw job " + j.ID + " on " + j.Node + " from " + j.BaseRepo var url string if llama { - b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "gguf") + b, err := a.train.Store.TrainArtifact(ctx, j.ID, "gguf") if err != nil { - writeError(w, http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf") - return + return none, refused(http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf") } - res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, j.Name+".gguf", b, source) + res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, j.Name+".gguf", b, source) if err != nil { - writeError(w, http.StatusBadGateway, "keeping it on the mirror: "+err.Error()) - return + return none, refused(http.StatusBadGateway, "%s", "keeping it on the mirror: "+err.Error()) } url = res.URL } else { - b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "peft") + b, err := a.train.Store.TrainArtifact(ctx, j.ID, "peft") if err != nil { - writeError(w, http.StatusConflict, "the job made no PEFT adapter") - return + return none, refused(http.StatusConflict, "the job made no PEFT adapter") } files, err := untarAll(b) if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return + return none, err } for name, data := range files { - res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, name, data, source) + res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, name, data, source) if err != nil { - writeError(w, http.StatusBadGateway, "keeping "+name+" on the mirror: "+err.Error()) - return + return none, refused(http.StatusBadGateway, "%s", "keeping "+name+" on the mirror: "+err.Error()) } url = res.URL[:strings.LastIndex(res.URL, "/")+1] } } ad := catalog.Adapter{Name: j.Name, Source: url, Owner: j.Owner} - if err := checkAdapters(r.Context(), m, []catalog.Adapter{ad}); err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return + if err := checkAdapters(ctx, m, []catalog.Adapter{ad}); err != nil { + return none, refused(http.StatusBadRequest, "%v", err) } - if _, err := a.models.SetAdapters(r.Context(), m.Name, append(slices.Clone(m.Adapters), ad), actorOf(r)); err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return + if _, err := a.models.SetAdapters(ctx, m.Name, append(slices.Clone(m.Adapters), ad), actor); err != nil { + return none, refused(http.StatusBadRequest, "%v", err) } v := TrainPublishView{Adapter: j.Name, Model: m.Name, Source: url, Owner: j.Owner} if llama { @@ -528,7 +762,48 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) { } else { v.Note = "ask for model " + j.Name + "; a replica loads it on the first request" } - writeJSON(w, http.StatusOK, v) + return v, nil +} + +func (a *API) trainLog() *slog.Logger { + if a.train.Log == nil { + return slog.New(slog.DiscardHandler) + } + return a.train.Log +} + +// TrainDone publishes what a job made, when it asked for that: on the +// mirror, as its model's adapter, and for a llama.cpp model with the +// model's replicas replaced, one at a time beside themselves, to load it. +func (a *API) TrainDone(j store.TrainJob) { + if !j.Publish || a.train == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + note := "published: ask for model " + j.Name + v, err := a.publishTrained(ctx, j, "training:"+j.ID) + if err != nil { + note = "trained, but publishing it failed: " + err.Error() + a.trainLog().Warn("a trained adapter could not be published", "job", j.ID, "adapter", j.Name, "err", err) + } else if strings.HasPrefix(v.Note, "llama.cpp") { + var ids []string + for _, rep := range a.ctl.Reconciler().ReadyReplicas() { + if rep.Model == v.Model { + ids = append(ids, rep.ID) + } + } + if len(ids) > 0 { + if err := a.ctl.Reconciler().RestartPaced(reconciler.RestartPace{}, false, ids...); err != nil { + note = "published, but " + v.Model + "'s replicas could not be restarted to load it: " + err.Error() + } else { + note = "published: " + v.Model + "'s replicas are restarting to load it; then ask for model " + j.Name + } + } + } + if err := a.train.Store.SetTrainNote(ctx, j.ID, note); err != nil { + a.trainLog().Warn("could not record a trained adapter's publishing", "job", j.ID, "err", err) + } } // untarAll reads a gzipped tar of a PEFT directory's files. diff --git a/internal/control/adminapi/train_portal.go b/internal/control/adminapi/train_portal.go new file mode 100644 index 0000000..1bf562c --- /dev/null +++ b/internal/control/adminapi/train_portal.go @@ -0,0 +1,114 @@ +package adminapi + +import ( + "context" + "errors" + "net/http" + "slices" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// Training from the portal's "customise a model" page: the page decides who +// may do what (members of the job's project), and these do it. + +// TrainableModels are the catalog's models an adapter can be trained for +// from the portal: chat models llama.cpp or vLLM serve whose weights came +// from the registry, so the weights to train on can be found. +func (a *API) TrainableModels() []string { + var out []string + for _, m := range a.ctl.Catalog().Models() { + if m.IsGuard() || m.IsExternal() || m.Capabilities.Embeddings || m.MakesImages() || m.MakesMeshes() || + m.Origin == nil || m.Origin.Repo == "" { + continue + } + if slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" || p.Engine == "vllm" }) { + out = append(out, m.Name) + } + } + slices.Sort(out) + return out +} + +// Trainings are the jobs started in projects, or by createdBy outside any. +func (a *API) Trainings(ctx context.Context, projects []string, createdBy string) ([]store.TrainJob, error) { + if a.train == nil { + return nil, nil + } + return a.train.Store.TrainJobsFor(ctx, projects, createdBy, 100) +} + +// Training is one job. +func (a *API) Training(ctx context.Context, id string) (store.TrainJob, error) { + if a.train == nil { + return store.TrainJob{}, store.ErrNotFound + } + return a.train.Store.TrainJob(ctx, id) +} + +// Published is whether a job's adapter is its model's in the catalog. +func (a *API) Published(j store.TrainJob) bool { + m, ok := a.ctl.Catalog().Resolve(j.Model) + return ok && slices.ContainsFunc(m.Adapters, func(ad catalog.Adapter) bool { return ad.Name == j.Name }) +} + +// ShareTraining changes who a job's adapter belongs to — owner as scope:ID, +// or "" for every caller its model allows — on the job and, once +// published, in the catalog. Refused wider than the knowledge bases its +// pairs came from may be read. +func (a *API) ShareTraining(ctx context.Context, id, owner, actor string) error { + j, err := a.Training(ctx, id) + if err != nil { + return err + } + _, colls, err := countPairs(string(j.Data)) + if err != nil { + return err + } + resolved, err := a.trainOwner(ctx, TrainRequest{Owner: owner, Public: owner == ""}, colls) + if err != nil { + return refused(http.StatusBadRequest, "%v", err) + } + if err := a.train.Store.SetTrainOwner(ctx, id, resolved); err != nil { + return err + } + m, ok := a.ctl.Catalog().Resolve(j.Model) + if !ok || a.models == nil { + return nil + } + ads := slices.Clone(m.Adapters) + i := slices.IndexFunc(ads, func(ad catalog.Adapter) bool { return ad.Name == j.Name }) + if i < 0 { + return nil + } + ads[i].Owner = resolved + _, err = a.models.SetAdapters(ctx, m.Name, ads, actor) + return err +} + +// DeleteTraining stops a job if it trains, takes its adapter out of the +// catalog if it was published, and forgets it. +func (a *API) DeleteTraining(ctx context.Context, id, actor string) error { + j, err := a.Training(ctx, id) + if err != nil { + return err + } + if !store.TrainOver(j.State) { + if err := a.train.Service.Cancel(ctx, id); err != nil { + return err + } + } + if m, ok := a.ctl.Catalog().Resolve(j.Model); ok && a.models != nil { + ads := slices.DeleteFunc(slices.Clone(m.Adapters), func(ad catalog.Adapter) bool { return ad.Name == j.Name }) + if len(ads) != len(m.Adapters) { + if _, err := a.models.SetAdapters(ctx, m.Name, ads, actor); err != nil { + return err + } + } + } + if err := a.train.Store.DeleteTrainJob(ctx, id); err != nil && !errors.Is(err, store.ErrNotFound) { + return err + } + return nil +} diff --git a/internal/control/adminapi/train_portal_test.go b/internal/control/adminapi/train_portal_test.go new file mode 100644 index 0000000..c1829a4 --- /dev/null +++ b/internal/control/adminapi/train_portal_test.go @@ -0,0 +1,137 @@ +package adminapi + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +type fakeRegistry struct { + repos map[string]hf.Repo + asked int +} + +func (f *fakeRegistry) RepoAt(_ context.Context, id, _ string) (hf.Repo, error) { + f.asked++ + r, ok := f.repos[id] + if !ok { + return hf.Repo{}, errors.New("404") + } + return r, nil +} + +func TestAPortalTrainingFindsItsWeightsAndMemoryAndOneTrainsPerProject(t *testing.T) { + ctx := context.Background() + models := append(testModels(), + catalog.Model{Name: "small", Source: "/s", Origin: &catalog.Origin{Repo: "bartowski/Small-GGUF"}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}, + catalog.Model{Name: "gated", Source: "/g", Origin: &catalog.Origin{Repo: "meta/Gated"}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}, + catalog.Model{Name: "local", Source: "/l", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}) + a, _ := setup(t, models, gpuNode("cuda01", 1)) + s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + reg := &fakeRegistry{repos: map[string]hf.Repo{ + "bartowski/Small-GGUF": {ID: "bartowski/Small-GGUF", Revision: "g1", BaseModel: []string{"Org/Small"}, + Files: []hf.File{{Path: "small-Q4_K_M.gguf", Bytes: 400 << 20}}}, + "Org/Small": {ID: "Org/Small", Revision: "abc", Files: []hf.File{{Path: "model.safetensors", Bytes: 1000 << 20}, + {Path: "config.json", Bytes: 1 << 10}}}, + "meta/Gated": {ID: "meta/Gated", Gated: hf.Gated{Reason: "manual"}, + Files: []hf.File{{Path: "model.safetensors", Bytes: 1 << 30}}}, + }} + a.SetTrain(&TrainAdmin{Store: s, Service: train.New(s, noAgents{}, nil)}) + a.SetTrainMirror(nil, reg) + + if got := a.TrainableModels(); strings.Join(got, ",") != "gated,small" { + t.Errorf("trainable: %v", got) + } + data := train.NDJSON([]train.Pair{{Q: "a", A: "b"}}) + req := TrainRequest{Name: "tone", Model: "small", Data: data, Owner: "project:p1", Project: "p1", Publish: true} + j, err := a.StartTraining(ctx, req, "ayse@x") + if err != nil { + t.Fatal(err) + } + if j.BaseRepo != "Org/Small" || j.BaseRevision != "abc" || j.Node != "" || j.GPUMemoryMiB != TrainMemoryMiB(1000<<20) || + !j.Publish || j.Project != "p1" || j.CreatedBy != "ayse@x" || !j.GGUF { + t.Errorf("queued %+v", j) + } + if j.GPUMemoryMiB != 3072 { + t.Errorf("1000 MiB of weights take %d MiB to train", j.GPUMemoryMiB) + } + + req.Name = "tone2" + if _, err := a.StartTraining(ctx, req, "mehmet@x"); err == nil || !strings.Contains(err.Error(), "already has a model training") { + t.Errorf("a second in the project: %v", err) + } + req.Name, req.Project, req.Owner = "tone", "p2", "project:p2" + if _, err := a.StartTraining(ctx, req, "x"); err == nil || !strings.Contains(err.Error(), "already being trained") { + t.Errorf("the same name twice: %v", err) + } + req.Name = "tone3" + asked := reg.asked + if _, err := a.StartTraining(ctx, req, "x"); err != nil { + t.Errorf("another project: %v", err) + } + if reg.asked != asked { + t.Errorf("the registry was asked again: %d", reg.asked-asked) + } + for model, says := range map[string]string{"gated": "gated", "local": "did not come from the model registry"} { + req.Model, req.Name, req.Project = model, "x-"+model, "p-"+model + _, err := a.StartTraining(ctx, req, "x") + if err == nil || !strings.Contains(err.Error(), says) { + t.Errorf("%s: %v", model, err) + } + if msg, theirs := TrainRefusal(err); !theirs || msg == "" { + t.Errorf("%s is not the request's fault: %q", model, msg) + } + } + + jobs, err := a.Trainings(ctx, []string{"p1"}, "nobody") + if err != nil || len(jobs) != 1 || jobs[0].Name != "tone" { + t.Errorf("p1's: %+v %v", jobs, err) + } + if err := a.ShareTraining(ctx, j.ID, "", "ayse@x"); err != nil { + t.Fatal(err) + } + if got, _ := a.Training(ctx, j.ID); got.Owner != "" { + t.Errorf("shared with everyone: %q", got.Owner) + } + if err := a.DeleteTraining(ctx, j.ID, "ayse@x"); err != nil { + t.Fatal(err) + } + if _, err := a.Training(ctx, j.ID); !errors.Is(err, store.ErrNotFound) { + t.Errorf("deleted: %v", err) + } +} + +func TestATrainingGoesToTheNodeWithTheMostFree(t *testing.T) { + busy := gpuNode("busy", 1) + busy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 23000}}} + roomy := gpuNode("roomy", 1) + roomy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 4000}}} + cpu := inventory.Node{ID: "cpu", State: inventory.Ready} + small := gpuNode("small", 1) + small.GPUs = []discovery.GPU{{Index: 0, MemoryTotalMB: 8000}} + a, _ := setup(t, testModels(), busy, roomy, cpu, small) + if got := a.PickTrainNode(3000, nil); got != "roomy" { + t.Errorf("picked %q", got) + } + if got := a.PickTrainNode(3000, map[string]bool{"roomy": true}); got != "small" { + t.Errorf("with roomy busy, picked %q", got) + } + if got := a.PickTrainNode(30000, nil); got != "" { + t.Errorf("none has 30 GB, picked %q", got) + } +} diff --git a/internal/control/train/pairs.go b/internal/control/train/pairs.go new file mode 100644 index 0000000..8dc96e9 --- /dev/null +++ b/internal/control/train/pairs.go @@ -0,0 +1,205 @@ +package train + +import ( + "bufio" + "bytes" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + "unicode/utf8" + + "gitea.mynodes.xyz/ilker/llmgw/internal/kb/extract" +) + +// Pairs from a file a person uploads: a spreadsheet of questions and +// answers (Excel, CSV, tab-separated) or llmgw's own NDJSON of {"q", "a"}. +// The columns are found by their header — Soru/Cevap, Question/Answer and +// the like — or are the first two when the first row is not a header. + +// Pair is one question and the answer to learn for it. +type Pair struct { + Q string `json:"q"` + A string `json:"a"` +} + +// Read is what a file gave. +type Read struct { + Pairs []Pair + // Skipped counts rows with a question and no answer, or the other way. + Skipped int + // Columns names the columns read, as the header has them; empty for + // NDJSON and for a table with no header. + Columns [2]string +} + +// MaxPairs bounds one file: past it a job trains for hours on a page that +// was meant to take minutes. +const MaxPairs = 20000 + +var ( + questionNames = []string{"soru", "sorular", "question", "questions", "q", "prompt", "instruction", "input", "girdi"} + answerNames = []string{"cevap", "cevaplar", "yanıt", "yanit", "answer", "answers", "a", "response", "output", "completion", "çıktı", "cikti"} +) + +// ReadPairs reads a file's pairs; name says what kind it is. +func ReadPairs(name string, b []byte) (Read, error) { + var rows [][]string + switch strings.ToLower(path.Ext(name)) { + case ".xlsx", ".xlsm": + r, err := extract.XLSXRows(b) + if err != nil { + return Read{}, fmt.Errorf("the Excel file could not be read: %w", err) + } + rows = r + case ".jsonl", ".ndjson", ".json": + return readNDJSON(b) + case ".csv", ".tsv", ".txt": + r, err := readDelimited(b) + if err != nil { + return Read{}, err + } + rows = r + case ".xls": + return Read{}, errors.New("an old Excel file (.xls): save it as .xlsx and choose it again") + default: + return Read{}, errors.New("choose an Excel (.xlsx), CSV or JSONL file") + } + return fromRows(rows) +} + +func readDelimited(b []byte) ([][]string, error) { + b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf")) + if !utf8.Valid(b) { + return nil, errors.New("the file is not UTF-8 text: save it as \"CSV UTF-8\"") + } + first, _, _ := strings.Cut(string(b), "\n") + // Whichever separator the header line has most of: Excel writes ";" + // where the decimal mark is a comma, as it is in Turkish. + sep, most := ',', strings.Count(first, ",") + for _, c := range []rune{';', '\t'} { + if n := strings.Count(first, string(c)); n > most { + sep, most = c, n + } + } + r := csv.NewReader(bytes.NewReader(b)) + r.Comma, r.FieldsPerRecord, r.LazyQuotes = sep, -1, true + rows, err := r.ReadAll() + if err != nil { + return nil, fmt.Errorf("the file could not be read as a table: %w", err) + } + return rows, nil +} + +func fromRows(rows [][]string) (Read, error) { + var out Read + qc, ac := 0, 1 + if len(rows) > 0 { + q, a := -1, -1 + for i, c := range rows[0] { + n := strings.ToLower(strings.TrimSpace(c)) + switch { + case q < 0 && contains(questionNames, n): + q = i + case a < 0 && contains(answerNames, n): + a = i + } + } + if q >= 0 && a >= 0 { + qc, ac = q, a + out.Columns = [2]string{strings.TrimSpace(rows[0][q]), strings.TrimSpace(rows[0][a])} + rows = rows[1:] + } + } + for _, r := range rows { + var q, a string + if qc < len(r) { + q = strings.TrimSpace(r[qc]) + } + if ac < len(r) { + a = strings.TrimSpace(r[ac]) + } + switch { + case q == "" && a == "": + case q == "" || a == "": + out.Skipped++ + default: + out.Pairs = append(out.Pairs, Pair{Q: q, A: a}) + } + } + return out, check(out) +} + +func contains(names []string, n string) bool { + for _, x := range names { + if x == n { + return true + } + } + return false +} + +func readNDJSON(b []byte) (Read, error) { + var out Read + sc := bufio.NewScanner(bytes.NewReader(b)) + sc.Buffer(make([]byte, 64<<10), 1<<20) + line := 0 + for sc.Scan() { + line++ + l := bytes.TrimSpace(sc.Bytes()) + if len(l) == 0 { + continue + } + var p struct { + Q, A string + Question, Answer string + Prompt, Completion string + } + if err := json.Unmarshal(l, &p); err != nil { + return Read{}, fmt.Errorf("line %d is not JSON: %v", line, err) + } + q, a := first(p.Q, p.Question, p.Prompt), first(p.A, p.Answer, p.Completion) + if q == "" || a == "" { + out.Skipped++ + continue + } + out.Pairs = append(out.Pairs, Pair{Q: q, A: a}) + } + if err := sc.Err(); err != nil { + return Read{}, err + } + return out, check(out) +} + +func first(s ...string) string { + for _, x := range s { + if x = strings.TrimSpace(x); x != "" { + return x + } + } + return "" +} + +func check(r Read) error { + switch { + case len(r.Pairs) == 0: + return errors.New("no question and answer found: the first column is the question and the second the answer, " + + "or columns headed Soru and Cevap") + case len(r.Pairs) > MaxPairs: + return fmt.Errorf("%d pairs, more than the %d one training takes", len(r.Pairs), MaxPairs) + } + return nil +} + +// NDJSON is the pairs as a job's data. +func NDJSON(pairs []Pair) string { + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + for _, p := range pairs { + _ = enc.Encode(p) + } + return b.String() +} diff --git a/internal/control/train/pairs_test.go b/internal/control/train/pairs_test.go new file mode 100644 index 0000000..84c4c1b --- /dev/null +++ b/internal/control/train/pairs_test.go @@ -0,0 +1,65 @@ +package train + +import ( + "strings" + "testing" +) + +func TestPairsFromATableByItsHeaderOrItsFirstColumns(t *testing.T) { + for _, c := range []struct { + name, file string + want []Pair + skipped int + cols [2]string + }{ + {"headed.csv", "No;Cevap;Soru\n1;Yirmi gün.;İzin kaç gün?\n2;;Boş cevap\n3;Evet;Uzaktan çalışabilir miyim?\n", + []Pair{{"İzin kaç gün?", "Yirmi gün."}, {"Uzaktan çalışabilir miyim?", "Evet"}}, 1, [2]string{"Soru", "Cevap"}}, + {"plain.csv", "\ufeffİzin kaç gün?,\"Yirmi gün, yılda.\"\n\nMasraf?,Formla.\n", + []Pair{{"İzin kaç gün?", "Yirmi gün, yılda."}, {"Masraf?", "Formla."}}, 0, [2]string{}}, + {"tabs.tsv", "Question\tAnswer\nA?\tB.\n", []Pair{{"A?", "B."}}, 0, [2]string{"Question", "Answer"}}, + {"d.jsonl", `{"q":"A?","a":"B."}` + "\n" + `{"question":"C?","answer":"D."}` + "\n" + `{"q":"E?"}` + "\n", + []Pair{{"A?", "B."}, {"C?", "D."}}, 1, [2]string{}}, + } { + got, err := ReadPairs(c.name, []byte(c.file)) + if err != nil { + t.Errorf("%s: %v", c.name, err) + continue + } + if len(got.Pairs) != len(c.want) || got.Skipped != c.skipped || got.Columns != c.cols { + t.Errorf("%s: %+v", c.name, got) + continue + } + for i := range c.want { + if got.Pairs[i] != c.want[i] { + t.Errorf("%s: pair %d = %+v, want %+v", c.name, i, got.Pairs[i], c.want[i]) + } + } + } +} + +func TestPairsRefusedWithAReason(t *testing.T) { + for name, file := range map[string]string{ + "a.csv": "only one column\n", + "a.xls": "x", + "a.pdf": "x", + "a.json": "not json\n", + } { + if _, err := ReadPairs(name, []byte(file)); err == nil { + t.Errorf("%s read", name) + } + } + var b strings.Builder + for range MaxPairs + 1 { + b.WriteString("x,y\n") + } + if _, err := ReadPairs("big.csv", []byte(b.String())); err == nil || !strings.Contains(err.Error(), "more than") { + t.Errorf("too many: %v", err) + } +} + +func TestNDJSONIsWhatAJobReads(t *testing.T) { + got := NDJSON([]Pair{{"?", "a & b"}}) + if got != `{"q":"?","a":"a & b"}`+"\n" { + t.Errorf("%q", got) + } +} diff --git a/internal/control/train/service.go b/internal/control/train/service.go index b2a646d..507b619 100644 --- a/internal/control/train/service.go +++ b/internal/control/train/service.go @@ -35,11 +35,26 @@ type Service struct { agents Agents log *slog.Logger + // pick chooses the node for a job that names none, among those not + // busy; "" is none can take it now. Nil leaves such jobs queued. + pick func(needMiB int, busy map[string]bool) string + // done is told of a job that finished, after what it made is kept. + done func(j store.TrainJob) + mu sync.Mutex // pieces are artifacts arriving, by job and kind. pieces map[[2]string][]byte } +// SetPicker has jobs that name no node sent where pick says. +func (s *Service) SetPicker(pick func(needMiB int, busy map[string]bool) string) { s.pick = pick } + +// SetOnDone has done told of each job that finishes. +func (s *Service) SetOnDone(done func(j store.TrainJob)) { s.done = done } + +// Waiting is what a job that names no node says while no node can take it. +const Waiting = "waiting for a card with enough memory free" + // New is a service. func New(s Store, a Agents, log *slog.Logger) *Service { if log == nil { @@ -81,13 +96,27 @@ func (s *Service) Dispatch(ctx context.Context) { busy[j.Node] = true } for _, q := range queued { - if busy[q.Node] || !s.agents.Connected(q.Node) { + node := q.Node + if node == "" { + if s.pick != nil { + node = s.pick(q.GPUMemoryMiB, busy) + } + if node == "" { + if q.Message != Waiting { + q.Message = Waiting + _ = s.store.UpdateTrainJob(ctx, q) + } + continue + } + } + if busy[node] || !s.agents.Connected(node) { continue } j, err := s.store.TrainJob(ctx, q.ID) if err != nil { continue } + j.Node = node err = s.agents.Send(j.Node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_TrainStart{TrainStart: &llmgwv1.TrainStart{ JobId: j.ID, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision, Data: j.Data, Params: j.Params, GpuMemoryMib: int32(j.GPUMemoryMiB), Gguf: j.GGUF}}}) //nolint:gosec // MiB @@ -135,6 +164,11 @@ func (s *Service) OnTrainStatus(node string, st *llmgwv1.TrainStatus) { } if err := s.store.UpdateTrainJob(ctx, j); err != nil { s.log.Warn("could not record a training job's progress", "job", j.ID, "err", err) + return + } + // What it made came before it said it was done, on the same stream. + if j.State == store.TrainDone && s.done != nil { + go s.done(j) } } diff --git a/internal/control/train/service_test.go b/internal/control/train/service_test.go index 5a7a8f0..dd3ed6c 100644 --- a/internal/control/train/service_test.go +++ b/internal/control/train/service_test.go @@ -122,3 +122,41 @@ func TestCancellingAJob(t *testing.T) { t.Errorf("the node was not told: %+v", a.sent) } } + +func TestAJobNamingNoNodeGoesWhereThePickerSaysAndIsToldWhenDone(t *testing.T) { + ctx := context.Background() + svc, s, a := setup(t) + job(t, s, "j1", "") + asked := 0 + free := "" + svc.SetPicker(func(need int, busy map[string]bool) string { + asked++ + if need != 2000 { + t.Errorf("asked for %d MiB", need) + } + return free + }) + done := make(chan store.TrainJob, 1) + svc.SetOnDone(func(j store.TrainJob) { done <- j }) + + svc.Dispatch(ctx) + if j, _ := s.TrainJob(ctx, "j1"); j.State != store.TrainQueued || j.Message != Waiting || len(a.sent) != 0 { + t.Fatalf("with no card free: %+v, sent %d", j, len(a.sent)) + } + free = "gpu2" + a.connected["gpu2"] = true + svc.Dispatch(ctx) + j, _ := s.TrainJob(ctx, "j1") + if len(a.sent) != 1 || j.Node != "gpu2" || j.State != store.TrainStarting { + t.Fatalf("picked: %+v, sent %d", j, len(a.sent)) + } + // Its node is the one it was sent to, which alone reports for it. + svc.OnTrainStatus("cuda01", &llmgwv1.TrainStatus{JobId: "j1", State: "done"}) + if j, _ := s.TrainJob(ctx, "j1"); j.State == store.TrainDone { + t.Fatal("another node finished it") + } + svc.OnTrainStatus("gpu2", &llmgwv1.TrainStatus{JobId: "j1", State: "done"}) + if got := <-done; got.ID != "j1" || got.Node != "gpu2" { + t.Errorf("told of %+v", got) + } +} diff --git a/internal/kb/extract/extract_test.go b/internal/kb/extract/extract_test.go index 73157ab..405cbff 100644 --- a/internal/kb/extract/extract_test.go +++ b/internal/kb/extract/extract_test.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "errors" "os" + "reflect" "strings" "testing" @@ -321,3 +322,25 @@ func TestHelpersWhenInstalled(t *testing.T) { } wants(t, got, "LibreOffice yolu") } + +func TestWorkbookRowsKeepTheirColumns(t *testing.T) { + b := zipOf(t, + "xl/workbook.xml", ``, + "xl/_rels/workbook.xml.rels", ``, + "xl/sharedStrings.xml", `SoruCevapİzin kaç gün?`, + "xl/worksheets/sheet1.xml", ``+ + `01`+ + ``+ + `2not`+ + `20`+ + ``, + ) + rows, err := XLSXRows(b) + if err != nil { + t.Fatal(err) + } + want := [][]string{{"Soru", "Cevap"}, {"İzin kaç gün?", "", "not"}, {"", "20"}} + if !reflect.DeepEqual(rows, want) { + t.Errorf("rows = %q, want %q", rows, want) + } +} diff --git a/internal/kb/extract/ooxml.go b/internal/kb/extract/ooxml.go index 8ca4eb4..0d2ce07 100644 --- a/internal/kb/extract/ooxml.go +++ b/internal/kb/extract/ooxml.go @@ -9,6 +9,7 @@ import ( "io" "path" "regexp" + "slices" "sort" "strconv" "strings" @@ -227,29 +228,7 @@ func xlsxText(b []byte) (string, error) { if err != nil { return "", err } - var shared []string - if ss, _ := member(zr, "xl/sharedStrings.xml"); ss != nil { - var cur strings.Builder - inT, inRPh := false, 0 - _ = walkXML(ss, func(e xmlEvent) { - switch { - case e.start && e.name == "si": - cur.Reset() - case e.end && e.name == "si": - shared = append(shared, cur.String()) - case e.start && e.name == "rPh": - inRPh++ // phonetic guides, not the cell's text - case e.end && e.name == "rPh": - inRPh-- - case e.start && e.name == "t": - inT = true - case e.end && e.name == "t": - inT = false - case !e.start && !e.end && inT && inRPh == 0: - cur.WriteString(e.text) - } - }) - } + shared := xlsxShared(zr) sheets, err := xlsxSheets(zr) if err != nil { return "", err @@ -307,6 +286,118 @@ func xlsxText(b []byte) (string, error) { return o.String(), nil } +// xlsxShared is a workbook's shared strings, which cells name by index. +func xlsxShared(zr *zip.Reader) []string { + var shared []string + ss, _ := member(zr, "xl/sharedStrings.xml") + if ss == nil { + return nil + } + var cur strings.Builder + inT, inRPh := false, 0 + _ = walkXML(ss, func(e xmlEvent) { + switch { + case e.start && e.name == "si": + cur.Reset() + case e.end && e.name == "si": + shared = append(shared, cur.String()) + case e.start && e.name == "rPh": + inRPh++ // phonetic guides, not the cell's text + case e.end && e.name == "rPh": + inRPh-- + case e.start && e.name == "t": + inT = true + case e.end && e.name == "t": + inT = false + case !e.start && !e.end && inT && inRPh == 0: + cur.WriteString(e.text) + } + }) + return shared +} + +// XLSXRows reads a workbook's first sheet as rows of cells, each cell in +// its column (an empty one is ""), for a table such as questions and +// answers. Rows with nothing in them are left out. +func XLSXRows(b []byte) ([][]string, error) { + zr, err := openZip(b) + if err != nil { + return nil, err + } + shared := xlsxShared(zr) + sheets, err := xlsxSheets(zr) + if err != nil { + return nil, err + } + if len(sheets) == 0 { + return nil, errors.New("the workbook has no sheets") + } + data, err := member(zr, sheets[0].path) + if err != nil || data == nil { + return nil, fmt.Errorf("the workbook's first sheet, %s, could not be read", sheets[0].name) + } + var rows [][]string + var row []string + var cell strings.Builder + cellType, col, next := "", 0, 0 + inV, inIs := false, false + _ = walkXML(data, func(e xmlEvent) { + switch { + case e.start && e.name == "row": + row, next = nil, 0 + case e.end && e.name == "row": + if slices.ContainsFunc(row, func(s string) bool { return s != "" }) { + rows = append(rows, row) + } + case e.start && e.name == "c": + cellType = e.get("t") + col = next + if c, ok := columnOf(e.get("r")); ok { + col = c + } + next = col + 1 + cell.Reset() + case e.end && e.name == "c": + v := cell.String() + if cellType == "s" { + if i, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && i >= 0 && i < len(shared) { + v = shared[i] + } + } + if col < 0 || col > 16383 { + return + } + for len(row) <= col { + row = append(row, "") + } + row[col] = strings.TrimSpace(v) + case e.start && e.name == "v": + inV = true + case e.end && e.name == "v": + inV = false + case e.start && e.name == "is": + inIs = true + case e.end && e.name == "is": + inIs = false + case !e.start && !e.end && (inV || inIs): + cell.WriteString(e.text) + } + }) + return rows, nil +} + +// columnOf is a cell reference's column, from 0: "B7" is 1. +func columnOf(ref string) (int, bool) { + n, i := 0, 0 + for ; i < len(ref) && ref[i] >= 'A' && ref[i] <= 'Z'; i++ { + n = n*26 + int(ref[i]-'A'+1) + } + if i == 0 { + return 0, false + } + return n - 1, true +} + type sheetRef struct{ name, path string } // xlsxSheets lists a workbook's sheets in their order, with the part each is diff --git a/internal/router/chat.go b/internal/router/chat.go index d3bdbc0..0f634df 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -58,6 +58,7 @@ func (r *Router) registerChat(mux *http.ServeMux, pages *web.AccountPages) { })) r.registerImagesPage(mux, pages) r.registerMeshesPage(mux, pages) + r.registerTrainPage(mux, pages) if r.store != nil { pages.SetLicences(r.licenceChoices, r.store) } diff --git a/internal/router/handlers.go b/internal/router/handlers.go index cba291d..0407f76 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -384,7 +384,7 @@ func (r *Router) Handler() http.Handler { } exempt := []string{"/health", "/ready", "/leader", "/v1/openapi.json", "/", "/about", "/install", "/install/", "/_static/", "/_lang/", "/claim/", "/signup", "/login", "/login/", "/logout", "/profile", "/profile/", "/scim/", - "/admin", "/admin/", "/chat", "/chat/", "/images", "/images/", "/3d", "/3d/", "/org", "/org/", "/setup/"} + "/admin", "/admin/", "/chat", "/chat/", "/images", "/images/", "/3d", "/3d/", "/train", "/train/", "/org", "/org/", "/setup/"} // The docs are public unless the site said otherwise, and then guarded by // the session check in front of them — or, on a gateway with keys and no // accounts to sign in to, by the key like any other path. diff --git a/internal/router/pages_auth_test.go b/internal/router/pages_auth_test.go index 1e21fe0..c5f02e1 100644 --- a/internal/router/pages_auth_test.go +++ b/internal/router/pages_auth_test.go @@ -17,7 +17,7 @@ func TestSessionPagesAreNotAskedForAnAPIKey(t *testing.T) { h := rt.Handler() // The /admin pages are mounted only with a cluster view, which this // router has none of; these are the session pages it does mount. - for _, path := range []string{"/profile", "/chat", "/images", "/3d", "/org", "/org/org-x"} { + for _, path := range []string{"/profile", "/chat", "/images", "/3d", "/train", "/org", "/org/org-x"} { w := httptest.NewRecorder() h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) if strings.Contains(w.Body.String(), "api key") { diff --git a/internal/router/router.go b/internal/router/router.go index d851fc8..4933859 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -192,6 +192,9 @@ type Router struct { // registry browses a model registry from the mirror page. Nil where the // deployment has no way out to the internet. registry web.Registry + // training is the "customise a model" page's control plane; nil + // leaves the page off. + training atomic.Pointer[Training] // catalogAdmin puts a mirrored model into service. Nil without a store, // where a model added at runtime would not survive a restart. catalogAdmin web.CatalogAdmin diff --git a/internal/router/train.go b/internal/router/train.go new file mode 100644 index 0000000..8722b69 --- /dev/null +++ b/internal/router/train.go @@ -0,0 +1,331 @@ +package router + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "slices" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" + "gitea.mynodes.xyz/ilker/llmgw/internal/web" +) + +// The "customise a model" page: a person in a project uploads questions +// and answers, picks a model, and presses a button. The node, the weights +// to train on and how much of a card it takes are worked out; what it +// makes is published as a model of its own, seen by the project unless it +// is shared wider. Members of the project see its jobs and manage them; +// one trains at a time per project. + +// Training is what the page needs of the control plane. +type Training interface { + TrainableModels() []string + StartTraining(ctx context.Context, in adminapi.TrainRequest, actor string) (store.TrainJob, error) + Trainings(ctx context.Context, projects []string, createdBy string) ([]store.TrainJob, error) + Training(ctx context.Context, id string) (store.TrainJob, error) + Published(j store.TrainJob) bool + ShareTraining(ctx context.Context, id, owner, actor string) error + DeleteTraining(ctx context.Context, id, actor string) error +} + +// SetTraining mounts the page's actions on t. +func (r *Router) SetTraining(t Training) { r.training.Store(&t) } + +func (r *Router) trainer() Training { + if t := r.training.Load(); t != nil { + return *t + } + return nil +} + +// trainMaxFile bounds an uploaded file of pairs. +const trainMaxFile = 32 << 20 + +// The widths an adapter can be shared at, narrowest first. +var trainShares = []string{"me", "project", "team", "org", "all"} + +func (r *Router) registerTrainPage(mux *http.ServeMux, pages *web.AccountPages) { + pages.SetTrain(func() bool { + t := r.trainer() + return t != nil && r.store != nil && len(t.TrainableModels()) > 0 + }) + mux.HandleFunc("GET /train", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) { + t := r.trainer() + if t == nil || r.store == nil { + http.NotFound(w, req) + return + } + projects, _ := r.store.ProjectsFor(req.Context(), user.ID) + var ps []web.ChatContext + for _, p := range projects { + ps = append(ps, web.ChatContext{Value: p.ID, Label: p.Path}) + } + web.Train(user, t.TrainableModels(), ps, version.Version)(w, req) + })) + mux.HandleFunc("POST /train/preview", pages.Guard(r.trainPreview)) + mux.HandleFunc("POST /train/start", pages.Guard(r.trainStart)) + mux.HandleFunc("GET /train/jobs", pages.Guard(r.trainJobs)) + mux.HandleFunc("POST /train/jobs/{id}/share", pages.Guard(r.trainShare)) + mux.HandleFunc("POST /train/jobs/{id}/delete", pages.Guard(r.trainDelete)) +} + +func trainFail(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} + +// trainFile reads the pairs of the uploaded file. +func trainFile(w http.ResponseWriter, req *http.Request) (train.Read, bool) { + req.Body = http.MaxBytesReader(w, req.Body, trainMaxFile+1<<20) + f, h, err := req.FormFile("file") + if err != nil { + trainFail(w, http.StatusBadRequest, "choose a file of questions and answers") + return train.Read{}, false + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, trainMaxFile+1)) + if err != nil || len(b) > trainMaxFile { + trainFail(w, http.StatusBadRequest, "the file is over 32 MB") + return train.Read{}, false + } + rd, err := train.ReadPairs(h.Filename, b) + if err != nil { + trainFail(w, http.StatusBadRequest, err.Error()) + return train.Read{}, false + } + return rd, true +} + +// trainPreview says what a file holds before anything is trained on it. +func (r *Router) trainPreview(w http.ResponseWriter, req *http.Request, _ store.User) { + rd, ok := trainFile(w, req) + if !ok { + return + } + sample := rd.Pairs[:min(len(rd.Pairs), 5)] + writeJSONBody(w, map[string]any{"pairs": len(rd.Pairs), "skipped": rd.Skipped, "columns": rd.Columns, "sample": sample}) +} + +// trainPlace is where a project is in the tree. +func (r *Router) trainPlace(ctx context.Context, user store.User, project string) (store.Effective, error) { + projects, err := r.store.ProjectsFor(ctx, user.ID) + if err != nil { + return store.Effective{}, err + } + if !slices.ContainsFunc(projects, func(p store.ProjectPath) bool { return p.ID == project }) { + return store.Effective{}, errors.New("you are not a member of that project") + } + return r.store.ResolveFor(ctx, "", "", project) +} + +// trainOwner is the adapter's owner for a share width, in a place. +func trainOwner(share string, user store.User, e store.Effective) (string, bool) { + switch share { + case "me": + return "user:" + user.ID, true + case "project": + return "project:" + e.ProjectID, true + case "team": + return "team:" + e.TeamID, true + case "org": + return "org:" + e.OrgID, true + case "all": + return "", true + } + return "", false +} + +// trainShareOf is the width an owner is, seen from the job's place. +func trainShareOf(owner string, e store.Effective) string { + switch { + case owner == "": + return "all" + case strings.HasPrefix(owner, "user:"): + return "me" + case owner == "project:"+e.ProjectID: + return "project" + case owner == "team:"+e.TeamID: + return "team" + case owner == "org:"+e.OrgID: + return "org" + } + return "other" +} + +func (r *Router) trainStart(w http.ResponseWriter, req *http.Request, user store.User) { + t := r.trainer() + if t == nil { + trainFail(w, http.StatusNotFound, "training is not offered here") + return + } + rd, ok := trainFile(w, req) + if !ok { + return + } + e, err := r.trainPlace(req.Context(), user, req.FormValue("project")) + if err != nil { + trainFail(w, http.StatusForbidden, err.Error()) + return + } + share := req.FormValue("share") + owner, ok := trainOwner(share, user, e) + if !ok { + trainFail(w, http.StatusBadRequest, "who may use it: me, the project, the team, the organization or everyone") + return + } + if slices.Index(trainShares, share) > slices.Index(trainShares, "project") && req.FormValue("confirm") != "yes" { + trainFail(w, http.StatusBadRequest, "confirm that the file holds nothing the wider group must not see") + return + } + in := adminapi.TrainRequest{Name: strings.TrimSpace(req.FormValue("name")), Model: req.FormValue("model"), + Owner: owner, Public: owner == "", Data: train.NDJSON(rd.Pairs), Project: e.ProjectID, Publish: true} + if !slices.Contains(t.TrainableModels(), in.Model) { + trainFail(w, http.StatusBadRequest, "choose one of the models offered") + return + } + j, err := t.StartTraining(req.Context(), in, user.Email) + if err != nil { + msg, theirs := adminapi.TrainRefusal(err) + code := http.StatusBadRequest + if !theirs { + code = http.StatusInternalServerError + r.log.Warn("training could not be started from the portal", "user", user.Email, "err", err) + } + trainFail(w, code, msg) + return + } + r.log.Info("training started from the portal", "job", j.ID, "adapter", j.Name, "model", j.Model, + "project", j.Project, "user", user.Email, "pairs", j.Pairs) + writeJSONBody(w, map[string]string{"id": j.ID}) +} + +// trainRow is a job as the page lists it. +type trainRow struct { + ID string `json:"id"` + Name string `json:"name"` + Model string `json:"model"` + Project string `json:"project"` + State string `json:"state"` + Step int `json:"step"` + Steps int `json:"steps"` + Message string `json:"message,omitempty"` + Pairs int `json:"pairs"` + Share string `json:"share"` + Published bool `json:"published"` + CreatedBy string `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + Minutes int `json:"minutes"` +} + +func (r *Router) trainJobs(w http.ResponseWriter, req *http.Request, user store.User) { + t := r.trainer() + if t == nil || r.store == nil { + writeJSONBody(w, []trainRow{}) + return + } + projects, err := r.store.ProjectsFor(req.Context(), user.ID) + if err != nil { + trainFail(w, http.StatusInternalServerError, err.Error()) + return + } + ids, names := []string{}, map[string]string{} + for _, p := range projects { + ids = append(ids, p.ID) + names[p.ID] = p.Path + } + jobs, err := t.Trainings(req.Context(), ids, user.Email) + if err != nil { + trainFail(w, http.StatusInternalServerError, err.Error()) + return + } + out := []trainRow{} + for _, j := range jobs { + e, _ := r.store.ResolveFor(req.Context(), "", "", j.Project) + row := trainRow{ID: j.ID, Name: j.Name, Model: j.Model, Project: names[j.Project], State: j.State, Step: j.Step, + Steps: j.Steps, Message: j.Message, Pairs: j.Pairs, Share: trainShareOf(j.Owner, e), + Published: j.State == store.TrainDone && t.Published(j), CreatedBy: j.CreatedBy, CreatedAt: j.CreatedAt} + if !j.StartedAt.IsZero() { + end := j.FinishedAt + if end.IsZero() { + end = time.Now() + } + row.Minutes = int(end.Sub(j.StartedAt).Minutes()) + } + out = append(out, row) + } + writeJSONBody(w, out) +} + +// trainMine is a job a person may manage: their project's, or their own. +func (r *Router) trainMine(w http.ResponseWriter, req *http.Request, user store.User) (store.TrainJob, Training, bool) { + t := r.trainer() + if t == nil || r.store == nil { + trainFail(w, http.StatusNotFound, "training is not offered here") + return store.TrainJob{}, nil, false + } + j, err := t.Training(req.Context(), req.PathValue("id")) + if err == nil { + if j.Project == "" { + if j.CreatedBy == user.Email { + return j, t, true + } + } else if _, err := r.trainPlace(req.Context(), user, j.Project); err == nil { + return j, t, true + } + } + trainFail(w, http.StatusNotFound, "no such training") + return store.TrainJob{}, nil, false +} + +func (r *Router) trainShare(w http.ResponseWriter, req *http.Request, user store.User) { + j, t, ok := r.trainMine(w, req, user) + if !ok { + return + } + e, _ := r.store.ResolveFor(req.Context(), "", "", j.Project) + share := req.FormValue("share") + owner, ok := trainOwner(share, user, e) + if !ok || (j.Project == "" && share != "me" && share != "all") { + trainFail(w, http.StatusBadRequest, "who may use it: me, the project, the team, the organization or everyone") + return + } + if slices.Index(trainShares, share) > slices.Index(trainShares, "project") && req.FormValue("confirm") != "yes" { + trainFail(w, http.StatusBadRequest, "confirm that the data holds nothing the wider group must not see") + return + } + if err := t.ShareTraining(req.Context(), j.ID, owner, user.Email); err != nil { + msg, _ := adminapi.TrainRefusal(err) + trainFail(w, http.StatusBadRequest, msg) + return + } + r.log.Info("a trained model's sharing changed from the portal", "job", j.ID, "adapter", j.Name, "share", share, + "user", user.Email) + writeJSONBody(w, map[string]string{"share": share}) +} + +func (r *Router) trainDelete(w http.ResponseWriter, req *http.Request, user store.User) { + j, t, ok := r.trainMine(w, req, user) + if !ok { + return + } + if err := t.DeleteTraining(req.Context(), j.ID, user.Email); err != nil { + msg, _ := adminapi.TrainRefusal(err) + trainFail(w, http.StatusInternalServerError, msg) + return + } + r.log.Info("a trained model was deleted from the portal", "job", j.ID, "adapter", j.Name, "user", user.Email) + writeJSONBody(w, map[string]bool{"deleted": true}) +} + +func writeJSONBody(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) +} diff --git a/internal/router/train_test.go b/internal/router/train_test.go new file mode 100644 index 0000000..bd6ee3e --- /dev/null +++ b/internal/router/train_test.go @@ -0,0 +1,165 @@ +package router + +import ( + "bytes" + "context" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +type fakeTraining struct { + started []adminapi.TrainRequest + jobs map[string]store.TrainJob + shared map[string]string + deleted []string +} + +func (f *fakeTraining) TrainableModels() []string { return []string{"qwen"} } +func (f *fakeTraining) StartTraining(_ context.Context, in adminapi.TrainRequest, actor string) (store.TrainJob, error) { + f.started = append(f.started, in) + j := store.TrainJob{ID: "train-" + in.Name, Name: in.Name, Model: in.Model, Owner: in.Owner, Project: in.Project, + CreatedBy: actor, State: store.TrainQueued, Pairs: strings.Count(in.Data, "\n"), CreatedAt: time.Now()} + f.jobs[j.ID] = j + return j, nil +} +func (f *fakeTraining) Trainings(_ context.Context, projects []string, by string) ([]store.TrainJob, error) { + var out []store.TrainJob + for _, j := range f.jobs { + for _, p := range projects { + if j.Project == p { + out = append(out, j) + } + } + } + return out, nil +} +func (f *fakeTraining) Training(_ context.Context, id string) (store.TrainJob, error) { + j, ok := f.jobs[id] + if !ok { + return j, store.ErrNotFound + } + return j, nil +} +func (f *fakeTraining) Published(store.TrainJob) bool { return false } +func (f *fakeTraining) ShareTraining(_ context.Context, id, owner, _ string) error { + f.shared[id] = owner + return nil +} +func (f *fakeTraining) DeleteTraining(_ context.Context, id, _ string) error { + f.deleted = append(f.deleted, id) + return nil +} + +func trainForm(t *testing.T, fields map[string]string, file string) *http.Request { + t.Helper() + var b bytes.Buffer + mw := multipart.NewWriter(&b) + for k, v := range fields { + _ = mw.WriteField(k, v) + } + if file != "" { + w, _ := mw.CreateFormFile("file", "ornekler.csv") + _, _ = w.Write([]byte(file)) + } + _ = mw.Close() + req := httptest.NewRequest("POST", "/train/start", &b) + req.Header.Set("Content-Type", mw.FormDataContentType()) + return req +} + +func TestThePortalTrainsForAProjectItsMembersManage(t *testing.T) { + cfg := baseConfig() + cfg.Auth.Enabled = true + cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db") + rt := newTestRouter(t, cfg) + ft := &fakeTraining{jobs: map[string]store.TrainJob{}, shared: map[string]string{}} + rt.SetTraining(ft) + ctx := t.Context() + org, _ := rt.store.CreateOrg(ctx, "acme") + team, _ := rt.store.CreateTeam(ctx, org, "support") + project, _ := rt.store.CreateProject(ctx, team, "bot") + ayse := store.User{ID: "usr-a", Email: "ayse@acme.example", State: store.UserActive, CreatedAt: time.Now()} + bob := store.User{ID: "usr-b", Email: "bob@else.example", State: store.UserActive, CreatedAt: time.Now()} + for _, u := range []store.User{ayse, bob} { + if err := rt.store.CreateUser(ctx, u); err != nil { + t.Fatal(err) + } + } + _ = rt.store.AddMembership(ctx, ayse.ID, store.ScopeProject, project, "member", "test", time.Now()) + csv := "Soru,Cevap\nİzin kaç gün?,Yirmi gün.\nMasraf nasıl?,Formla.\n" + start := func(u store.User, fields map[string]string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + rt.trainStart(rec, trainForm(t, fields, csv), u) + return rec + } + good := map[string]string{"name": "destek", "model": "qwen", "project": project, "share": "project"} + + if rec := start(bob, good); rec.Code != http.StatusForbidden { + t.Errorf("someone outside the project started one: %d %s", rec.Code, rec.Body.String()) + } + wide := map[string]string{"name": "destek", "model": "qwen", "project": project, "share": "org"} + if rec := start(ayse, wide); rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "confirm") { + t.Errorf("shared with the organization unconfirmed: %d %s", rec.Code, rec.Body.String()) + } + other := map[string]string{"name": "destek", "model": "llama-70b", "project": project, "share": "project"} + if rec := start(ayse, other); rec.Code != http.StatusBadRequest { + t.Errorf("a model not offered: %d", rec.Code) + } + if rec := start(ayse, good); rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + in := ft.started[0] + if in.Owner != "project:"+project || in.Project != project || !in.Publish || in.Public || + in.Data != `{"q":"İzin kaç gün?","a":"Yirmi gün."}`+"\n"+`{"q":"Masraf nasıl?","a":"Formla."}`+"\n" { + t.Errorf("asked for %+v", in) + } + + list := func(u store.User) []trainRow { + rec := httptest.NewRecorder() + rt.trainJobs(rec, httptest.NewRequest("GET", "/train/jobs", nil), u) + var rows []trainRow + _ = json.Unmarshal(rec.Body.Bytes(), &rows) + return rows + } + if rows := list(ayse); len(rows) != 1 || rows[0].Share != "project" || rows[0].Project != "acme / support / bot" || + rows[0].CreatedBy != ayse.Email { + t.Errorf("ayse sees %+v", rows) + } + if rows := list(bob); len(rows) != 0 { + t.Errorf("bob sees %+v", rows) + } + + act := func(u store.User, what string, fields map[string]string) int { + req := trainForm(t, fields, "") + req.SetPathValue("id", "train-destek") + rec := httptest.NewRecorder() + if what == "share" { + rt.trainShare(rec, req, u) + } else { + rt.trainDelete(rec, req, u) + } + return rec.Code + } + if code := act(bob, "delete", nil); code != http.StatusNotFound || len(ft.deleted) != 0 { + t.Errorf("bob deleted it: %d", code) + } + if code := act(ayse, "share", map[string]string{"share": "team", "confirm": "yes"}); code != http.StatusOK || + ft.shared["train-destek"] != "team:"+team { + t.Errorf("shared with the team: %d %v", code, ft.shared) + } + if code := act(ayse, "share", map[string]string{"share": "all"}); code != http.StatusBadRequest { + t.Errorf("shared with everyone unconfirmed: %d", code) + } + if code := act(ayse, "delete", nil); code != http.StatusOK || len(ft.deleted) != 1 { + t.Errorf("ayse deleting: %d", code) + } +} diff --git a/internal/store/migrations/postgres/0055_train_portal.sql b/internal/store/migrations/postgres/0055_train_portal.sql new file mode 100644 index 0000000..9a9c396 --- /dev/null +++ b/internal/store/migrations/postgres/0055_train_portal.sql @@ -0,0 +1,5 @@ +-- Training from the portal: the project a job was started in, which sees +-- it, manages it and pays for it; and whether what it makes is published +-- by itself when it is done, rather than by an administrator. +ALTER TABLE train_jobs ADD COLUMN project_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE train_jobs ADD COLUMN publish INTEGER NOT NULL DEFAULT 0; diff --git a/internal/store/migrations/sqlite/0055_train_portal.sql b/internal/store/migrations/sqlite/0055_train_portal.sql new file mode 100644 index 0000000..9a9c396 --- /dev/null +++ b/internal/store/migrations/sqlite/0055_train_portal.sql @@ -0,0 +1,5 @@ +-- Training from the portal: the project a job was started in, which sees +-- it, manages it and pays for it; and whether what it makes is published +-- by itself when it is done, rather than by an administrator. +ALTER TABLE train_jobs ADD COLUMN project_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE train_jobs ADD COLUMN publish INTEGER NOT NULL DEFAULT 0; diff --git a/internal/store/train.go b/internal/store/train.go index bed35dd..dc15b9f 100644 --- a/internal/store/train.go +++ b/internal/store/train.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "strings" "time" ) @@ -34,6 +35,11 @@ type TrainJob struct { StartedAt time.Time FinishedAt time.Time CreatedBy string + // Project is the project a job was started in from the portal: its + // members see and manage it, and it pays. Empty for an administrator's. + Project string + // Publish has what the job makes published when it is done. + Publish bool } // The states of a training job. @@ -55,27 +61,25 @@ func (s *DB) CreateTrainJob(ctx context.Context, j TrainJob) error { if j.CreatedAt.IsZero() { j.CreatedAt = time.Now() } - gguf := 0 - if j.GGUF { - gguf = 1 - } _, err := s.db.ExecContext(ctx, `INSERT INTO train_jobs (id, name, model, base_repo, base_revision, node, owner, gpu_memory_mib, - params, gguf, data, pairs, state, created_at, created_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - j.ID, j.Name, j.Model, j.BaseRepo, j.BaseRevision, j.Node, j.Owner, j.GPUMemoryMiB, j.Params, gguf, j.Data, j.Pairs, - TrainQueued, millis(j.CreatedAt), j.CreatedBy) + params, gguf, data, pairs, state, created_at, created_by, project_id, publish) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + j.ID, j.Name, j.Model, j.BaseRepo, j.BaseRevision, j.Node, j.Owner, j.GPUMemoryMiB, j.Params, boolInt(j.GGUF), j.Data, j.Pairs, + TrainQueued, millis(j.CreatedAt), j.CreatedBy, j.Project, boolInt(j.Publish)) return err } const trainCols = `id, name, model, base_repo, base_revision, node, owner, gpu_memory_mib, params, gguf, pairs, state, step, steps, - loss, message, peak_mib, created_at, started_at, finished_at, created_by` + loss, message, peak_mib, created_at, started_at, finished_at, created_by, project_id, publish` func scanTrainJob(sc scanner) (TrainJob, error) { var j TrainJob - var gguf int + var gguf, publish int var created, started, finished int64 err := sc.Scan(&j.ID, &j.Name, &j.Model, &j.BaseRepo, &j.BaseRevision, &j.Node, &j.Owner, &j.GPUMemoryMiB, &j.Params, &gguf, - &j.Pairs, &j.State, &j.Step, &j.Steps, &j.Loss, &j.Message, &j.PeakMiB, &created, &started, &finished, &j.CreatedBy) - j.GGUF = gguf != 0 + &j.Pairs, &j.State, &j.Step, &j.Steps, &j.Loss, &j.Message, &j.PeakMiB, &created, &started, &finished, &j.CreatedBy, + &j.Project, &publish) + j.GGUF, j.Publish = gguf != 0, publish != 0 j.CreatedAt = fromMillis(created) if started > 0 { j.StartedAt = fromMillis(started) @@ -128,13 +132,47 @@ func (s *DB) UpdateTrainJob(ctx context.Context, j TrainJob) error { if !j.FinishedAt.IsZero() { finished = millis(j.FinishedAt) } - _, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET state = ?, step = ?, steps = ?, loss = ?, message = ?, + _, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET node = ?, state = ?, step = ?, steps = ?, loss = ?, message = ?, peak_mib = ?, started_at = ?, finished_at = ? WHERE id = ? AND state NOT IN (?, ?, ?)`, - j.State, j.Step, j.Steps, j.Loss, j.Message, j.PeakMiB, started, finished, j.ID, + j.Node, j.State, j.Step, j.Steps, j.Loss, j.Message, j.PeakMiB, started, finished, j.ID, TrainDone, TrainFailed, TrainCancelled) return err } +// TrainJobsFor lists the jobs started in projects, or by a person outside +// any project, newest first. +func (s *DB) TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]TrainJob, error) { + q := `SELECT ` + trainCols + ` FROM train_jobs WHERE (project_id = '' AND publish = 1 AND created_by = ?)` + args := []any{createdBy} + if len(projects) > 0 { + q += ` OR project_id IN (?` + strings.Repeat(", ?", len(projects)-1) + `)` + for _, p := range projects { + args = append(args, p) + } + } + return queryAll(ctx, s.db, q+` ORDER BY created_at DESC LIMIT ?`, append(args, limit), + func(r *sql.Rows) (TrainJob, error) { return scanTrainJob(r) }) +} + +// SetTrainNote changes what a job says of itself, ended or not: what +// happened when what it made was published. +func (s *DB) SetTrainNote(ctx context.Context, id, message string) error { + _, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET message = ? WHERE id = ?`, message, id) + return err +} + +// SetTrainOwner changes who a job's adapter belongs to. +func (s *DB) SetTrainOwner(ctx context.Context, id, owner string) error { + res, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET owner = ? WHERE id = ?`, owner, id) + if err != nil { + return err + } + if n, _ := res.RowsAffected(); n == 0 { + return ErrNotFound + } + return nil +} + // PutTrainArtifact keeps what a job made. func (s *DB) PutTrainArtifact(ctx context.Context, job, kind string, data []byte) error { _, err := s.db.ExecContext(ctx, `INSERT INTO train_artifacts (job_id, kind, data) VALUES (?, ?, ?) diff --git a/internal/web/account.go b/internal/web/account.go index 67569ae..ac24474 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -109,6 +109,7 @@ type AccountPages struct { licencePrefs LicencePrefs images func() bool meshes func() bool + train func() bool guardLibrary func() guard.Config // guardFor names the guardrail policies in force for someone, and // whether they keep nothing; nil when there are none. diff --git a/internal/web/locales/en/nav.json b/internal/web/locales/en/nav.json index 9f49797..3dfe73d 100644 --- a/internal/web/locales/en/nav.json +++ b/internal/web/locales/en/nav.json @@ -14,5 +14,6 @@ "nav.mcp": "MCP servers", "nav.settings": "Settings", "nav.images": "Pictures", - "nav.meshes": "3D" + "nav.meshes": "3D", + "nav.train": "Customise a model" } diff --git a/internal/web/locales/en/train.json b/internal/web/locales/en/train.json new file mode 100644 index 0000000..f4ed612 --- /dev/null +++ b/internal/web/locales/en/train.json @@ -0,0 +1,55 @@ +{ + "train.title": "Customise a model", + "train.intro": "Teach a model how to answer from your own examples: an Excel file with a column of questions and a column of answers. When training is done the new model appears under its own name in the chat and the API; it runs on the card its base model runs on, with no card of its own.", + "train.not_knowledge": "A model learns tone and the shape of answers from examples; it does not reliably learn the facts in your documents. For answers from your documents, use a knowledge base: the chat answers from them and shows where.", + "train.no_project": "Customising a model needs a project: training is done for a project, which pays for it. Ask an administrator to add you to one.", + "train.no_models": "There is no model that can be customised just now.", + "train.step.file": "1. Examples", + "train.file_hint": "Excel (.xlsx), CSV or JSONL. Columns headed Question and Answer are read; with no header the first column is the question and the second the answer. At least 50 examples, better a few hundred.", + "train.question": "Question", + "train.answer": "Answer", + "train.step.model": "2. Model", + "train.base": "Model to start from", + "train.name": "The new model's name", + "train.name_hint": "what it is chosen by in the chat: letters, digits, - and _", + "train.step.who": "3. Who may use it", + "train.project": "Project", + "train.share": "Who may use it", + "train.share.me": "Only me", + "train.share.project": "This project", + "train.share.team": "My team", + "train.share.org": "My organization", + "train.share.all": "Everyone", + "train.share.other": "Somewhere else", + "train.confirm": "This file holds nothing this group must not see.", + "train.go": "Start training", + "train.jobs": "Trainings", + "train.col.name": "Model", + "train.col.state": "State", + "train.col.share": "Who may use it", + "train.no_jobs": "No trainings yet.", + "train.after": "One training runs at a time in a project. It runs on a GPU with enough memory free; small models take minutes, large ones longer. You can leave the page: training goes on.", + "train.js.found": "{n} questions and answers found.", + "train.js.skipped": "{n} rows were skipped, with no question or no answer.", + "train.js.few": "That is few: at least 50 examples are advised.", + "train.js.no_file": "Choose the file of questions and answers first.", + "train.js.no_name": "Give the new model a name.", + "train.js.need_confirm": "To share with a wider group, confirm that the file holds nothing it must not see.", + "train.js.started": "Training is queued; it is listed below.", + "train.js.failed": "Could not start: {why}", + "train.js.queued": "Queued", + "train.js.waiting": "Queued: waiting for a GPU with enough memory free", + "train.js.preparing": "Preparing (fetching the base model)", + "train.js.running": "Training: {p}% ({m} min)", + "train.js.publishing": "Trained, publishing…", + "train.js.ready": "Ready: choose {name} in the chat", + "train.js.restarting": "Ready; {name} appears in the chat within a few minutes", + "train.js.not_published": "Trained, but not published: {why}", + "train.js.job_failed": "Failed: {why}", + "train.js.cancelled": "Stopped", + "train.js.delete": "Delete", + "train.js.delete_ask": "Delete {name}? It goes from the chat and the API.", + "train.js.share_ask": "It will be shared with a wider group. Do you confirm the training data holds nothing that group must not see?", + "train.js.try": "Try it in the chat", + "train.js.by": "started by {who}" +} diff --git a/internal/web/locales/tr/nav.json b/internal/web/locales/tr/nav.json index 8f51f9c..b5e099f 100644 --- a/internal/web/locales/tr/nav.json +++ b/internal/web/locales/tr/nav.json @@ -14,5 +14,6 @@ "nav.mcp": "MCP sunucuları", "nav.settings": "Ayarlar", "nav.images": "Görseller", - "nav.meshes": "3D" + "nav.meshes": "3D", + "nav.train": "Modeli özelleştir" } diff --git a/internal/web/locales/tr/train.json b/internal/web/locales/tr/train.json new file mode 100644 index 0000000..dd12a17 --- /dev/null +++ b/internal/web/locales/tr/train.json @@ -0,0 +1,55 @@ +{ + "train.title": "Modeli özelleştir", + "train.intro": "Bir modele kendi örneklerinizle nasıl cevap vereceğini öğretin: bir Excel dosyasında bir sütun soru, bir sütun cevap. Eğitim bitince yeni model sohbette ve API'de kendi adıyla çıkar; temel modelin çalıştığı kartta çalışır, ayrı bir kart gerekmez.", + "train.not_knowledge": "Model örneklerden üslubu ve cevap biçimini öğrenir; belgelerinizdeki bilgileri güvenilir biçimde ezberlemez. Modelin belgelerinize göre cevap vermesini istiyorsanız bilgi tabanını kullanın: sohbette belgelerden cevaplar, kaynağını da gösterir.", + "train.no_project": "Model özelleştirmek için bir projeye üye olmalısınız: eğitim bir proje adına yapılır ve maliyeti projeye yazılır. Yöneticinizden sizi bir projeye eklemesini isteyin.", + "train.no_models": "Şu anda özelleştirilebilecek bir model yok.", + "train.step.file": "1. Örnekler", + "train.file_hint": "Excel (.xlsx), CSV ya da JSONL. Başlıkları Soru ve Cevap olan sütunlar okunur; başlık yoksa ilk sütun soru, ikincisi cevaptır. En az 50, tercihen birkaç yüz örnek.", + "train.question": "Soru", + "train.answer": "Cevap", + "train.step.model": "2. Model", + "train.base": "Temel alınacak model", + "train.name": "Yeni modelin adı", + "train.name_hint": "sohbette bu adla seçilir: harf, rakam, - ve _", + "train.step.who": "3. Kimler kullanabilir", + "train.project": "Proje", + "train.share": "Kullanabilecekler", + "train.share.me": "Yalnız ben", + "train.share.project": "Bu proje", + "train.share.team": "Takımım", + "train.share.org": "Kuruluşum", + "train.share.all": "Herkes", + "train.share.other": "Başka bir yer", + "train.confirm": "Bu dosyada bu gruptan saklanması gereken bilgi yok.", + "train.go": "Eğitimi başlat", + "train.jobs": "Eğitimler", + "train.col.name": "Model", + "train.col.state": "Durum", + "train.col.share": "Kullanabilecekler", + "train.no_jobs": "Henüz bir eğitim yok.", + "train.after": "Bir projede aynı anda bir eğitim çalışır. Eğitim, boş belleği yeterli bir GPU'da yapılır; küçük modeller dakikalar, büyükler daha uzun sürer. Sayfadan ayrılabilirsiniz, eğitim sürer.", + "train.js.found": "{n} soru-cevap bulundu.", + "train.js.skipped": "{n} satırda soru ya da cevap boş olduğu için atlandı.", + "train.js.few": "Bu az: en az 50 örnek önerilir.", + "train.js.no_file": "Önce soru-cevap dosyasını seçin.", + "train.js.no_name": "Yeni modele bir ad verin.", + "train.js.need_confirm": "Daha geniş bir grupla paylaşmak için dosyada saklanması gereken bilgi olmadığını onaylayın.", + "train.js.started": "Eğitim sıraya alındı; durumu aşağıda.", + "train.js.failed": "Başlatılamadı: {why}", + "train.js.queued": "Sırada", + "train.js.waiting": "Sırada: boş belleği yeterli bir GPU bekleniyor", + "train.js.preparing": "Hazırlanıyor (temel model indiriliyor)", + "train.js.running": "Eğitiliyor: %{p} ({m} dk)", + "train.js.publishing": "Eğitildi, yayınlanıyor…", + "train.js.ready": "Hazır: sohbette {name} adıyla seçin", + "train.js.restarting": "Hazır, birkaç dakika içinde sohbette {name} adıyla çıkacak", + "train.js.not_published": "Eğitildi ama yayınlanamadı: {why}", + "train.js.job_failed": "Başarısız: {why}", + "train.js.cancelled": "Durduruldu", + "train.js.delete": "Sil", + "train.js.delete_ask": "{name} silinsin mi? Model sohbetten ve API'den kalkar.", + "train.js.share_ask": "Daha geniş bir grupla paylaşılacak. Eğitim verisinde bu gruptan saklanması gereken bilgi olmadığını onaylıyor musunuz?", + "train.js.try": "Sohbette dene", + "train.js.by": "{who} başlattı" +} diff --git a/internal/web/nav.go b/internal/web/nav.go index a0f9227..7017bb7 100644 --- a/internal/web/nav.go +++ b/internal/web/nav.go @@ -40,6 +40,8 @@ type Nav struct { // Images offers the pictures page, while some model makes them; Meshes // the 3D page. Images, Meshes bool + // Train offers the page that customises a model, while some model can be. + Train bool } type navKey struct{} @@ -58,6 +60,7 @@ func (p *AccountPages) navFor(r *http.Request, user store.User, current string) CSRF: csrfFor(r), Images: p.images != nil && p.images(), Meshes: p.meshes != nil && p.meshes(), + Train: p.train != nil && p.train(), } // Only when there is an organization to show: a link that leads to "there // are none" on every page is noise. diff --git a/internal/web/templates/nav.html b/internal/web/templates/nav.html index ced3c2b..ceff948 100644 --- a/internal/web/templates/nav.html +++ b/internal/web/templates/nav.html @@ -37,6 +37,9 @@ {{- if .Meshes}} {{t "nav.meshes"}} {{- end}} + {{- if .Train}} + {{t "nav.train"}} + {{- end}} {{t "nav.account"}} {{t "nav.docs"}}
diff --git a/internal/web/templates/train.html b/internal/web/templates/train.html new file mode 100644 index 0000000..cdf3d59 --- /dev/null +++ b/internal/web/templates/train.html @@ -0,0 +1,287 @@ +{{define "train"}} + + +{{t "train.title"}} · llmgw + + + +
+
+

{{t "train.title"}}

+

{{.User.Email}}

+ {{template "nav" .Nav}} +
+ +

{{t "train.intro"}}

+

{{th "train.not_knowledge"}}

+ +{{if not .Projects}} +

{{t "train.no_project"}}

+{{else if not .Models}} +

{{t "train.no_models"}}

+{{else}} + +
+

{{t "train.step.file"}}

+

{{t "train.file_hint"}}

+ +

+ + + +
+
+

{{t "train.step.model"}}

+ + +
+
+

{{t "train.step.who"}}

+ {{if gt (len .Projects) 1}} + + {{else}}{{with index .Projects 0}} +

{{t "train.project"}}: {{.Label}}

{{end}}{{end}} + + +
+
+

+ +{{end}} + +

{{t "train.jobs"}}

+ + + +
{{t "train.col.name"}}{{t "train.col.state"}}{{t "train.col.share"}}
+ +

{{t "train.after"}}

+ +
llmgw {{.Version}}{{stamp .Now}}{{template "langs"}}
+
+ + +{{end}} diff --git a/internal/web/train.go b/internal/web/train.go new file mode 100644 index 0000000..1b33451 --- /dev/null +++ b/internal/web/train.go @@ -0,0 +1,34 @@ +package web + +import ( + "net/http" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +type trainData struct { + Nav Nav + User store.User + Models []string + Projects []ChatContext + CSRF string + Version string + Now string +} + +// Train renders the page that customises a model: a file of questions and +// answers, a model, a button, and the trainings of the person's projects. +func Train(user store.User, models []string, projects []ChatContext, version string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + noStore(w) + render(w, r, "train", trainData{ + Nav: navFrom(r, "train"), User: user, Models: models, Projects: projects, + CSRF: csrfFor(r), Version: version, Now: time.Now().UTC().Format(time.RFC3339), + }) + } +} + +// SetTrain has the menu offer the page while a model can be customised. +// Asked on every page, so it must be cheap. +func (p *AccountPages) SetTrain(available func() bool) { p.train = available } diff --git a/internal/web/train_test.go b/internal/web/train_test.go new file mode 100644 index 0000000..a7e1688 --- /dev/null +++ b/internal/web/train_test.go @@ -0,0 +1,49 @@ +package web + +import ( + "net/http/httptest" + "regexp" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +var untranslated = regexp.MustCompile(`train\.[a-z]`) + +func TestTheTrainingPageDrawsInEveryLanguage(t *testing.T) { + for _, lang := range []string{"en", "tr"} { + req := httptest.NewRequest("GET", "/train", nil) + req.Header.Set("Accept-Language", lang) + rec := httptest.NewRecorder() + Train(store.User{Email: "a@b.c"}, []string{"qwen2.5-7b"}, []ChatContext{{Value: "p1", Label: "Acme / hr / bot"}}, "v0")(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: %d %s", lang, rec.Code, rec.Body.String()) + } + page := rec.Body.String() + for _, want := range []string{``, `id="file"`, `/train/start`, `value="p1"`, `Acme / hr / bot`} { + if !strings.Contains(page, want) { + t.Errorf("%s: %s missing", lang, want) + } + } + if untranslated.MatchString(page) { + t.Errorf("%s: an untranslated key is on the page", lang) + } + } + for name, page := range map[string]func() string{ + "no project": func() string { + rec := httptest.NewRecorder() + Train(store.User{}, []string{"q"}, nil, "v0")(rec, httptest.NewRequest("GET", "/train", nil)) + return rec.Body.String() + }, + "no model": func() string { + rec := httptest.NewRecorder() + Train(store.User{}, nil, []ChatContext{{Value: "p"}}, "v0")(rec, httptest.NewRequest("GET", "/train", nil)) + return rec.Body.String() + }, + } { + if p := page(); !strings.Contains(p, `class="empty"`) || strings.Contains(p, `id="start"`) { + t.Errorf("%s: the page offers training", name) + } + } +}