diff --git a/BACKLOG.md b/BACKLOG.md index 4122a16..8e5ff50 100644 --- a/BACKLOG.md +++ b/BACKLOG.md @@ -2107,6 +2107,15 @@ lent, partition`); the leader tells each named node's agent (root) what to be, t controller ~20 s later, sbatch ran on n1 as ada. Disabled in AD: suspended in 56 s, the Linux user gone in 89 s, ssh refused, home kept. Enabled: back in 32 s, same UID, key kept. Taken out of ml-team: out of the team in 11 s; put back: in again in 56 s. +- P5, replaced 2026-11-17 by the portal (decided 2026-11-15: users of every level, build for the one who + knows least first; any project member may start, no approval, one per project, the project's by default): + **Customise a model** (`/train`): an xlsx/CSV/JSONL of questions and answers, checked and previewed; base + weights from the model's origin (a GGUF repo's card base_model), memory from their size (2x + 1 GiB), + the node with the most free; published by itself when done (llama.cpp replicas replaced to load it); + shared me/project/team/org/everyone (wider than the project only confirmed, never wider than its + knowledge bases), deleted from the page. Left: pairs generated from a knowledge base on the page, a + base-vs-new comparison, the training's GPU time charged to the project, big jobs through Slurm, the + server's own error messages in Turkish. - P6, done 2026-11-15 but for budgets: each project a Slurm account (org and team accounts above it, ids keyed by kind: the first install's org, team and project share the id "default"), its members and its team's and org's on it; AccountingStorageEnforce=associations; the controller's agent keeps them with diff --git a/CHANGELOG.md b/CHANGELOG.md index d1fb8ed..0723f1a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -30,6 +30,22 @@ APIs change between minors without a deprecation period. ### Added +- **Customise a model from the portal.** A new page, **Customise a model** + (`/train`), lets anyone in a project teach a model how to answer without a + terminal: an Excel, CSV or JSONL file of questions and answers (columns + headed Soru/Cevap or Question/Answer, else the first two), checked and + shown before anything trains; a model; a name; who may use it; and a + button. The weights to train on are found from where the model's came (a + GGUF repository's base model on Hugging Face), the card's memory from their + size, the node as the one with the most free; when it is done the adapter + is published by itself and appears as a model of its own in the chat and + the API, a llama.cpp model's replicas replaced one at a time to load it. + It is the project's by default; its members may share it with the person, + the team, the organization or everyone (wider than the project only once + they confirm the file holds nothing that group must not see, and never + wider than the knowledge bases its pairs came from), and delete it. One + trains at a time per project. `llmgw ctl train start` needs only `--model` + and `--data` now, and `--publish` publishes when done. - **An organization's admin.** Besides member and manager, a membership of an organization may now be `admin` (`llmgw users join … --org Acme --role admin`, the role chosen on a membership on the people page, the console, diff --git a/docs/knowledge.md b/docs/knowledge.md index 500a81e..edb8f9b 100644 --- a/docs/knowledge.md +++ b/docs/knowledge.md @@ -385,18 +385,25 @@ right and mixed facts up on the rest, while the same questions through the knowledge base got 9 of 9. Use both: the knowledge base for what answers say, an adapter for how they sound. +Anyone in a project can do it from the portal: **Customise a model** +(`/train`) takes an Excel, CSV or JSONL file of questions and answers, a +model, a name and who may use it, and publishes the adapter by itself when it +is trained. The weights, the node and the card's memory are worked out. From +the command line: + ```sh # 1. Questions and answers from the collection, written by a model of the # cluster. Read the file through and correct it. llmgw ctl kb dataset Acme/policies --model qwen2.5-coder-14b-instruct --out policies.jsonl -# 2. Train on a node. --gpu-memory is what the job needs of a card; it -# runs beside an engine when the engine leaves that much free. -llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct \ - --base-repo Qwen/Qwen2.5-0.5B-Instruct --node cuda01 --gpu-memory 2300 --data policies.jsonl +# 2. Train. The weights to train on are found from where the model's came +# (--base-repo names others), the card's memory from their size +# (--gpu-memory), the node as the one whose card has the most free +# (--node); it runs beside an engine when the engine leaves that much. +llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct --data policies.jsonl llmgw ctl train show JOB --follow -# 3. Make it the model's adapter, kept on the mirror. +# 3. Make it the model's adapter, kept on the mirror (or --publish in step 2). llmgw ctl train publish JOB --restart ``` diff --git a/internal/cli/server.go b/internal/cli/server.go index faa5059..bcaa68c 100644 --- a/internal/cli/server.go +++ b/internal/cli/server.go @@ -465,7 +465,12 @@ func newServerCmd() *cobra.Command { trainer := train.New(st, agentSvc, log) agentSvc.SetTrain(trainer) go trainer.Run(ctx) - api.SetTrain(&adminapi.TrainAdmin{Store: st, Service: trainer}) + api.SetTrain(&adminapi.TrainAdmin{Store: st, Service: trainer, Log: log}) + // From the portal: a node with a card free enough, and + // what was made published when it is done. + trainer.SetPicker(api.PickTrainNode) + trainer.SetOnDone(api.TrainDone) + rt.SetTraining(api) // Slurm on the nodes the settings name, its munge key // made from a cluster secret. slurmSvc = slurmsvc.New(func() config.SlurmSettings { return current.Load().Slurm }, @@ -661,7 +666,7 @@ func newServerCmd() *cobra.Command { // takes longer than the mirror's usual answers. if adminAPI != nil { if up, err := mirror.NewFailoverClient(mirrors, cfg.Mirror.TLS, 10*time.Minute); err == nil { - adminAPI.SetTrainMirror(up) + adminAPI.SetTrainMirror(up, hf.New(cfg.Mirror.Registry, cfg.Mirror.Timeout)) } } // Browsing the registry is what turns "paste a resolve URL" diff --git a/internal/cli/train.go b/internal/cli/train.go index a51ebef..3de82f7 100644 --- a/internal/cli/train.go +++ b/internal/cli/train.go @@ -21,9 +21,13 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { Long: "A node trains an adapter for a catalog model on pairs (llmgw ctl kb dataset\n" + "writes them from a knowledge base; read them through first). It trains with\n" + "its installed vLLM's torch and a small overlay, as the engines' user, on a card\n" + - "with --gpu-memory free — beside an engine, if it leaves that much. What it\n" + - "makes is kept by the control plane: llmgw ctl train download, or publish it\n" + - "as the model's adapter.\n\n" + + "with --gpu-memory free — beside an engine, if it leaves that much. Without\n" + + "--base-repo the weights are found from where the model's came, without\n" + + "--gpu-memory from their size, and without --node it goes to the node whose\n" + + "card has the most free. What it makes is kept by the control plane: llmgw ctl\n" + + "train download, or publish it as the model's adapter (--publish does that by\n" + + "itself when it is done). People in a project can do all this from the portal's\n" + + "\"Customise a model\" page.\n\n" + "An adapter learns how answers sound — words, format, tone — far better than\n" + "what they say. Facts belong to the knowledge base.", } @@ -33,11 +37,12 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { var gpuMem int var epochs, rank int var lr float64 - var noGGUF bool + var noGGUF, publishWhenDone bool start := &cobra.Command{ - Use: "start NAME --model MODEL --base-repo OWNER/NAME --node NODE --data FILE", + Use: "start NAME --model MODEL --data FILE", Short: "Queue an adapter's training on a node", - Example: " llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct \\\n" + + Example: " llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct --data policies.jsonl --public --publish\n" + + " llmgw ctl train start policies-tone --model qwen2.5-0.5b-instruct \\\n" + " --base-repo Qwen/Qwen2.5-0.5B-Instruct --node cuda01 --gpu-memory 2300 --data policies.jsonl", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { @@ -46,7 +51,7 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { return err } req := adminapi.TrainRequest{Name: args[0], Model: model, BaseRepo: base, BaseRevision: rev, Node: node, - GPUMemoryMiB: gpuMem, Data: string(b), Params: map[string]any{}, Owner: owner, Public: public} + GPUMemoryMiB: gpuMem, Data: string(b), Params: map[string]any{}, Owner: owner, Public: public, Publish: publishWhenDone} if cmd.Flags().Changed("epochs") { req.Params["epochs"] = epochs } @@ -69,8 +74,8 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { if v.Owner != "" { who = v.Owner + " only" } - fmt.Fprintf(cmd.OutOrStdout(), "queued %s: %s for %s on %s, %d pairs; it will be seen by %s\n", - v.ID, v.Name, v.Model, v.Node, v.Pairs, who) + fmt.Fprintf(cmd.OutOrStdout(), "queued %s: %s for %s on %s, %d pairs from %s, %d MiB of a card; it will be seen by %s\n", + v.ID, v.Name, v.Model, or(v.Node, "the node with a card free enough"), v.Pairs, v.BaseRepo, v.GPUMemoryMiB, who) fmt.Fprintf(cmd.OutOrStdout(), "follow it with: llmgw ctl train show %s --follow\n", v.ID) return nil }, @@ -79,8 +84,9 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { f.StringVar(&model, "model", "", "the catalog model the adapter is for") f.StringVar(&base, "base-repo", "", "the weights to train on, as safetensors on Hugging Face (owner/name)") f.StringVar(&rev, "base-revision", "", "a revision of them (default: the newest)") - f.StringVar(&node, "node", "", "the node that trains") - f.IntVar(&gpuMem, "gpu-memory", 4096, "MiB of a card the job needs free") + f.StringVar(&node, "node", "", "the node that trains (default: the one whose card has the most free)") + f.IntVar(&gpuMem, "gpu-memory", 0, "MiB of a card the job needs free (default: from the weights' size)") + f.BoolVar(&publishWhenDone, "publish", false, "publish it as the model's adapter when it is done, restarting a llama.cpp model's replicas") f.StringVar(&data, "data", "", "the pairs, NDJSON of {\"q\", \"a\"}") f.IntVar(&epochs, "epochs", 6, "passes over the pairs") f.IntVar(&rank, "rank", 16, "the adapter's rank") @@ -89,7 +95,7 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { f.StringVar(&owner, "owner", "", "who the adapter belongs to — org:, team:, project: (path or ID), user: or key: — and only they see it "+ "(default: the knowledge base's owner, when the pairs come from one)") f.BoolVar(&public, "public", false, "every caller of the model sees it; refused for pairs from a knowledge base") - for _, req := range []string{"model", "base-repo", "node", "data"} { + for _, req := range []string{"model", "data"} { _ = start.MarkFlagRequired(req) } c.AddCommand(start) @@ -138,8 +144,11 @@ func newTrainCmd(client func() *ctlClient, asJSON *bool) *cobra.Command { } line := fmt.Sprintf("%s %s %s loss %s peak %s %s", j.State, trainProgress(j), j.Node, trainLoss(j), trainPeak(j), j.Message) if !follow { - fmt.Fprintf(out, "%s: adapter %s for %s, owned by %s, on %s (base %s), %d pairs\n", j.ID, j.Name, j.Model, - or(j.Owner, "everyone"), j.Node, j.BaseRepo, j.Pairs) + fmt.Fprintf(out, "%s: adapter %s for %s, owned by %s, on %s (base %s), %d pairs, %d MiB of a card\n", j.ID, j.Name, j.Model, + or(j.Owner, "everyone"), or(j.Node, "the node whose card has the most free"), j.BaseRepo, j.Pairs, j.GPUMemoryMiB) + if j.Project != "" { + fmt.Fprintf(out, "started from the portal in project %s by %s; published when done: %t\n", j.Project, j.CreatedBy, j.Publish) + } fmt.Fprintln(out, line) if len(j.Artifacts) > 0 { fmt.Fprintf(out, "made: %s (llmgw ctl train download %s --kind KIND)\n", strings.Join(j.Artifacts, ", "), j.ID) diff --git a/internal/control/adminapi/train.go b/internal/control/adminapi/train.go index b777799..e2545f5 100644 --- a/internal/control/adminapi/train.go +++ b/internal/control/adminapi/train.go @@ -10,15 +10,20 @@ import ( "errors" "fmt" "io" + "log/slog" "net/http" "regexp" "slices" "strconv" "strings" + "sync" "time" "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler" "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" "gitea.mynodes.xyz/ilker/llmgw/internal/mirror" "gitea.mynodes.xyz/ilker/llmgw/internal/store" ) @@ -36,6 +41,11 @@ type TrainStore interface { TrainArtifact(ctx context.Context, job, kind string) ([]byte, error) TrainArtifactKinds(ctx context.Context, job string) ([]string, error) DeleteTrainJob(ctx context.Context, id string) error + QueuedTrainJobs(ctx context.Context) ([]store.TrainJob, error) + ActiveTrainJobs(ctx context.Context) ([]store.TrainJob, error) + TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]store.TrainJob, error) + SetTrainOwner(ctx context.Context, id, owner string) error + SetTrainNote(ctx context.Context, id, message string) error } // TrainAdmin is what the training endpoints need. @@ -45,6 +55,27 @@ type TrainAdmin struct { // Mirror keeps what a job made where the nodes fetch weights; nil // leaves publishing off. Mirror TrainMirror + // Registry finds the weights to train a catalog model on, from where + // its own came; nil has every job name them. + Registry TrainRegistry + // Log says what publishing by itself could not do. + Log *slog.Logger + + mu sync.Mutex + bases map[string]trainBaseInfo // by the model's origin repository +} + +// TrainRegistry reads a repository of the model registry. +type TrainRegistry interface { + RepoAt(ctx context.Context, id, rev string) (hf.Repo, error) +} + +// trainBaseInfo is the safetensors weights a model is trained on. +type trainBaseInfo struct { + repo, revision string + bytes int64 + err error + at time.Time } // TrainMirror is the model mirror, as publishing needs it. @@ -66,11 +97,11 @@ type TrainPublishView struct { // SetTrain enables the training endpoints. func (a *API) SetTrain(t *TrainAdmin) { a.train = t } -// SetTrainMirror gives publishing the mirror, which is set up after the -// training endpoints are. -func (a *API) SetTrainMirror(m TrainMirror) { +// SetTrainMirror gives publishing the mirror, and finding a model's +// weights the registry, which are set up after the training endpoints are. +func (a *API) SetTrainMirror(m TrainMirror, reg TrainRegistry) { if a.train != nil { - a.train.Mirror = m + a.train.Mirror, a.train.Registry = m, reg } } @@ -81,13 +112,23 @@ type TrainRequest struct { // Model is the catalog model the adapter is for. Model string `json:"model"` // BaseRepo and BaseRevision are the weights trained on: the model's own - // on its registry, in safetensors (Hugging Face), not a GGUF. - BaseRepo string `json:"base_repo"` + // on its registry, in safetensors (Hugging Face), not a GGUF. Empty + // finds them from where the model's came: the repository itself, or the + // one its card says it was made from. + BaseRepo string `json:"base_repo,omitempty"` BaseRevision string `json:"base_revision,omitempty"` - // Node trains it; GPUMemoryMiB is what it needs of a card, which can be - // what an engine there leaves free. - Node string `json:"node"` - GPUMemoryMiB int `json:"gpu_memory_mib"` + // Node trains it; empty is the first with a card that has enough free. + // GPUMemoryMiB is what it needs of a card, which can be what an engine + // there leaves free; 0 is worked out from the weights' size. + Node string `json:"node,omitempty"` + GPUMemoryMiB int `json:"gpu_memory_mib,omitempty"` + // Project is the project the job is started in: its members see and + // manage it, and only one of its jobs trains at a time. + Project string `json:"project,omitempty"` + // Publish puts what the job makes on the mirror and makes it the + // model's adapter by itself when the job is done, restarting a llama.cpp + // model's replicas to load it. + Publish bool `json:"publish,omitempty"` // Owner is who the adapter will belong to: org:, team:, project: (by // path or ID), user: or key:. Only callers there see it. When the // pairs come from one knowledge base, it defaults to that collection's @@ -129,6 +170,8 @@ type TrainJobView struct { StartedAt time.Time `json:"started_at,omitzero"` FinishedAt time.Time `json:"finished_at,omitzero"` CreatedBy string `json:"created_by,omitempty"` + Project string `json:"project,omitempty"` + Publish bool `json:"publish,omitempty"` // Artifacts are what it made: peft (a gzipped tar), gguf. Artifacts []string `json:"artifacts,omitempty"` } @@ -137,7 +180,8 @@ func trainView(j store.TrainJob) TrainJobView { return TrainJobView{ID: j.ID, Name: j.Name, Model: j.Model, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision, Node: j.Node, Owner: j.Owner, GPUMemoryMiB: j.GPUMemoryMiB, Params: j.Params, GGUF: j.GGUF, Pairs: j.Pairs, State: j.State, Step: j.Step, Steps: j.Steps, Loss: j.Loss, Message: j.Message, PeakMiB: j.PeakMiB, CreatedAt: j.CreatedAt, - StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy} + StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy, + Project: j.Project, Publish: j.Publish} } func (a *API) registerTrain(mux Mux) { @@ -178,50 +222,122 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, err.Error()) return } - bad := func(format string, args ...any) { - writeError(w, http.StatusBadRequest, fmt.Sprintf(format, args...)) + j, err := a.StartTraining(r.Context(), in, actorOf(r)) + if err != nil { + writeTrainErr(w, err) + return + } + writeJSON(w, http.StatusCreated, trainView(j)) +} + +// trainErr is a refusal, with the status it is answered with. +type trainErr struct { + code int + msg string +} + +func (e *trainErr) Error() string { return e.msg } + +func refused(code int, format string, args ...any) error { + return &trainErr{code: code, msg: fmt.Sprintf(format, args...)} +} + +func writeTrainErr(w http.ResponseWriter, err error) { + var te *trainErr + if errors.As(err, &te) { + writeError(w, te.code, te.msg) + return + } + writeError(w, http.StatusInternalServerError, err.Error()) +} + +// TrainRefusal is the reason a training request was refused, for someone +// to read, and whether it was the request's fault rather than the +// cluster's. +func TrainRefusal(err error) (string, bool) { + if err == nil { + return "", false + } + var te *trainErr + if errors.As(err, &te) { + return te.msg, te.code < 500 + } + return err.Error(), false +} + +// StartTraining checks a job and queues it, as actor. +func (a *API) StartTraining(ctx context.Context, in TrainRequest, actor string) (store.TrainJob, error) { + if a.train == nil { + return store.TrainJob{}, refused(http.StatusNotImplemented, + "training needs the store and the nodes' agents, which this control plane runs without") + } + bad := func(format string, args ...any) (store.TrainJob, error) { + return store.TrainJob{}, refused(http.StatusBadRequest, format, args...) } if !adapterName.MatchString(in.Name) { - bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name) - return + return bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name) } m, ok := a.ctl.Catalog().Resolve(in.Model) if !ok { - bad("no model %s in the catalog", in.Model) - return + return bad("no model %s in the catalog", in.Model) } if _, _, _, taken := a.ctl.Catalog().ResolveAdapter(in.Name); taken || in.Name == m.Name { - bad("%s is already a model's or an adapter's name", in.Name) - return + return bad("%s is already a model's or an adapter's name", in.Name) } - if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") { - bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo) - return + if _, taken := a.ctl.Catalog().Resolve(in.Name); taken { + return bad("%s is already a model's or an adapter's name", in.Name) } - if _, ok := a.ctl.Inventory().Get(in.Node); !ok { - bad("no node %s", in.Node) - return + queued, err := a.train.Store.QueuedTrainJobs(ctx) + if err != nil { + return store.TrainJob{}, err + } + active, err := a.train.Store.ActiveTrainJobs(ctx) + if err != nil { + return store.TrainJob{}, err + } + for _, j := range append(queued, active...) { + if j.Name == in.Name { + return bad("%s is already being trained (job %s)", in.Name, j.ID) + } + if in.Project != "" && j.Project == in.Project { + return store.TrainJob{}, refused(http.StatusConflict, + "this project already has a model training (%s): one at a time, so that one project does not hold every card", j.Name) + } + } + var weights int64 + if in.BaseRepo == "" { + b := a.trainBase(ctx, m) + if b.err != nil { + return store.TrainJob{}, b.err + } + in.BaseRepo, in.BaseRevision, weights = b.repo, b.revision, b.bytes + } else if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") { + return bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo) + } + if in.Node != "" { + if _, ok := a.ctl.Inventory().Get(in.Node); !ok { + return bad("no node %s", in.Node) + } } if in.GPUMemoryMiB <= 0 { - bad("gpu_memory_mib is what the job needs of a card, in MiB") - return + if weights <= 0 { + return bad("gpu_memory_mib is what the job needs of a card, in MiB") + } + in.GPUMemoryMiB = TrainMemoryMiB(weights) } pairs, colls, err := countPairs(in.Data) if err != nil { - bad("%v", err) - return + return bad("%v", err) } - owner, err := a.trainOwner(r.Context(), in, colls) + owner, err := a.trainOwner(ctx, in, colls) if err != nil { - bad("%v", err) - return + return bad("%v", err) } for k, v := range in.Params { lim, known := trainParams[k] f, isNum := v.(float64) if !known || !isNum || f < lim[0] || f > lim[1] { - bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k) - return + return bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k) } } params := "" @@ -240,15 +356,132 @@ func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) { } j := store.TrainJob{ID: store.NewID("train-"), Name: in.Name, Model: m.Name, BaseRepo: in.BaseRepo, BaseRevision: in.BaseRevision, Node: in.Node, Owner: owner, GPUMemoryMiB: in.GPUMemoryMiB, Params: params, GGUF: gguf, - Data: []byte(in.Data), Pairs: pairs, CreatedBy: actorOf(r)} - if err := a.train.Store.CreateTrainJob(r.Context(), j); err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return + Data: []byte(in.Data), Pairs: pairs, CreatedBy: actor, Project: in.Project, Publish: in.Publish} + if err := a.train.Store.CreateTrainJob(ctx, j); err != nil { + return store.TrainJob{}, err } // At once rather than at the next look, when the node is here. - go a.train.Service.Dispatch(context.WithoutCancel(r.Context())) + go a.train.Service.Dispatch(context.WithoutCancel(ctx)) j.State = store.TrainQueued - writeJSON(w, http.StatusCreated, trainView(j)) + return j, nil +} + +// TrainMemoryMiB is what training a LoRA adapter on weights of this size +// takes of a card: the weights in 16 bits, as much again for what passes +// through them, and the CUDA context. Measured at 2.3 GB for a 0.5B model. +func TrainMemoryMiB(weightBytes int64) int { + mib := int(weightBytes>>20)*2 + 1024 + return (mib + 255) / 256 * 256 +} + +// trainBaseFor is how long what the registry said of a model's weights is +// kept: it changes when the model is updated, which is rare. +const trainBaseFor = time.Hour + +// trainBase is the safetensors weights to train m on: its own origin's, +// when they are safetensors, or those its card says it was made from — a +// GGUF's base model. +func (a *API) trainBase(ctx context.Context, m catalog.Model) trainBaseInfo { + if m.Origin == nil || m.Origin.Repo == "" { + return trainBaseInfo{err: refused(http.StatusBadRequest, "%s's weights did not come from the model registry, so "+ + "what to train it on is not known: an administrator can name them (llmgw ctl train start --base-repo)", m.Name)} + } + if a.train.Registry == nil { + return trainBaseInfo{err: refused(http.StatusNotImplemented, "the model registry is not reachable from here "+ + "(mirror.registry): the weights to train %s on cannot be found", m.Name)} + } + a.train.mu.Lock() + b, ok := a.train.bases[m.Origin.Repo] + a.train.mu.Unlock() + if ok && time.Since(b.at) < trainBaseFor { + return b + } + b = a.findTrainBase(ctx, m) + if b.err != nil { + var te *trainErr + if !errors.As(b.err, &te) { + return b // the registry did not answer: asked again next time + } + } + b.at = time.Now() + a.train.mu.Lock() + if a.train.bases == nil { + a.train.bases = map[string]trainBaseInfo{} + } + a.train.bases[m.Origin.Repo] = b + a.train.mu.Unlock() + return b +} + +func (a *API) findTrainBase(ctx context.Context, m catalog.Model) trainBaseInfo { + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + repo, err := a.train.Registry.RepoAt(ctx, m.Origin.Repo, "") + if err != nil { + return trainBaseInfo{err: fmt.Errorf("asking the model registry about %s: %w", m.Origin.Repo, err)} + } + candidates := []hf.Repo{repo} + for _, id := range repo.BaseModel { + if id == repo.ID || strings.Count(id, "/") != 1 { + continue + } + r, err := a.train.Registry.RepoAt(ctx, id, "") + if err != nil { + continue + } + candidates = append(candidates, r) + } + gated := "" + for _, r := range candidates { + var n int64 + for _, f := range r.Files { + if strings.HasSuffix(strings.ToLower(f.Path), ".safetensors") { + n += f.Bytes + } + } + if n == 0 { + continue + } + if r.Gated.Is() { + gated = r.ID + continue + } + return trainBaseInfo{repo: r.ID, revision: r.Revision, bytes: n} + } + if gated != "" { + return trainBaseInfo{err: refused(http.StatusBadRequest, "the weights to train %s on, %s, are gated on the registry: "+ + "the nodes cannot fetch them without an account that accepted their terms", m.Name, gated)} + } + return trainBaseInfo{err: refused(http.StatusBadRequest, "neither %s, where %s's weights came from, nor the model its card "+ + "names has safetensors weights to train on", m.Origin.Repo, m.Name)} +} + +// PickTrainNode is the node whose freest card has the most memory free, +// if that is need, among those ready and not busy. +func (a *API) PickTrainNode(need int, busy map[string]bool) string { + best, bestFree := "", 0 + for _, n := range a.ctl.Inventory().Nodes() { + if busy[n.ID] || n.State != inventory.Ready { + continue + } + free := 0 + if n.Load != nil { + for _, g := range n.Load.GPUs { + if g.MemoryTotalMB > 0 { + free = max(free, int(g.MemoryTotalMB-g.MemoryUsedMB)) + } + } + } + if free == 0 { + for _, g := range n.GPUs { + free = max(free, g.MemoryTotalMB-g.MemoryUsedMB) + } + } + if free >= need && free > bestFree { + best, bestFree = n.ID, free + } + } + return best } // trainOwner is who a job's adapter will belong to, as scope:ID: one that @@ -453,73 +686,74 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) { if !ok { return } - if a.train.Mirror == nil { - writeError(w, http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+ - "llmgw ctl train download, then llmgw mirror import there, does it by hand") + v, err := a.publishTrained(r.Context(), j, actorOf(r)) + if err != nil { + writeTrainErr(w, err) return } + writeJSON(w, http.StatusOK, v) +} + +// publishTrained puts what a job made on the mirror and makes it the +// model's adapter: the GGUF for a model llama.cpp serves, the PEFT +// directory for one vLLM serves. +func (a *API) publishTrained(ctx context.Context, j store.TrainJob, actor string) (TrainPublishView, error) { + var none TrainPublishView + if a.train.Mirror == nil { + return none, refused(http.StatusNotImplemented, "publishing needs the model mirror (mirror.api in the settings); "+ + "llmgw ctl train download, then llmgw mirror import there, does it by hand") + } if a.models == nil { - writeError(w, http.StatusNotImplemented, "this control plane has no store for the catalog") - return + return none, refused(http.StatusNotImplemented, "this control plane has no store for the catalog") } if j.State != store.TrainDone { - writeError(w, http.StatusConflict, "the job has not finished: it is "+j.State) - return + return none, refused(http.StatusConflict, "%s", "the job has not finished: it is "+j.State) } m, found := a.ctl.Catalog().Resolve(j.Model) if !found { - writeError(w, http.StatusConflict, "the model "+j.Model+" is no longer in the catalog") - return + return none, refused(http.StatusConflict, "%s", "the model "+j.Model+" is no longer in the catalog") } for _, ad := range m.Adapters { if ad.Name == j.Name { - writeError(w, http.StatusConflict, j.Name+" is already "+m.Name+"'s adapter; train under another name") - return + return none, refused(http.StatusConflict, "%s", j.Name+" is already "+m.Name+"'s adapter; train under another name") } } llama := slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" }) source := "trained by llmgw job " + j.ID + " on " + j.Node + " from " + j.BaseRepo var url string if llama { - b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "gguf") + b, err := a.train.Store.TrainArtifact(ctx, j.ID, "gguf") if err != nil { - writeError(w, http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf") - return + return none, refused(http.StatusConflict, "the job made no GGUF, which llama.cpp needs; train again without --no-gguf") } - res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, j.Name+".gguf", b, source) + res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, j.Name+".gguf", b, source) if err != nil { - writeError(w, http.StatusBadGateway, "keeping it on the mirror: "+err.Error()) - return + return none, refused(http.StatusBadGateway, "%s", "keeping it on the mirror: "+err.Error()) } url = res.URL } else { - b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, "peft") + b, err := a.train.Store.TrainArtifact(ctx, j.ID, "peft") if err != nil { - writeError(w, http.StatusConflict, "the job made no PEFT adapter") - return + return none, refused(http.StatusConflict, "the job made no PEFT adapter") } files, err := untarAll(b) if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return + return none, err } for name, data := range files { - res, err := a.train.Mirror.Upload(r.Context(), j.Name, j.ID, name, data, source) + res, err := a.train.Mirror.Upload(ctx, j.Name, j.ID, name, data, source) if err != nil { - writeError(w, http.StatusBadGateway, "keeping "+name+" on the mirror: "+err.Error()) - return + return none, refused(http.StatusBadGateway, "%s", "keeping "+name+" on the mirror: "+err.Error()) } url = res.URL[:strings.LastIndex(res.URL, "/")+1] } } ad := catalog.Adapter{Name: j.Name, Source: url, Owner: j.Owner} - if err := checkAdapters(r.Context(), m, []catalog.Adapter{ad}); err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return + if err := checkAdapters(ctx, m, []catalog.Adapter{ad}); err != nil { + return none, refused(http.StatusBadRequest, "%v", err) } - if _, err := a.models.SetAdapters(r.Context(), m.Name, append(slices.Clone(m.Adapters), ad), actorOf(r)); err != nil { - writeError(w, http.StatusBadRequest, err.Error()) - return + if _, err := a.models.SetAdapters(ctx, m.Name, append(slices.Clone(m.Adapters), ad), actor); err != nil { + return none, refused(http.StatusBadRequest, "%v", err) } v := TrainPublishView{Adapter: j.Name, Model: m.Name, Source: url, Owner: j.Owner} if llama { @@ -528,7 +762,48 @@ func (a *API) trainPublish(w http.ResponseWriter, r *http.Request) { } else { v.Note = "ask for model " + j.Name + "; a replica loads it on the first request" } - writeJSON(w, http.StatusOK, v) + return v, nil +} + +func (a *API) trainLog() *slog.Logger { + if a.train.Log == nil { + return slog.New(slog.DiscardHandler) + } + return a.train.Log +} + +// TrainDone publishes what a job made, when it asked for that: on the +// mirror, as its model's adapter, and for a llama.cpp model with the +// model's replicas replaced, one at a time beside themselves, to load it. +func (a *API) TrainDone(j store.TrainJob) { + if !j.Publish || a.train == nil { + return + } + ctx, cancel := context.WithTimeout(context.Background(), 15*time.Minute) + defer cancel() + note := "published: ask for model " + j.Name + v, err := a.publishTrained(ctx, j, "training:"+j.ID) + if err != nil { + note = "trained, but publishing it failed: " + err.Error() + a.trainLog().Warn("a trained adapter could not be published", "job", j.ID, "adapter", j.Name, "err", err) + } else if strings.HasPrefix(v.Note, "llama.cpp") { + var ids []string + for _, rep := range a.ctl.Reconciler().ReadyReplicas() { + if rep.Model == v.Model { + ids = append(ids, rep.ID) + } + } + if len(ids) > 0 { + if err := a.ctl.Reconciler().RestartPaced(reconciler.RestartPace{}, false, ids...); err != nil { + note = "published, but " + v.Model + "'s replicas could not be restarted to load it: " + err.Error() + } else { + note = "published: " + v.Model + "'s replicas are restarting to load it; then ask for model " + j.Name + } + } + } + if err := a.train.Store.SetTrainNote(ctx, j.ID, note); err != nil { + a.trainLog().Warn("could not record a trained adapter's publishing", "job", j.ID, "err", err) + } } // untarAll reads a gzipped tar of a PEFT directory's files. diff --git a/internal/control/adminapi/train_portal.go b/internal/control/adminapi/train_portal.go new file mode 100644 index 0000000..1bf562c --- /dev/null +++ b/internal/control/adminapi/train_portal.go @@ -0,0 +1,114 @@ +package adminapi + +import ( + "context" + "errors" + "net/http" + "slices" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +// Training from the portal's "customise a model" page: the page decides who +// may do what (members of the job's project), and these do it. + +// TrainableModels are the catalog's models an adapter can be trained for +// from the portal: chat models llama.cpp or vLLM serve whose weights came +// from the registry, so the weights to train on can be found. +func (a *API) TrainableModels() []string { + var out []string + for _, m := range a.ctl.Catalog().Models() { + if m.IsGuard() || m.IsExternal() || m.Capabilities.Embeddings || m.MakesImages() || m.MakesMeshes() || + m.Origin == nil || m.Origin.Repo == "" { + continue + } + if slices.ContainsFunc(m.Profiles, func(p catalog.Profile) bool { return p.Engine == "llamacpp" || p.Engine == "vllm" }) { + out = append(out, m.Name) + } + } + slices.Sort(out) + return out +} + +// Trainings are the jobs started in projects, or by createdBy outside any. +func (a *API) Trainings(ctx context.Context, projects []string, createdBy string) ([]store.TrainJob, error) { + if a.train == nil { + return nil, nil + } + return a.train.Store.TrainJobsFor(ctx, projects, createdBy, 100) +} + +// Training is one job. +func (a *API) Training(ctx context.Context, id string) (store.TrainJob, error) { + if a.train == nil { + return store.TrainJob{}, store.ErrNotFound + } + return a.train.Store.TrainJob(ctx, id) +} + +// Published is whether a job's adapter is its model's in the catalog. +func (a *API) Published(j store.TrainJob) bool { + m, ok := a.ctl.Catalog().Resolve(j.Model) + return ok && slices.ContainsFunc(m.Adapters, func(ad catalog.Adapter) bool { return ad.Name == j.Name }) +} + +// ShareTraining changes who a job's adapter belongs to — owner as scope:ID, +// or "" for every caller its model allows — on the job and, once +// published, in the catalog. Refused wider than the knowledge bases its +// pairs came from may be read. +func (a *API) ShareTraining(ctx context.Context, id, owner, actor string) error { + j, err := a.Training(ctx, id) + if err != nil { + return err + } + _, colls, err := countPairs(string(j.Data)) + if err != nil { + return err + } + resolved, err := a.trainOwner(ctx, TrainRequest{Owner: owner, Public: owner == ""}, colls) + if err != nil { + return refused(http.StatusBadRequest, "%v", err) + } + if err := a.train.Store.SetTrainOwner(ctx, id, resolved); err != nil { + return err + } + m, ok := a.ctl.Catalog().Resolve(j.Model) + if !ok || a.models == nil { + return nil + } + ads := slices.Clone(m.Adapters) + i := slices.IndexFunc(ads, func(ad catalog.Adapter) bool { return ad.Name == j.Name }) + if i < 0 { + return nil + } + ads[i].Owner = resolved + _, err = a.models.SetAdapters(ctx, m.Name, ads, actor) + return err +} + +// DeleteTraining stops a job if it trains, takes its adapter out of the +// catalog if it was published, and forgets it. +func (a *API) DeleteTraining(ctx context.Context, id, actor string) error { + j, err := a.Training(ctx, id) + if err != nil { + return err + } + if !store.TrainOver(j.State) { + if err := a.train.Service.Cancel(ctx, id); err != nil { + return err + } + } + if m, ok := a.ctl.Catalog().Resolve(j.Model); ok && a.models != nil { + ads := slices.DeleteFunc(slices.Clone(m.Adapters), func(ad catalog.Adapter) bool { return ad.Name == j.Name }) + if len(ads) != len(m.Adapters) { + if _, err := a.models.SetAdapters(ctx, m.Name, ads, actor); err != nil { + return err + } + } + } + if err := a.train.Store.DeleteTrainJob(ctx, id); err != nil && !errors.Is(err, store.ErrNotFound) { + return err + } + return nil +} diff --git a/internal/control/adminapi/train_portal_test.go b/internal/control/adminapi/train_portal_test.go new file mode 100644 index 0000000..c1829a4 --- /dev/null +++ b/internal/control/adminapi/train_portal_test.go @@ -0,0 +1,137 @@ +package adminapi + +import ( + "context" + "errors" + "path/filepath" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" + "gitea.mynodes.xyz/ilker/llmgw/internal/hf" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +type fakeRegistry struct { + repos map[string]hf.Repo + asked int +} + +func (f *fakeRegistry) RepoAt(_ context.Context, id, _ string) (hf.Repo, error) { + f.asked++ + r, ok := f.repos[id] + if !ok { + return hf.Repo{}, errors.New("404") + } + return r, nil +} + +func TestAPortalTrainingFindsItsWeightsAndMemoryAndOneTrainsPerProject(t *testing.T) { + ctx := context.Background() + models := append(testModels(), + catalog.Model{Name: "small", Source: "/s", Origin: &catalog.Origin{Repo: "bartowski/Small-GGUF"}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}, + catalog.Model{Name: "gated", Source: "/g", Origin: &catalog.Origin{Repo: "meta/Gated"}, + Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}, + catalog.Model{Name: "local", Source: "/l", Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}}) + a, _ := setup(t, models, gpuNode("cuda01", 1)) + s, err := store.OpenSQLite(ctx, filepath.Join(t.TempDir(), "s.db")) + if err != nil { + t.Fatal(err) + } + defer s.Close() + reg := &fakeRegistry{repos: map[string]hf.Repo{ + "bartowski/Small-GGUF": {ID: "bartowski/Small-GGUF", Revision: "g1", BaseModel: []string{"Org/Small"}, + Files: []hf.File{{Path: "small-Q4_K_M.gguf", Bytes: 400 << 20}}}, + "Org/Small": {ID: "Org/Small", Revision: "abc", Files: []hf.File{{Path: "model.safetensors", Bytes: 1000 << 20}, + {Path: "config.json", Bytes: 1 << 10}}}, + "meta/Gated": {ID: "meta/Gated", Gated: hf.Gated{Reason: "manual"}, + Files: []hf.File{{Path: "model.safetensors", Bytes: 1 << 30}}}, + }} + a.SetTrain(&TrainAdmin{Store: s, Service: train.New(s, noAgents{}, nil)}) + a.SetTrainMirror(nil, reg) + + if got := a.TrainableModels(); strings.Join(got, ",") != "gated,small" { + t.Errorf("trainable: %v", got) + } + data := train.NDJSON([]train.Pair{{Q: "a", A: "b"}}) + req := TrainRequest{Name: "tone", Model: "small", Data: data, Owner: "project:p1", Project: "p1", Publish: true} + j, err := a.StartTraining(ctx, req, "ayse@x") + if err != nil { + t.Fatal(err) + } + if j.BaseRepo != "Org/Small" || j.BaseRevision != "abc" || j.Node != "" || j.GPUMemoryMiB != TrainMemoryMiB(1000<<20) || + !j.Publish || j.Project != "p1" || j.CreatedBy != "ayse@x" || !j.GGUF { + t.Errorf("queued %+v", j) + } + if j.GPUMemoryMiB != 3072 { + t.Errorf("1000 MiB of weights take %d MiB to train", j.GPUMemoryMiB) + } + + req.Name = "tone2" + if _, err := a.StartTraining(ctx, req, "mehmet@x"); err == nil || !strings.Contains(err.Error(), "already has a model training") { + t.Errorf("a second in the project: %v", err) + } + req.Name, req.Project, req.Owner = "tone", "p2", "project:p2" + if _, err := a.StartTraining(ctx, req, "x"); err == nil || !strings.Contains(err.Error(), "already being trained") { + t.Errorf("the same name twice: %v", err) + } + req.Name = "tone3" + asked := reg.asked + if _, err := a.StartTraining(ctx, req, "x"); err != nil { + t.Errorf("another project: %v", err) + } + if reg.asked != asked { + t.Errorf("the registry was asked again: %d", reg.asked-asked) + } + for model, says := range map[string]string{"gated": "gated", "local": "did not come from the model registry"} { + req.Model, req.Name, req.Project = model, "x-"+model, "p-"+model + _, err := a.StartTraining(ctx, req, "x") + if err == nil || !strings.Contains(err.Error(), says) { + t.Errorf("%s: %v", model, err) + } + if msg, theirs := TrainRefusal(err); !theirs || msg == "" { + t.Errorf("%s is not the request's fault: %q", model, msg) + } + } + + jobs, err := a.Trainings(ctx, []string{"p1"}, "nobody") + if err != nil || len(jobs) != 1 || jobs[0].Name != "tone" { + t.Errorf("p1's: %+v %v", jobs, err) + } + if err := a.ShareTraining(ctx, j.ID, "", "ayse@x"); err != nil { + t.Fatal(err) + } + if got, _ := a.Training(ctx, j.ID); got.Owner != "" { + t.Errorf("shared with everyone: %q", got.Owner) + } + if err := a.DeleteTraining(ctx, j.ID, "ayse@x"); err != nil { + t.Fatal(err) + } + if _, err := a.Training(ctx, j.ID); !errors.Is(err, store.ErrNotFound) { + t.Errorf("deleted: %v", err) + } +} + +func TestATrainingGoesToTheNodeWithTheMostFree(t *testing.T) { + busy := gpuNode("busy", 1) + busy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 23000}}} + roomy := gpuNode("roomy", 1) + roomy.Load = &inventory.Load{GPUs: []inventory.GPUSensor{{Index: 0, MemoryTotalMB: 24000, MemoryUsedMB: 4000}}} + cpu := inventory.Node{ID: "cpu", State: inventory.Ready} + small := gpuNode("small", 1) + small.GPUs = []discovery.GPU{{Index: 0, MemoryTotalMB: 8000}} + a, _ := setup(t, testModels(), busy, roomy, cpu, small) + if got := a.PickTrainNode(3000, nil); got != "roomy" { + t.Errorf("picked %q", got) + } + if got := a.PickTrainNode(3000, map[string]bool{"roomy": true}); got != "small" { + t.Errorf("with roomy busy, picked %q", got) + } + if got := a.PickTrainNode(30000, nil); got != "" { + t.Errorf("none has 30 GB, picked %q", got) + } +} diff --git a/internal/control/train/pairs.go b/internal/control/train/pairs.go new file mode 100644 index 0000000..8dc96e9 --- /dev/null +++ b/internal/control/train/pairs.go @@ -0,0 +1,205 @@ +package train + +import ( + "bufio" + "bytes" + "encoding/csv" + "encoding/json" + "errors" + "fmt" + "path" + "strings" + "unicode/utf8" + + "gitea.mynodes.xyz/ilker/llmgw/internal/kb/extract" +) + +// Pairs from a file a person uploads: a spreadsheet of questions and +// answers (Excel, CSV, tab-separated) or llmgw's own NDJSON of {"q", "a"}. +// The columns are found by their header — Soru/Cevap, Question/Answer and +// the like — or are the first two when the first row is not a header. + +// Pair is one question and the answer to learn for it. +type Pair struct { + Q string `json:"q"` + A string `json:"a"` +} + +// Read is what a file gave. +type Read struct { + Pairs []Pair + // Skipped counts rows with a question and no answer, or the other way. + Skipped int + // Columns names the columns read, as the header has them; empty for + // NDJSON and for a table with no header. + Columns [2]string +} + +// MaxPairs bounds one file: past it a job trains for hours on a page that +// was meant to take minutes. +const MaxPairs = 20000 + +var ( + questionNames = []string{"soru", "sorular", "question", "questions", "q", "prompt", "instruction", "input", "girdi"} + answerNames = []string{"cevap", "cevaplar", "yanıt", "yanit", "answer", "answers", "a", "response", "output", "completion", "çıktı", "cikti"} +) + +// ReadPairs reads a file's pairs; name says what kind it is. +func ReadPairs(name string, b []byte) (Read, error) { + var rows [][]string + switch strings.ToLower(path.Ext(name)) { + case ".xlsx", ".xlsm": + r, err := extract.XLSXRows(b) + if err != nil { + return Read{}, fmt.Errorf("the Excel file could not be read: %w", err) + } + rows = r + case ".jsonl", ".ndjson", ".json": + return readNDJSON(b) + case ".csv", ".tsv", ".txt": + r, err := readDelimited(b) + if err != nil { + return Read{}, err + } + rows = r + case ".xls": + return Read{}, errors.New("an old Excel file (.xls): save it as .xlsx and choose it again") + default: + return Read{}, errors.New("choose an Excel (.xlsx), CSV or JSONL file") + } + return fromRows(rows) +} + +func readDelimited(b []byte) ([][]string, error) { + b = bytes.TrimPrefix(b, []byte("\xef\xbb\xbf")) + if !utf8.Valid(b) { + return nil, errors.New("the file is not UTF-8 text: save it as \"CSV UTF-8\"") + } + first, _, _ := strings.Cut(string(b), "\n") + // Whichever separator the header line has most of: Excel writes ";" + // where the decimal mark is a comma, as it is in Turkish. + sep, most := ',', strings.Count(first, ",") + for _, c := range []rune{';', '\t'} { + if n := strings.Count(first, string(c)); n > most { + sep, most = c, n + } + } + r := csv.NewReader(bytes.NewReader(b)) + r.Comma, r.FieldsPerRecord, r.LazyQuotes = sep, -1, true + rows, err := r.ReadAll() + if err != nil { + return nil, fmt.Errorf("the file could not be read as a table: %w", err) + } + return rows, nil +} + +func fromRows(rows [][]string) (Read, error) { + var out Read + qc, ac := 0, 1 + if len(rows) > 0 { + q, a := -1, -1 + for i, c := range rows[0] { + n := strings.ToLower(strings.TrimSpace(c)) + switch { + case q < 0 && contains(questionNames, n): + q = i + case a < 0 && contains(answerNames, n): + a = i + } + } + if q >= 0 && a >= 0 { + qc, ac = q, a + out.Columns = [2]string{strings.TrimSpace(rows[0][q]), strings.TrimSpace(rows[0][a])} + rows = rows[1:] + } + } + for _, r := range rows { + var q, a string + if qc < len(r) { + q = strings.TrimSpace(r[qc]) + } + if ac < len(r) { + a = strings.TrimSpace(r[ac]) + } + switch { + case q == "" && a == "": + case q == "" || a == "": + out.Skipped++ + default: + out.Pairs = append(out.Pairs, Pair{Q: q, A: a}) + } + } + return out, check(out) +} + +func contains(names []string, n string) bool { + for _, x := range names { + if x == n { + return true + } + } + return false +} + +func readNDJSON(b []byte) (Read, error) { + var out Read + sc := bufio.NewScanner(bytes.NewReader(b)) + sc.Buffer(make([]byte, 64<<10), 1<<20) + line := 0 + for sc.Scan() { + line++ + l := bytes.TrimSpace(sc.Bytes()) + if len(l) == 0 { + continue + } + var p struct { + Q, A string + Question, Answer string + Prompt, Completion string + } + if err := json.Unmarshal(l, &p); err != nil { + return Read{}, fmt.Errorf("line %d is not JSON: %v", line, err) + } + q, a := first(p.Q, p.Question, p.Prompt), first(p.A, p.Answer, p.Completion) + if q == "" || a == "" { + out.Skipped++ + continue + } + out.Pairs = append(out.Pairs, Pair{Q: q, A: a}) + } + if err := sc.Err(); err != nil { + return Read{}, err + } + return out, check(out) +} + +func first(s ...string) string { + for _, x := range s { + if x = strings.TrimSpace(x); x != "" { + return x + } + } + return "" +} + +func check(r Read) error { + switch { + case len(r.Pairs) == 0: + return errors.New("no question and answer found: the first column is the question and the second the answer, " + + "or columns headed Soru and Cevap") + case len(r.Pairs) > MaxPairs: + return fmt.Errorf("%d pairs, more than the %d one training takes", len(r.Pairs), MaxPairs) + } + return nil +} + +// NDJSON is the pairs as a job's data. +func NDJSON(pairs []Pair) string { + var b strings.Builder + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + for _, p := range pairs { + _ = enc.Encode(p) + } + return b.String() +} diff --git a/internal/control/train/pairs_test.go b/internal/control/train/pairs_test.go new file mode 100644 index 0000000..84c4c1b --- /dev/null +++ b/internal/control/train/pairs_test.go @@ -0,0 +1,65 @@ +package train + +import ( + "strings" + "testing" +) + +func TestPairsFromATableByItsHeaderOrItsFirstColumns(t *testing.T) { + for _, c := range []struct { + name, file string + want []Pair + skipped int + cols [2]string + }{ + {"headed.csv", "No;Cevap;Soru\n1;Yirmi gün.;İzin kaç gün?\n2;;Boş cevap\n3;Evet;Uzaktan çalışabilir miyim?\n", + []Pair{{"İzin kaç gün?", "Yirmi gün."}, {"Uzaktan çalışabilir miyim?", "Evet"}}, 1, [2]string{"Soru", "Cevap"}}, + {"plain.csv", "\ufeffİzin kaç gün?,\"Yirmi gün, yılda.\"\n\nMasraf?,Formla.\n", + []Pair{{"İzin kaç gün?", "Yirmi gün, yılda."}, {"Masraf?", "Formla."}}, 0, [2]string{}}, + {"tabs.tsv", "Question\tAnswer\nA?\tB.\n", []Pair{{"A?", "B."}}, 0, [2]string{"Question", "Answer"}}, + {"d.jsonl", `{"q":"A?","a":"B."}` + "\n" + `{"question":"C?","answer":"D."}` + "\n" + `{"q":"E?"}` + "\n", + []Pair{{"A?", "B."}, {"C?", "D."}}, 1, [2]string{}}, + } { + got, err := ReadPairs(c.name, []byte(c.file)) + if err != nil { + t.Errorf("%s: %v", c.name, err) + continue + } + if len(got.Pairs) != len(c.want) || got.Skipped != c.skipped || got.Columns != c.cols { + t.Errorf("%s: %+v", c.name, got) + continue + } + for i := range c.want { + if got.Pairs[i] != c.want[i] { + t.Errorf("%s: pair %d = %+v, want %+v", c.name, i, got.Pairs[i], c.want[i]) + } + } + } +} + +func TestPairsRefusedWithAReason(t *testing.T) { + for name, file := range map[string]string{ + "a.csv": "only one column\n", + "a.xls": "x", + "a.pdf": "x", + "a.json": "not json\n", + } { + if _, err := ReadPairs(name, []byte(file)); err == nil { + t.Errorf("%s read", name) + } + } + var b strings.Builder + for range MaxPairs + 1 { + b.WriteString("x,y\n") + } + if _, err := ReadPairs("big.csv", []byte(b.String())); err == nil || !strings.Contains(err.Error(), "more than") { + t.Errorf("too many: %v", err) + } +} + +func TestNDJSONIsWhatAJobReads(t *testing.T) { + got := NDJSON([]Pair{{"?", "a & b"}}) + if got != `{"q":"?","a":"a & b"}`+"\n" { + t.Errorf("%q", got) + } +} diff --git a/internal/control/train/service.go b/internal/control/train/service.go index b2a646d..507b619 100644 --- a/internal/control/train/service.go +++ b/internal/control/train/service.go @@ -35,11 +35,26 @@ type Service struct { agents Agents log *slog.Logger + // pick chooses the node for a job that names none, among those not + // busy; "" is none can take it now. Nil leaves such jobs queued. + pick func(needMiB int, busy map[string]bool) string + // done is told of a job that finished, after what it made is kept. + done func(j store.TrainJob) + mu sync.Mutex // pieces are artifacts arriving, by job and kind. pieces map[[2]string][]byte } +// SetPicker has jobs that name no node sent where pick says. +func (s *Service) SetPicker(pick func(needMiB int, busy map[string]bool) string) { s.pick = pick } + +// SetOnDone has done told of each job that finishes. +func (s *Service) SetOnDone(done func(j store.TrainJob)) { s.done = done } + +// Waiting is what a job that names no node says while no node can take it. +const Waiting = "waiting for a card with enough memory free" + // New is a service. func New(s Store, a Agents, log *slog.Logger) *Service { if log == nil { @@ -81,13 +96,27 @@ func (s *Service) Dispatch(ctx context.Context) { busy[j.Node] = true } for _, q := range queued { - if busy[q.Node] || !s.agents.Connected(q.Node) { + node := q.Node + if node == "" { + if s.pick != nil { + node = s.pick(q.GPUMemoryMiB, busy) + } + if node == "" { + if q.Message != Waiting { + q.Message = Waiting + _ = s.store.UpdateTrainJob(ctx, q) + } + continue + } + } + if busy[node] || !s.agents.Connected(node) { continue } j, err := s.store.TrainJob(ctx, q.ID) if err != nil { continue } + j.Node = node err = s.agents.Send(j.Node, &llmgwv1.ControlMessage{Msg: &llmgwv1.ControlMessage_TrainStart{TrainStart: &llmgwv1.TrainStart{ JobId: j.ID, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision, Data: j.Data, Params: j.Params, GpuMemoryMib: int32(j.GPUMemoryMiB), Gguf: j.GGUF}}}) //nolint:gosec // MiB @@ -135,6 +164,11 @@ func (s *Service) OnTrainStatus(node string, st *llmgwv1.TrainStatus) { } if err := s.store.UpdateTrainJob(ctx, j); err != nil { s.log.Warn("could not record a training job's progress", "job", j.ID, "err", err) + return + } + // What it made came before it said it was done, on the same stream. + if j.State == store.TrainDone && s.done != nil { + go s.done(j) } } diff --git a/internal/control/train/service_test.go b/internal/control/train/service_test.go index 5a7a8f0..dd3ed6c 100644 --- a/internal/control/train/service_test.go +++ b/internal/control/train/service_test.go @@ -122,3 +122,41 @@ func TestCancellingAJob(t *testing.T) { t.Errorf("the node was not told: %+v", a.sent) } } + +func TestAJobNamingNoNodeGoesWhereThePickerSaysAndIsToldWhenDone(t *testing.T) { + ctx := context.Background() + svc, s, a := setup(t) + job(t, s, "j1", "") + asked := 0 + free := "" + svc.SetPicker(func(need int, busy map[string]bool) string { + asked++ + if need != 2000 { + t.Errorf("asked for %d MiB", need) + } + return free + }) + done := make(chan store.TrainJob, 1) + svc.SetOnDone(func(j store.TrainJob) { done <- j }) + + svc.Dispatch(ctx) + if j, _ := s.TrainJob(ctx, "j1"); j.State != store.TrainQueued || j.Message != Waiting || len(a.sent) != 0 { + t.Fatalf("with no card free: %+v, sent %d", j, len(a.sent)) + } + free = "gpu2" + a.connected["gpu2"] = true + svc.Dispatch(ctx) + j, _ := s.TrainJob(ctx, "j1") + if len(a.sent) != 1 || j.Node != "gpu2" || j.State != store.TrainStarting { + t.Fatalf("picked: %+v, sent %d", j, len(a.sent)) + } + // Its node is the one it was sent to, which alone reports for it. + svc.OnTrainStatus("cuda01", &llmgwv1.TrainStatus{JobId: "j1", State: "done"}) + if j, _ := s.TrainJob(ctx, "j1"); j.State == store.TrainDone { + t.Fatal("another node finished it") + } + svc.OnTrainStatus("gpu2", &llmgwv1.TrainStatus{JobId: "j1", State: "done"}) + if got := <-done; got.ID != "j1" || got.Node != "gpu2" { + t.Errorf("told of %+v", got) + } +} diff --git a/internal/kb/extract/extract_test.go b/internal/kb/extract/extract_test.go index 73157ab..405cbff 100644 --- a/internal/kb/extract/extract_test.go +++ b/internal/kb/extract/extract_test.go @@ -7,6 +7,7 @@ import ( "encoding/base64" "errors" "os" + "reflect" "strings" "testing" @@ -321,3 +322,25 @@ func TestHelpersWhenInstalled(t *testing.T) { } wants(t, got, "LibreOffice yolu") } + +func TestWorkbookRowsKeepTheirColumns(t *testing.T) { + b := zipOf(t, + "xl/workbook.xml", ``, + "xl/_rels/workbook.xml.rels", ``, + "xl/sharedStrings.xml", `SoruCevapİzin kaç gün?`, + "xl/worksheets/sheet1.xml", ``+ + `01`+ + ``+ + `2not`+ + `20`+ + ``, + ) + rows, err := XLSXRows(b) + if err != nil { + t.Fatal(err) + } + want := [][]string{{"Soru", "Cevap"}, {"İzin kaç gün?", "", "not"}, {"", "20"}} + if !reflect.DeepEqual(rows, want) { + t.Errorf("rows = %q, want %q", rows, want) + } +} diff --git a/internal/kb/extract/ooxml.go b/internal/kb/extract/ooxml.go index 8ca4eb4..0d2ce07 100644 --- a/internal/kb/extract/ooxml.go +++ b/internal/kb/extract/ooxml.go @@ -9,6 +9,7 @@ import ( "io" "path" "regexp" + "slices" "sort" "strconv" "strings" @@ -227,29 +228,7 @@ func xlsxText(b []byte) (string, error) { if err != nil { return "", err } - var shared []string - if ss, _ := member(zr, "xl/sharedStrings.xml"); ss != nil { - var cur strings.Builder - inT, inRPh := false, 0 - _ = walkXML(ss, func(e xmlEvent) { - switch { - case e.start && e.name == "si": - cur.Reset() - case e.end && e.name == "si": - shared = append(shared, cur.String()) - case e.start && e.name == "rPh": - inRPh++ // phonetic guides, not the cell's text - case e.end && e.name == "rPh": - inRPh-- - case e.start && e.name == "t": - inT = true - case e.end && e.name == "t": - inT = false - case !e.start && !e.end && inT && inRPh == 0: - cur.WriteString(e.text) - } - }) - } + shared := xlsxShared(zr) sheets, err := xlsxSheets(zr) if err != nil { return "", err @@ -307,6 +286,118 @@ func xlsxText(b []byte) (string, error) { return o.String(), nil } +// xlsxShared is a workbook's shared strings, which cells name by index. +func xlsxShared(zr *zip.Reader) []string { + var shared []string + ss, _ := member(zr, "xl/sharedStrings.xml") + if ss == nil { + return nil + } + var cur strings.Builder + inT, inRPh := false, 0 + _ = walkXML(ss, func(e xmlEvent) { + switch { + case e.start && e.name == "si": + cur.Reset() + case e.end && e.name == "si": + shared = append(shared, cur.String()) + case e.start && e.name == "rPh": + inRPh++ // phonetic guides, not the cell's text + case e.end && e.name == "rPh": + inRPh-- + case e.start && e.name == "t": + inT = true + case e.end && e.name == "t": + inT = false + case !e.start && !e.end && inT && inRPh == 0: + cur.WriteString(e.text) + } + }) + return shared +} + +// XLSXRows reads a workbook's first sheet as rows of cells, each cell in +// its column (an empty one is ""), for a table such as questions and +// answers. Rows with nothing in them are left out. +func XLSXRows(b []byte) ([][]string, error) { + zr, err := openZip(b) + if err != nil { + return nil, err + } + shared := xlsxShared(zr) + sheets, err := xlsxSheets(zr) + if err != nil { + return nil, err + } + if len(sheets) == 0 { + return nil, errors.New("the workbook has no sheets") + } + data, err := member(zr, sheets[0].path) + if err != nil || data == nil { + return nil, fmt.Errorf("the workbook's first sheet, %s, could not be read", sheets[0].name) + } + var rows [][]string + var row []string + var cell strings.Builder + cellType, col, next := "", 0, 0 + inV, inIs := false, false + _ = walkXML(data, func(e xmlEvent) { + switch { + case e.start && e.name == "row": + row, next = nil, 0 + case e.end && e.name == "row": + if slices.ContainsFunc(row, func(s string) bool { return s != "" }) { + rows = append(rows, row) + } + case e.start && e.name == "c": + cellType = e.get("t") + col = next + if c, ok := columnOf(e.get("r")); ok { + col = c + } + next = col + 1 + cell.Reset() + case e.end && e.name == "c": + v := cell.String() + if cellType == "s" { + if i, err := strconv.Atoi(strings.TrimSpace(v)); err == nil && i >= 0 && i < len(shared) { + v = shared[i] + } + } + if col < 0 || col > 16383 { + return + } + for len(row) <= col { + row = append(row, "") + } + row[col] = strings.TrimSpace(v) + case e.start && e.name == "v": + inV = true + case e.end && e.name == "v": + inV = false + case e.start && e.name == "is": + inIs = true + case e.end && e.name == "is": + inIs = false + case !e.start && !e.end && (inV || inIs): + cell.WriteString(e.text) + } + }) + return rows, nil +} + +// columnOf is a cell reference's column, from 0: "B7" is 1. +func columnOf(ref string) (int, bool) { + n, i := 0, 0 + for ; i < len(ref) && ref[i] >= 'A' && ref[i] <= 'Z'; i++ { + n = n*26 + int(ref[i]-'A'+1) + } + if i == 0 { + return 0, false + } + return n - 1, true +} + type sheetRef struct{ name, path string } // xlsxSheets lists a workbook's sheets in their order, with the part each is diff --git a/internal/router/chat.go b/internal/router/chat.go index d3bdbc0..0f634df 100644 --- a/internal/router/chat.go +++ b/internal/router/chat.go @@ -58,6 +58,7 @@ func (r *Router) registerChat(mux *http.ServeMux, pages *web.AccountPages) { })) r.registerImagesPage(mux, pages) r.registerMeshesPage(mux, pages) + r.registerTrainPage(mux, pages) if r.store != nil { pages.SetLicences(r.licenceChoices, r.store) } diff --git a/internal/router/handlers.go b/internal/router/handlers.go index cba291d..0407f76 100644 --- a/internal/router/handlers.go +++ b/internal/router/handlers.go @@ -384,7 +384,7 @@ func (r *Router) Handler() http.Handler { } exempt := []string{"/health", "/ready", "/leader", "/v1/openapi.json", "/", "/about", "/install", "/install/", "/_static/", "/_lang/", "/claim/", "/signup", "/login", "/login/", "/logout", "/profile", "/profile/", "/scim/", - "/admin", "/admin/", "/chat", "/chat/", "/images", "/images/", "/3d", "/3d/", "/org", "/org/", "/setup/"} + "/admin", "/admin/", "/chat", "/chat/", "/images", "/images/", "/3d", "/3d/", "/train", "/train/", "/org", "/org/", "/setup/"} // The docs are public unless the site said otherwise, and then guarded by // the session check in front of them — or, on a gateway with keys and no // accounts to sign in to, by the key like any other path. diff --git a/internal/router/pages_auth_test.go b/internal/router/pages_auth_test.go index 1e21fe0..c5f02e1 100644 --- a/internal/router/pages_auth_test.go +++ b/internal/router/pages_auth_test.go @@ -17,7 +17,7 @@ func TestSessionPagesAreNotAskedForAnAPIKey(t *testing.T) { h := rt.Handler() // The /admin pages are mounted only with a cluster view, which this // router has none of; these are the session pages it does mount. - for _, path := range []string{"/profile", "/chat", "/images", "/3d", "/org", "/org/org-x"} { + for _, path := range []string{"/profile", "/chat", "/images", "/3d", "/train", "/org", "/org/org-x"} { w := httptest.NewRecorder() h.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) if strings.Contains(w.Body.String(), "api key") { diff --git a/internal/router/router.go b/internal/router/router.go index d851fc8..4933859 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -192,6 +192,9 @@ type Router struct { // registry browses a model registry from the mirror page. Nil where the // deployment has no way out to the internet. registry web.Registry + // training is the "customise a model" page's control plane; nil + // leaves the page off. + training atomic.Pointer[Training] // catalogAdmin puts a mirrored model into service. Nil without a store, // where a model added at runtime would not survive a restart. catalogAdmin web.CatalogAdmin diff --git a/internal/router/train.go b/internal/router/train.go new file mode 100644 index 0000000..8722b69 --- /dev/null +++ b/internal/router/train.go @@ -0,0 +1,331 @@ +package router + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "slices" + "strings" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/control/train" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" + "gitea.mynodes.xyz/ilker/llmgw/internal/version" + "gitea.mynodes.xyz/ilker/llmgw/internal/web" +) + +// The "customise a model" page: a person in a project uploads questions +// and answers, picks a model, and presses a button. The node, the weights +// to train on and how much of a card it takes are worked out; what it +// makes is published as a model of its own, seen by the project unless it +// is shared wider. Members of the project see its jobs and manage them; +// one trains at a time per project. + +// Training is what the page needs of the control plane. +type Training interface { + TrainableModels() []string + StartTraining(ctx context.Context, in adminapi.TrainRequest, actor string) (store.TrainJob, error) + Trainings(ctx context.Context, projects []string, createdBy string) ([]store.TrainJob, error) + Training(ctx context.Context, id string) (store.TrainJob, error) + Published(j store.TrainJob) bool + ShareTraining(ctx context.Context, id, owner, actor string) error + DeleteTraining(ctx context.Context, id, actor string) error +} + +// SetTraining mounts the page's actions on t. +func (r *Router) SetTraining(t Training) { r.training.Store(&t) } + +func (r *Router) trainer() Training { + if t := r.training.Load(); t != nil { + return *t + } + return nil +} + +// trainMaxFile bounds an uploaded file of pairs. +const trainMaxFile = 32 << 20 + +// The widths an adapter can be shared at, narrowest first. +var trainShares = []string{"me", "project", "team", "org", "all"} + +func (r *Router) registerTrainPage(mux *http.ServeMux, pages *web.AccountPages) { + pages.SetTrain(func() bool { + t := r.trainer() + return t != nil && r.store != nil && len(t.TrainableModels()) > 0 + }) + mux.HandleFunc("GET /train", pages.Guard(func(w http.ResponseWriter, req *http.Request, user store.User) { + t := r.trainer() + if t == nil || r.store == nil { + http.NotFound(w, req) + return + } + projects, _ := r.store.ProjectsFor(req.Context(), user.ID) + var ps []web.ChatContext + for _, p := range projects { + ps = append(ps, web.ChatContext{Value: p.ID, Label: p.Path}) + } + web.Train(user, t.TrainableModels(), ps, version.Version)(w, req) + })) + mux.HandleFunc("POST /train/preview", pages.Guard(r.trainPreview)) + mux.HandleFunc("POST /train/start", pages.Guard(r.trainStart)) + mux.HandleFunc("GET /train/jobs", pages.Guard(r.trainJobs)) + mux.HandleFunc("POST /train/jobs/{id}/share", pages.Guard(r.trainShare)) + mux.HandleFunc("POST /train/jobs/{id}/delete", pages.Guard(r.trainDelete)) +} + +func trainFail(w http.ResponseWriter, code int, msg string) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) + _ = json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} + +// trainFile reads the pairs of the uploaded file. +func trainFile(w http.ResponseWriter, req *http.Request) (train.Read, bool) { + req.Body = http.MaxBytesReader(w, req.Body, trainMaxFile+1<<20) + f, h, err := req.FormFile("file") + if err != nil { + trainFail(w, http.StatusBadRequest, "choose a file of questions and answers") + return train.Read{}, false + } + defer f.Close() + b, err := io.ReadAll(io.LimitReader(f, trainMaxFile+1)) + if err != nil || len(b) > trainMaxFile { + trainFail(w, http.StatusBadRequest, "the file is over 32 MB") + return train.Read{}, false + } + rd, err := train.ReadPairs(h.Filename, b) + if err != nil { + trainFail(w, http.StatusBadRequest, err.Error()) + return train.Read{}, false + } + return rd, true +} + +// trainPreview says what a file holds before anything is trained on it. +func (r *Router) trainPreview(w http.ResponseWriter, req *http.Request, _ store.User) { + rd, ok := trainFile(w, req) + if !ok { + return + } + sample := rd.Pairs[:min(len(rd.Pairs), 5)] + writeJSONBody(w, map[string]any{"pairs": len(rd.Pairs), "skipped": rd.Skipped, "columns": rd.Columns, "sample": sample}) +} + +// trainPlace is where a project is in the tree. +func (r *Router) trainPlace(ctx context.Context, user store.User, project string) (store.Effective, error) { + projects, err := r.store.ProjectsFor(ctx, user.ID) + if err != nil { + return store.Effective{}, err + } + if !slices.ContainsFunc(projects, func(p store.ProjectPath) bool { return p.ID == project }) { + return store.Effective{}, errors.New("you are not a member of that project") + } + return r.store.ResolveFor(ctx, "", "", project) +} + +// trainOwner is the adapter's owner for a share width, in a place. +func trainOwner(share string, user store.User, e store.Effective) (string, bool) { + switch share { + case "me": + return "user:" + user.ID, true + case "project": + return "project:" + e.ProjectID, true + case "team": + return "team:" + e.TeamID, true + case "org": + return "org:" + e.OrgID, true + case "all": + return "", true + } + return "", false +} + +// trainShareOf is the width an owner is, seen from the job's place. +func trainShareOf(owner string, e store.Effective) string { + switch { + case owner == "": + return "all" + case strings.HasPrefix(owner, "user:"): + return "me" + case owner == "project:"+e.ProjectID: + return "project" + case owner == "team:"+e.TeamID: + return "team" + case owner == "org:"+e.OrgID: + return "org" + } + return "other" +} + +func (r *Router) trainStart(w http.ResponseWriter, req *http.Request, user store.User) { + t := r.trainer() + if t == nil { + trainFail(w, http.StatusNotFound, "training is not offered here") + return + } + rd, ok := trainFile(w, req) + if !ok { + return + } + e, err := r.trainPlace(req.Context(), user, req.FormValue("project")) + if err != nil { + trainFail(w, http.StatusForbidden, err.Error()) + return + } + share := req.FormValue("share") + owner, ok := trainOwner(share, user, e) + if !ok { + trainFail(w, http.StatusBadRequest, "who may use it: me, the project, the team, the organization or everyone") + return + } + if slices.Index(trainShares, share) > slices.Index(trainShares, "project") && req.FormValue("confirm") != "yes" { + trainFail(w, http.StatusBadRequest, "confirm that the file holds nothing the wider group must not see") + return + } + in := adminapi.TrainRequest{Name: strings.TrimSpace(req.FormValue("name")), Model: req.FormValue("model"), + Owner: owner, Public: owner == "", Data: train.NDJSON(rd.Pairs), Project: e.ProjectID, Publish: true} + if !slices.Contains(t.TrainableModels(), in.Model) { + trainFail(w, http.StatusBadRequest, "choose one of the models offered") + return + } + j, err := t.StartTraining(req.Context(), in, user.Email) + if err != nil { + msg, theirs := adminapi.TrainRefusal(err) + code := http.StatusBadRequest + if !theirs { + code = http.StatusInternalServerError + r.log.Warn("training could not be started from the portal", "user", user.Email, "err", err) + } + trainFail(w, code, msg) + return + } + r.log.Info("training started from the portal", "job", j.ID, "adapter", j.Name, "model", j.Model, + "project", j.Project, "user", user.Email, "pairs", j.Pairs) + writeJSONBody(w, map[string]string{"id": j.ID}) +} + +// trainRow is a job as the page lists it. +type trainRow struct { + ID string `json:"id"` + Name string `json:"name"` + Model string `json:"model"` + Project string `json:"project"` + State string `json:"state"` + Step int `json:"step"` + Steps int `json:"steps"` + Message string `json:"message,omitempty"` + Pairs int `json:"pairs"` + Share string `json:"share"` + Published bool `json:"published"` + CreatedBy string `json:"created_by"` + CreatedAt time.Time `json:"created_at"` + Minutes int `json:"minutes"` +} + +func (r *Router) trainJobs(w http.ResponseWriter, req *http.Request, user store.User) { + t := r.trainer() + if t == nil || r.store == nil { + writeJSONBody(w, []trainRow{}) + return + } + projects, err := r.store.ProjectsFor(req.Context(), user.ID) + if err != nil { + trainFail(w, http.StatusInternalServerError, err.Error()) + return + } + ids, names := []string{}, map[string]string{} + for _, p := range projects { + ids = append(ids, p.ID) + names[p.ID] = p.Path + } + jobs, err := t.Trainings(req.Context(), ids, user.Email) + if err != nil { + trainFail(w, http.StatusInternalServerError, err.Error()) + return + } + out := []trainRow{} + for _, j := range jobs { + e, _ := r.store.ResolveFor(req.Context(), "", "", j.Project) + row := trainRow{ID: j.ID, Name: j.Name, Model: j.Model, Project: names[j.Project], State: j.State, Step: j.Step, + Steps: j.Steps, Message: j.Message, Pairs: j.Pairs, Share: trainShareOf(j.Owner, e), + Published: j.State == store.TrainDone && t.Published(j), CreatedBy: j.CreatedBy, CreatedAt: j.CreatedAt} + if !j.StartedAt.IsZero() { + end := j.FinishedAt + if end.IsZero() { + end = time.Now() + } + row.Minutes = int(end.Sub(j.StartedAt).Minutes()) + } + out = append(out, row) + } + writeJSONBody(w, out) +} + +// trainMine is a job a person may manage: their project's, or their own. +func (r *Router) trainMine(w http.ResponseWriter, req *http.Request, user store.User) (store.TrainJob, Training, bool) { + t := r.trainer() + if t == nil || r.store == nil { + trainFail(w, http.StatusNotFound, "training is not offered here") + return store.TrainJob{}, nil, false + } + j, err := t.Training(req.Context(), req.PathValue("id")) + if err == nil { + if j.Project == "" { + if j.CreatedBy == user.Email { + return j, t, true + } + } else if _, err := r.trainPlace(req.Context(), user, j.Project); err == nil { + return j, t, true + } + } + trainFail(w, http.StatusNotFound, "no such training") + return store.TrainJob{}, nil, false +} + +func (r *Router) trainShare(w http.ResponseWriter, req *http.Request, user store.User) { + j, t, ok := r.trainMine(w, req, user) + if !ok { + return + } + e, _ := r.store.ResolveFor(req.Context(), "", "", j.Project) + share := req.FormValue("share") + owner, ok := trainOwner(share, user, e) + if !ok || (j.Project == "" && share != "me" && share != "all") { + trainFail(w, http.StatusBadRequest, "who may use it: me, the project, the team, the organization or everyone") + return + } + if slices.Index(trainShares, share) > slices.Index(trainShares, "project") && req.FormValue("confirm") != "yes" { + trainFail(w, http.StatusBadRequest, "confirm that the data holds nothing the wider group must not see") + return + } + if err := t.ShareTraining(req.Context(), j.ID, owner, user.Email); err != nil { + msg, _ := adminapi.TrainRefusal(err) + trainFail(w, http.StatusBadRequest, msg) + return + } + r.log.Info("a trained model's sharing changed from the portal", "job", j.ID, "adapter", j.Name, "share", share, + "user", user.Email) + writeJSONBody(w, map[string]string{"share": share}) +} + +func (r *Router) trainDelete(w http.ResponseWriter, req *http.Request, user store.User) { + j, t, ok := r.trainMine(w, req, user) + if !ok { + return + } + if err := t.DeleteTraining(req.Context(), j.ID, user.Email); err != nil { + msg, _ := adminapi.TrainRefusal(err) + trainFail(w, http.StatusInternalServerError, msg) + return + } + r.log.Info("a trained model was deleted from the portal", "job", j.ID, "adapter", j.Name, "user", user.Email) + writeJSONBody(w, map[string]bool{"deleted": true}) +} + +func writeJSONBody(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + _ = json.NewEncoder(w).Encode(v) +} diff --git a/internal/router/train_test.go b/internal/router/train_test.go new file mode 100644 index 0000000..bd6ee3e --- /dev/null +++ b/internal/router/train_test.go @@ -0,0 +1,165 @@ +package router + +import ( + "bytes" + "context" + "encoding/json" + "mime/multipart" + "net/http" + "net/http/httptest" + "path/filepath" + "strings" + "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi" + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +type fakeTraining struct { + started []adminapi.TrainRequest + jobs map[string]store.TrainJob + shared map[string]string + deleted []string +} + +func (f *fakeTraining) TrainableModels() []string { return []string{"qwen"} } +func (f *fakeTraining) StartTraining(_ context.Context, in adminapi.TrainRequest, actor string) (store.TrainJob, error) { + f.started = append(f.started, in) + j := store.TrainJob{ID: "train-" + in.Name, Name: in.Name, Model: in.Model, Owner: in.Owner, Project: in.Project, + CreatedBy: actor, State: store.TrainQueued, Pairs: strings.Count(in.Data, "\n"), CreatedAt: time.Now()} + f.jobs[j.ID] = j + return j, nil +} +func (f *fakeTraining) Trainings(_ context.Context, projects []string, by string) ([]store.TrainJob, error) { + var out []store.TrainJob + for _, j := range f.jobs { + for _, p := range projects { + if j.Project == p { + out = append(out, j) + } + } + } + return out, nil +} +func (f *fakeTraining) Training(_ context.Context, id string) (store.TrainJob, error) { + j, ok := f.jobs[id] + if !ok { + return j, store.ErrNotFound + } + return j, nil +} +func (f *fakeTraining) Published(store.TrainJob) bool { return false } +func (f *fakeTraining) ShareTraining(_ context.Context, id, owner, _ string) error { + f.shared[id] = owner + return nil +} +func (f *fakeTraining) DeleteTraining(_ context.Context, id, _ string) error { + f.deleted = append(f.deleted, id) + return nil +} + +func trainForm(t *testing.T, fields map[string]string, file string) *http.Request { + t.Helper() + var b bytes.Buffer + mw := multipart.NewWriter(&b) + for k, v := range fields { + _ = mw.WriteField(k, v) + } + if file != "" { + w, _ := mw.CreateFormFile("file", "ornekler.csv") + _, _ = w.Write([]byte(file)) + } + _ = mw.Close() + req := httptest.NewRequest("POST", "/train/start", &b) + req.Header.Set("Content-Type", mw.FormDataContentType()) + return req +} + +func TestThePortalTrainsForAProjectItsMembersManage(t *testing.T) { + cfg := baseConfig() + cfg.Auth.Enabled = true + cfg.Auth.Store = filepath.Join(t.TempDir(), "store.db") + rt := newTestRouter(t, cfg) + ft := &fakeTraining{jobs: map[string]store.TrainJob{}, shared: map[string]string{}} + rt.SetTraining(ft) + ctx := t.Context() + org, _ := rt.store.CreateOrg(ctx, "acme") + team, _ := rt.store.CreateTeam(ctx, org, "support") + project, _ := rt.store.CreateProject(ctx, team, "bot") + ayse := store.User{ID: "usr-a", Email: "ayse@acme.example", State: store.UserActive, CreatedAt: time.Now()} + bob := store.User{ID: "usr-b", Email: "bob@else.example", State: store.UserActive, CreatedAt: time.Now()} + for _, u := range []store.User{ayse, bob} { + if err := rt.store.CreateUser(ctx, u); err != nil { + t.Fatal(err) + } + } + _ = rt.store.AddMembership(ctx, ayse.ID, store.ScopeProject, project, "member", "test", time.Now()) + csv := "Soru,Cevap\nİzin kaç gün?,Yirmi gün.\nMasraf nasıl?,Formla.\n" + start := func(u store.User, fields map[string]string) *httptest.ResponseRecorder { + rec := httptest.NewRecorder() + rt.trainStart(rec, trainForm(t, fields, csv), u) + return rec + } + good := map[string]string{"name": "destek", "model": "qwen", "project": project, "share": "project"} + + if rec := start(bob, good); rec.Code != http.StatusForbidden { + t.Errorf("someone outside the project started one: %d %s", rec.Code, rec.Body.String()) + } + wide := map[string]string{"name": "destek", "model": "qwen", "project": project, "share": "org"} + if rec := start(ayse, wide); rec.Code != http.StatusBadRequest || !strings.Contains(rec.Body.String(), "confirm") { + t.Errorf("shared with the organization unconfirmed: %d %s", rec.Code, rec.Body.String()) + } + other := map[string]string{"name": "destek", "model": "llama-70b", "project": project, "share": "project"} + if rec := start(ayse, other); rec.Code != http.StatusBadRequest { + t.Errorf("a model not offered: %d", rec.Code) + } + if rec := start(ayse, good); rec.Code != http.StatusOK { + t.Fatalf("%d %s", rec.Code, rec.Body.String()) + } + in := ft.started[0] + if in.Owner != "project:"+project || in.Project != project || !in.Publish || in.Public || + in.Data != `{"q":"İzin kaç gün?","a":"Yirmi gün."}`+"\n"+`{"q":"Masraf nasıl?","a":"Formla."}`+"\n" { + t.Errorf("asked for %+v", in) + } + + list := func(u store.User) []trainRow { + rec := httptest.NewRecorder() + rt.trainJobs(rec, httptest.NewRequest("GET", "/train/jobs", nil), u) + var rows []trainRow + _ = json.Unmarshal(rec.Body.Bytes(), &rows) + return rows + } + if rows := list(ayse); len(rows) != 1 || rows[0].Share != "project" || rows[0].Project != "acme / support / bot" || + rows[0].CreatedBy != ayse.Email { + t.Errorf("ayse sees %+v", rows) + } + if rows := list(bob); len(rows) != 0 { + t.Errorf("bob sees %+v", rows) + } + + act := func(u store.User, what string, fields map[string]string) int { + req := trainForm(t, fields, "") + req.SetPathValue("id", "train-destek") + rec := httptest.NewRecorder() + if what == "share" { + rt.trainShare(rec, req, u) + } else { + rt.trainDelete(rec, req, u) + } + return rec.Code + } + if code := act(bob, "delete", nil); code != http.StatusNotFound || len(ft.deleted) != 0 { + t.Errorf("bob deleted it: %d", code) + } + if code := act(ayse, "share", map[string]string{"share": "team", "confirm": "yes"}); code != http.StatusOK || + ft.shared["train-destek"] != "team:"+team { + t.Errorf("shared with the team: %d %v", code, ft.shared) + } + if code := act(ayse, "share", map[string]string{"share": "all"}); code != http.StatusBadRequest { + t.Errorf("shared with everyone unconfirmed: %d", code) + } + if code := act(ayse, "delete", nil); code != http.StatusOK || len(ft.deleted) != 1 { + t.Errorf("ayse deleting: %d", code) + } +} diff --git a/internal/store/migrations/postgres/0055_train_portal.sql b/internal/store/migrations/postgres/0055_train_portal.sql new file mode 100644 index 0000000..9a9c396 --- /dev/null +++ b/internal/store/migrations/postgres/0055_train_portal.sql @@ -0,0 +1,5 @@ +-- Training from the portal: the project a job was started in, which sees +-- it, manages it and pays for it; and whether what it makes is published +-- by itself when it is done, rather than by an administrator. +ALTER TABLE train_jobs ADD COLUMN project_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE train_jobs ADD COLUMN publish INTEGER NOT NULL DEFAULT 0; diff --git a/internal/store/migrations/sqlite/0055_train_portal.sql b/internal/store/migrations/sqlite/0055_train_portal.sql new file mode 100644 index 0000000..9a9c396 --- /dev/null +++ b/internal/store/migrations/sqlite/0055_train_portal.sql @@ -0,0 +1,5 @@ +-- Training from the portal: the project a job was started in, which sees +-- it, manages it and pays for it; and whether what it makes is published +-- by itself when it is done, rather than by an administrator. +ALTER TABLE train_jobs ADD COLUMN project_id TEXT NOT NULL DEFAULT ''; +ALTER TABLE train_jobs ADD COLUMN publish INTEGER NOT NULL DEFAULT 0; diff --git a/internal/store/train.go b/internal/store/train.go index bed35dd..dc15b9f 100644 --- a/internal/store/train.go +++ b/internal/store/train.go @@ -4,6 +4,7 @@ import ( "context" "database/sql" "errors" + "strings" "time" ) @@ -34,6 +35,11 @@ type TrainJob struct { StartedAt time.Time FinishedAt time.Time CreatedBy string + // Project is the project a job was started in from the portal: its + // members see and manage it, and it pays. Empty for an administrator's. + Project string + // Publish has what the job makes published when it is done. + Publish bool } // The states of a training job. @@ -55,27 +61,25 @@ func (s *DB) CreateTrainJob(ctx context.Context, j TrainJob) error { if j.CreatedAt.IsZero() { j.CreatedAt = time.Now() } - gguf := 0 - if j.GGUF { - gguf = 1 - } _, err := s.db.ExecContext(ctx, `INSERT INTO train_jobs (id, name, model, base_repo, base_revision, node, owner, gpu_memory_mib, - params, gguf, data, pairs, state, created_at, created_by) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - j.ID, j.Name, j.Model, j.BaseRepo, j.BaseRevision, j.Node, j.Owner, j.GPUMemoryMiB, j.Params, gguf, j.Data, j.Pairs, - TrainQueued, millis(j.CreatedAt), j.CreatedBy) + params, gguf, data, pairs, state, created_at, created_by, project_id, publish) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + j.ID, j.Name, j.Model, j.BaseRepo, j.BaseRevision, j.Node, j.Owner, j.GPUMemoryMiB, j.Params, boolInt(j.GGUF), j.Data, j.Pairs, + TrainQueued, millis(j.CreatedAt), j.CreatedBy, j.Project, boolInt(j.Publish)) return err } const trainCols = `id, name, model, base_repo, base_revision, node, owner, gpu_memory_mib, params, gguf, pairs, state, step, steps, - loss, message, peak_mib, created_at, started_at, finished_at, created_by` + loss, message, peak_mib, created_at, started_at, finished_at, created_by, project_id, publish` func scanTrainJob(sc scanner) (TrainJob, error) { var j TrainJob - var gguf int + var gguf, publish int var created, started, finished int64 err := sc.Scan(&j.ID, &j.Name, &j.Model, &j.BaseRepo, &j.BaseRevision, &j.Node, &j.Owner, &j.GPUMemoryMiB, &j.Params, &gguf, - &j.Pairs, &j.State, &j.Step, &j.Steps, &j.Loss, &j.Message, &j.PeakMiB, &created, &started, &finished, &j.CreatedBy) - j.GGUF = gguf != 0 + &j.Pairs, &j.State, &j.Step, &j.Steps, &j.Loss, &j.Message, &j.PeakMiB, &created, &started, &finished, &j.CreatedBy, + &j.Project, &publish) + j.GGUF, j.Publish = gguf != 0, publish != 0 j.CreatedAt = fromMillis(created) if started > 0 { j.StartedAt = fromMillis(started) @@ -128,13 +132,47 @@ func (s *DB) UpdateTrainJob(ctx context.Context, j TrainJob) error { if !j.FinishedAt.IsZero() { finished = millis(j.FinishedAt) } - _, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET state = ?, step = ?, steps = ?, loss = ?, message = ?, + _, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET node = ?, state = ?, step = ?, steps = ?, loss = ?, message = ?, peak_mib = ?, started_at = ?, finished_at = ? WHERE id = ? AND state NOT IN (?, ?, ?)`, - j.State, j.Step, j.Steps, j.Loss, j.Message, j.PeakMiB, started, finished, j.ID, + j.Node, j.State, j.Step, j.Steps, j.Loss, j.Message, j.PeakMiB, started, finished, j.ID, TrainDone, TrainFailed, TrainCancelled) return err } +// TrainJobsFor lists the jobs started in projects, or by a person outside +// any project, newest first. +func (s *DB) TrainJobsFor(ctx context.Context, projects []string, createdBy string, limit int) ([]TrainJob, error) { + q := `SELECT ` + trainCols + ` FROM train_jobs WHERE (project_id = '' AND publish = 1 AND created_by = ?)` + args := []any{createdBy} + if len(projects) > 0 { + q += ` OR project_id IN (?` + strings.Repeat(", ?", len(projects)-1) + `)` + for _, p := range projects { + args = append(args, p) + } + } + return queryAll(ctx, s.db, q+` ORDER BY created_at DESC LIMIT ?`, append(args, limit), + func(r *sql.Rows) (TrainJob, error) { return scanTrainJob(r) }) +} + +// SetTrainNote changes what a job says of itself, ended or not: what +// happened when what it made was published. +func (s *DB) SetTrainNote(ctx context.Context, id, message string) error { + _, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET message = ? WHERE id = ?`, message, id) + return err +} + +// SetTrainOwner changes who a job's adapter belongs to. +func (s *DB) SetTrainOwner(ctx context.Context, id, owner string) error { + res, err := s.db.ExecContext(ctx, `UPDATE train_jobs SET owner = ? WHERE id = ?`, owner, id) + if err != nil { + return err + } + if n, _ := res.RowsAffected(); n == 0 { + return ErrNotFound + } + return nil +} + // PutTrainArtifact keeps what a job made. func (s *DB) PutTrainArtifact(ctx context.Context, job, kind string, data []byte) error { _, err := s.db.ExecContext(ctx, `INSERT INTO train_artifacts (job_id, kind, data) VALUES (?, ?, ?) diff --git a/internal/web/account.go b/internal/web/account.go index 67569ae..ac24474 100644 --- a/internal/web/account.go +++ b/internal/web/account.go @@ -109,6 +109,7 @@ type AccountPages struct { licencePrefs LicencePrefs images func() bool meshes func() bool + train func() bool guardLibrary func() guard.Config // guardFor names the guardrail policies in force for someone, and // whether they keep nothing; nil when there are none. diff --git a/internal/web/locales/en/nav.json b/internal/web/locales/en/nav.json index 9f49797..3dfe73d 100644 --- a/internal/web/locales/en/nav.json +++ b/internal/web/locales/en/nav.json @@ -14,5 +14,6 @@ "nav.mcp": "MCP servers", "nav.settings": "Settings", "nav.images": "Pictures", - "nav.meshes": "3D" + "nav.meshes": "3D", + "nav.train": "Customise a model" } diff --git a/internal/web/locales/en/train.json b/internal/web/locales/en/train.json new file mode 100644 index 0000000..f4ed612 --- /dev/null +++ b/internal/web/locales/en/train.json @@ -0,0 +1,55 @@ +{ + "train.title": "Customise a model", + "train.intro": "Teach a model how to answer from your own examples: an Excel file with a column of questions and a column of answers. When training is done the new model appears under its own name in the chat and the API; it runs on the card its base model runs on, with no card of its own.", + "train.not_knowledge": "A model learns tone and the shape of answers from examples; it does not reliably learn the facts in your documents. For answers from your documents, use a knowledge base: the chat answers from them and shows where.", + "train.no_project": "Customising a model needs a project: training is done for a project, which pays for it. Ask an administrator to add you to one.", + "train.no_models": "There is no model that can be customised just now.", + "train.step.file": "1. Examples", + "train.file_hint": "Excel (.xlsx), CSV or JSONL. Columns headed Question and Answer are read; with no header the first column is the question and the second the answer. At least 50 examples, better a few hundred.", + "train.question": "Question", + "train.answer": "Answer", + "train.step.model": "2. Model", + "train.base": "Model to start from", + "train.name": "The new model's name", + "train.name_hint": "what it is chosen by in the chat: letters, digits, - and _", + "train.step.who": "3. Who may use it", + "train.project": "Project", + "train.share": "Who may use it", + "train.share.me": "Only me", + "train.share.project": "This project", + "train.share.team": "My team", + "train.share.org": "My organization", + "train.share.all": "Everyone", + "train.share.other": "Somewhere else", + "train.confirm": "This file holds nothing this group must not see.", + "train.go": "Start training", + "train.jobs": "Trainings", + "train.col.name": "Model", + "train.col.state": "State", + "train.col.share": "Who may use it", + "train.no_jobs": "No trainings yet.", + "train.after": "One training runs at a time in a project. It runs on a GPU with enough memory free; small models take minutes, large ones longer. You can leave the page: training goes on.", + "train.js.found": "{n} questions and answers found.", + "train.js.skipped": "{n} rows were skipped, with no question or no answer.", + "train.js.few": "That is few: at least 50 examples are advised.", + "train.js.no_file": "Choose the file of questions and answers first.", + "train.js.no_name": "Give the new model a name.", + "train.js.need_confirm": "To share with a wider group, confirm that the file holds nothing it must not see.", + "train.js.started": "Training is queued; it is listed below.", + "train.js.failed": "Could not start: {why}", + "train.js.queued": "Queued", + "train.js.waiting": "Queued: waiting for a GPU with enough memory free", + "train.js.preparing": "Preparing (fetching the base model)", + "train.js.running": "Training: {p}% ({m} min)", + "train.js.publishing": "Trained, publishing…", + "train.js.ready": "Ready: choose {name} in the chat", + "train.js.restarting": "Ready; {name} appears in the chat within a few minutes", + "train.js.not_published": "Trained, but not published: {why}", + "train.js.job_failed": "Failed: {why}", + "train.js.cancelled": "Stopped", + "train.js.delete": "Delete", + "train.js.delete_ask": "Delete {name}? It goes from the chat and the API.", + "train.js.share_ask": "It will be shared with a wider group. Do you confirm the training data holds nothing that group must not see?", + "train.js.try": "Try it in the chat", + "train.js.by": "started by {who}" +} diff --git a/internal/web/locales/tr/nav.json b/internal/web/locales/tr/nav.json index 8f51f9c..b5e099f 100644 --- a/internal/web/locales/tr/nav.json +++ b/internal/web/locales/tr/nav.json @@ -14,5 +14,6 @@ "nav.mcp": "MCP sunucuları", "nav.settings": "Ayarlar", "nav.images": "Görseller", - "nav.meshes": "3D" + "nav.meshes": "3D", + "nav.train": "Modeli özelleştir" } diff --git a/internal/web/locales/tr/train.json b/internal/web/locales/tr/train.json new file mode 100644 index 0000000..dd12a17 --- /dev/null +++ b/internal/web/locales/tr/train.json @@ -0,0 +1,55 @@ +{ + "train.title": "Modeli özelleştir", + "train.intro": "Bir modele kendi örneklerinizle nasıl cevap vereceğini öğretin: bir Excel dosyasında bir sütun soru, bir sütun cevap. Eğitim bitince yeni model sohbette ve API'de kendi adıyla çıkar; temel modelin çalıştığı kartta çalışır, ayrı bir kart gerekmez.", + "train.not_knowledge": "Model örneklerden üslubu ve cevap biçimini öğrenir; belgelerinizdeki bilgileri güvenilir biçimde ezberlemez. Modelin belgelerinize göre cevap vermesini istiyorsanız bilgi tabanını kullanın: sohbette belgelerden cevaplar, kaynağını da gösterir.", + "train.no_project": "Model özelleştirmek için bir projeye üye olmalısınız: eğitim bir proje adına yapılır ve maliyeti projeye yazılır. Yöneticinizden sizi bir projeye eklemesini isteyin.", + "train.no_models": "Şu anda özelleştirilebilecek bir model yok.", + "train.step.file": "1. Örnekler", + "train.file_hint": "Excel (.xlsx), CSV ya da JSONL. Başlıkları Soru ve Cevap olan sütunlar okunur; başlık yoksa ilk sütun soru, ikincisi cevaptır. En az 50, tercihen birkaç yüz örnek.", + "train.question": "Soru", + "train.answer": "Cevap", + "train.step.model": "2. Model", + "train.base": "Temel alınacak model", + "train.name": "Yeni modelin adı", + "train.name_hint": "sohbette bu adla seçilir: harf, rakam, - ve _", + "train.step.who": "3. Kimler kullanabilir", + "train.project": "Proje", + "train.share": "Kullanabilecekler", + "train.share.me": "Yalnız ben", + "train.share.project": "Bu proje", + "train.share.team": "Takımım", + "train.share.org": "Kuruluşum", + "train.share.all": "Herkes", + "train.share.other": "Başka bir yer", + "train.confirm": "Bu dosyada bu gruptan saklanması gereken bilgi yok.", + "train.go": "Eğitimi başlat", + "train.jobs": "Eğitimler", + "train.col.name": "Model", + "train.col.state": "Durum", + "train.col.share": "Kullanabilecekler", + "train.no_jobs": "Henüz bir eğitim yok.", + "train.after": "Bir projede aynı anda bir eğitim çalışır. Eğitim, boş belleği yeterli bir GPU'da yapılır; küçük modeller dakikalar, büyükler daha uzun sürer. Sayfadan ayrılabilirsiniz, eğitim sürer.", + "train.js.found": "{n} soru-cevap bulundu.", + "train.js.skipped": "{n} satırda soru ya da cevap boş olduğu için atlandı.", + "train.js.few": "Bu az: en az 50 örnek önerilir.", + "train.js.no_file": "Önce soru-cevap dosyasını seçin.", + "train.js.no_name": "Yeni modele bir ad verin.", + "train.js.need_confirm": "Daha geniş bir grupla paylaşmak için dosyada saklanması gereken bilgi olmadığını onaylayın.", + "train.js.started": "Eğitim sıraya alındı; durumu aşağıda.", + "train.js.failed": "Başlatılamadı: {why}", + "train.js.queued": "Sırada", + "train.js.waiting": "Sırada: boş belleği yeterli bir GPU bekleniyor", + "train.js.preparing": "Hazırlanıyor (temel model indiriliyor)", + "train.js.running": "Eğitiliyor: %{p} ({m} dk)", + "train.js.publishing": "Eğitildi, yayınlanıyor…", + "train.js.ready": "Hazır: sohbette {name} adıyla seçin", + "train.js.restarting": "Hazır, birkaç dakika içinde sohbette {name} adıyla çıkacak", + "train.js.not_published": "Eğitildi ama yayınlanamadı: {why}", + "train.js.job_failed": "Başarısız: {why}", + "train.js.cancelled": "Durduruldu", + "train.js.delete": "Sil", + "train.js.delete_ask": "{name} silinsin mi? Model sohbetten ve API'den kalkar.", + "train.js.share_ask": "Daha geniş bir grupla paylaşılacak. Eğitim verisinde bu gruptan saklanması gereken bilgi olmadığını onaylıyor musunuz?", + "train.js.try": "Sohbette dene", + "train.js.by": "{who} başlattı" +} diff --git a/internal/web/nav.go b/internal/web/nav.go index a0f9227..7017bb7 100644 --- a/internal/web/nav.go +++ b/internal/web/nav.go @@ -40,6 +40,8 @@ type Nav struct { // Images offers the pictures page, while some model makes them; Meshes // the 3D page. Images, Meshes bool + // Train offers the page that customises a model, while some model can be. + Train bool } type navKey struct{} @@ -58,6 +60,7 @@ func (p *AccountPages) navFor(r *http.Request, user store.User, current string) CSRF: csrfFor(r), Images: p.images != nil && p.images(), Meshes: p.meshes != nil && p.meshes(), + Train: p.train != nil && p.train(), } // Only when there is an organization to show: a link that leads to "there // are none" on every page is noise. diff --git a/internal/web/templates/nav.html b/internal/web/templates/nav.html index ced3c2b..ceff948 100644 --- a/internal/web/templates/nav.html +++ b/internal/web/templates/nav.html @@ -37,6 +37,9 @@ {{- if .Meshes}} {{t "nav.meshes"}} {{- end}} + {{- if .Train}} + {{t "nav.train"}} + {{- end}} {{t "nav.account"}} {{t "nav.docs"}}
diff --git a/internal/web/templates/train.html b/internal/web/templates/train.html new file mode 100644 index 0000000..cdf3d59 --- /dev/null +++ b/internal/web/templates/train.html @@ -0,0 +1,287 @@ +{{define "train"}} + + +{{t "train.title"}} · llmgw + + + +
+
+

{{t "train.title"}}

+

{{.User.Email}}

+ {{template "nav" .Nav}} +
+ +

{{t "train.intro"}}

+

{{th "train.not_knowledge"}}

+ +{{if not .Projects}} +

{{t "train.no_project"}}

+{{else if not .Models}} +

{{t "train.no_models"}}

+{{else}} + +
+

{{t "train.step.file"}}

+

{{t "train.file_hint"}}

+ +

+ + + +
+
+

{{t "train.step.model"}}

+ + +
+
+

{{t "train.step.who"}}

+ {{if gt (len .Projects) 1}} + + {{else}}{{with index .Projects 0}} +

{{t "train.project"}}: {{.Label}}

{{end}}{{end}} + + +
+
+

+ +{{end}} + +

{{t "train.jobs"}}

+ + + +
{{t "train.col.name"}}{{t "train.col.state"}}{{t "train.col.share"}}
+ +

{{t "train.after"}}

+ +
llmgw {{.Version}}{{stamp .Now}}{{template "langs"}}
+
+ + +{{end}} diff --git a/internal/web/train.go b/internal/web/train.go new file mode 100644 index 0000000..1b33451 --- /dev/null +++ b/internal/web/train.go @@ -0,0 +1,34 @@ +package web + +import ( + "net/http" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +type trainData struct { + Nav Nav + User store.User + Models []string + Projects []ChatContext + CSRF string + Version string + Now string +} + +// Train renders the page that customises a model: a file of questions and +// answers, a model, a button, and the trainings of the person's projects. +func Train(user store.User, models []string, projects []ChatContext, version string) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + noStore(w) + render(w, r, "train", trainData{ + Nav: navFrom(r, "train"), User: user, Models: models, Projects: projects, + CSRF: csrfFor(r), Version: version, Now: time.Now().UTC().Format(time.RFC3339), + }) + } +} + +// SetTrain has the menu offer the page while a model can be customised. +// Asked on every page, so it must be cheap. +func (p *AccountPages) SetTrain(available func() bool) { p.train = available } diff --git a/internal/web/train_test.go b/internal/web/train_test.go new file mode 100644 index 0000000..a7e1688 --- /dev/null +++ b/internal/web/train_test.go @@ -0,0 +1,49 @@ +package web + +import ( + "net/http/httptest" + "regexp" + "strings" + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/store" +) + +var untranslated = regexp.MustCompile(`train\.[a-z]`) + +func TestTheTrainingPageDrawsInEveryLanguage(t *testing.T) { + for _, lang := range []string{"en", "tr"} { + req := httptest.NewRequest("GET", "/train", nil) + req.Header.Set("Accept-Language", lang) + rec := httptest.NewRecorder() + Train(store.User{Email: "a@b.c"}, []string{"qwen2.5-7b"}, []ChatContext{{Value: "p1", Label: "Acme / hr / bot"}}, "v0")(rec, req) + if rec.Code != 200 { + t.Fatalf("%s: %d %s", lang, rec.Code, rec.Body.String()) + } + page := rec.Body.String() + for _, want := range []string{``, `id="file"`, `/train/start`, `value="p1"`, `Acme / hr / bot`} { + if !strings.Contains(page, want) { + t.Errorf("%s: %s missing", lang, want) + } + } + if untranslated.MatchString(page) { + t.Errorf("%s: an untranslated key is on the page", lang) + } + } + for name, page := range map[string]func() string{ + "no project": func() string { + rec := httptest.NewRecorder() + Train(store.User{}, []string{"q"}, nil, "v0")(rec, httptest.NewRequest("GET", "/train", nil)) + return rec.Body.String() + }, + "no model": func() string { + rec := httptest.NewRecorder() + Train(store.User{}, nil, []ChatContext{{Value: "p"}}, "v0")(rec, httptest.NewRequest("GET", "/train", nil)) + return rec.Body.String() + }, + } { + if p := page(); !strings.Contains(p, `class="empty"`) || strings.Contains(p, `id="start"`) { + t.Errorf("%s: the page offers training", name) + } + } +}