LoRA adapters trained on the nodes: llmgw ctl train
A job names the catalog model the adapter is for, the safetensors weights to train on, the node and the MiB of a card it needs, and question/answer pairs. The control plane sends it when the node's agent is there and nothing else trains on it; the agent trains with its installed vLLM's torch and a pinned overlay of peft and accelerate (no second environment), as the engines' user, on the card with that much free — beside an engine, if it leaves that much — converts the adapter to GGUF with the installed llama.cpp's own conversion code when the model is served by llama.cpp, and sends what it made back in pieces, kept in the store to download. Progress, loss and peak card memory are reported as it goes; a node that goes fails its job. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
ae95ac8261
commit
79b3e9f887
24 files changed
+3185
-72
No files matched your search
@@ -70,7 +70,9 @@ type API struct {
|
||||
// guard is the guardrail policies' library and attachments.
|
||||
guard *GuardAdmin
|
||||
// kb is knowledge bases; nil without a store.
|
||||
kb *KBAdmin
|
||||
kb *KBAdmin
|
||||
// train is adapter training; nil without a store and agents.
|
||||
train *TrainAdmin
|
||||
settings *Settings
|
||||
// restartAgents tells every connected agent to start again.
|
||||
restartAgents func(reason string) []string
|
||||
@@ -117,6 +119,7 @@ func (a *API) Register(mux Mux) {
|
||||
mux.HandleFunc("POST /api/v1/models/{name}/resources", a.setResources)
|
||||
a.registerQuotas(mux)
|
||||
a.registerKB(mux)
|
||||
a.registerTrain(mux)
|
||||
a.registerGuard(mux)
|
||||
a.registerSettings(mux)
|
||||
mux.HandleFunc("POST /api/v1/agents/restart", a.restartAgentsHandler)
|
||||
|
||||
@@ -67,6 +67,7 @@ var PathParams = map[string]string{
|
||||
"federation/{peer}": "The federation peer's name, as `llmgw ctl federation list` lists it.",
|
||||
"holds/{domain}": "The domain as level:name, rack:r12-07, as `llmgw ctl facility` lists it.",
|
||||
"collections/{id}": "The collection's ID, its name when that is unique, or ORG/NAME.",
|
||||
"artifacts/{kind}": "What the job made: peft, a gzipped tar of the PEFT adapter, or gguf, the adapter for llama.cpp.",
|
||||
}
|
||||
|
||||
// Routes is every endpoint Register mounts.
|
||||
@@ -295,6 +296,23 @@ var Routes = []Route{
|
||||
Description: "No tenancy applies: this is for checking what a collection holds and finds.",
|
||||
Request: KBSearchRequest{}, Response: []KBHitView{}},
|
||||
|
||||
// LoRA adapters trained on the nodes.
|
||||
{Method: "POST", Path: "/api/v1/train/jobs", Summary: "Queue a LoRA adapter's training on a node",
|
||||
Description: "The node trains when its agent is there, nothing else trains on it, and a card has gpu_memory_mib free; " +
|
||||
"the pairs are NDJSON of {\"q\", \"a\"}. An adapter learns how answers sound far better than what they say.",
|
||||
Request: TrainRequest{}, Response: TrainJobView{}, Status: http.StatusCreated},
|
||||
{Method: "GET", Path: "/api/v1/train/jobs", Summary: "Training jobs, newest first",
|
||||
Query: []Param{{Name: "limit", Description: "How many; 50 when absent.", Type: "integer"}},
|
||||
Response: []TrainJobView{}},
|
||||
{Method: "GET", Path: "/api/v1/train/jobs/{id}", Summary: "One training job: how it goes, and what it made",
|
||||
Response: TrainJobView{}},
|
||||
{Method: "POST", Path: "/api/v1/train/jobs/{id}/cancel", Summary: "Stop a training job",
|
||||
Response: TrainJobView{}},
|
||||
{Method: "DELETE", Path: "/api/v1/train/jobs/{id}", Summary: "Remove an ended training job and what it made",
|
||||
Response: map[string]any{}},
|
||||
{Method: "GET", Path: "/api/v1/train/jobs/{id}/artifacts/{kind}", Summary: "What a training job made: peft (a gzipped tar) or gguf",
|
||||
Response: map[string]any{}},
|
||||
|
||||
{Method: "POST", Path: "/api/v1/agents/restart", Summary: "Have every node's agent start again; their engines keep serving",
|
||||
Response: AgentRestartView{}},
|
||||
|
||||
|
||||
@@ -0,0 +1,338 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Training LoRA adapters (BACKLOG F4), administered: a job names the model
|
||||
// the adapter is for, the weights to train on, the node and how much of a
|
||||
// card it needs, and the pairs; the node trains when it can, and what it
|
||||
// made is kept here to download or publish.
|
||||
|
||||
// TrainStore is where jobs are kept.
|
||||
type TrainStore interface {
|
||||
CreateTrainJob(ctx context.Context, j store.TrainJob) error
|
||||
TrainJobs(ctx context.Context, limit int) ([]store.TrainJob, error)
|
||||
TrainJob(ctx context.Context, id string) (store.TrainJob, error)
|
||||
TrainArtifact(ctx context.Context, job, kind string) ([]byte, error)
|
||||
TrainArtifactKinds(ctx context.Context, job string) ([]string, error)
|
||||
DeleteTrainJob(ctx context.Context, id string) error
|
||||
}
|
||||
|
||||
// TrainAdmin is what the training endpoints need.
|
||||
type TrainAdmin struct {
|
||||
Store TrainStore
|
||||
Service *train.Service
|
||||
}
|
||||
|
||||
// SetTrain enables the training endpoints.
|
||||
func (a *API) SetTrain(t *TrainAdmin) { a.train = t }
|
||||
|
||||
// TrainRequest queues a job.
|
||||
type TrainRequest struct {
|
||||
// Name is the adapter's: what requests will ask for it by.
|
||||
Name string `json:"name"`
|
||||
// Model is the catalog model the adapter is for.
|
||||
Model string `json:"model"`
|
||||
// BaseRepo and BaseRevision are the weights trained on: the model's own
|
||||
// on its registry, in safetensors (Hugging Face), not a GGUF.
|
||||
BaseRepo string `json:"base_repo"`
|
||||
BaseRevision string `json:"base_revision,omitempty"`
|
||||
// Node trains it; GPUMemoryMiB is what it needs of a card, which can be
|
||||
// what an engine there leaves free.
|
||||
Node string `json:"node"`
|
||||
GPUMemoryMiB int `json:"gpu_memory_mib"`
|
||||
// Data is the pairs, NDJSON of {"q", "a"} (llmgw ctl kb dataset).
|
||||
Data string `json:"data"`
|
||||
// Params tune the trainer: rank, alpha, dropout, epochs, lr, batch,
|
||||
// max_len, seed.
|
||||
Params map[string]any `json:"params,omitempty"`
|
||||
// GGUF also makes the adapter in llama.cpp's format; when absent, it
|
||||
// does when the model is served by llama.cpp.
|
||||
GGUF *bool `json:"gguf,omitempty"`
|
||||
}
|
||||
|
||||
// TrainJobView is a job as listed.
|
||||
type TrainJobView struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Model string `json:"model"`
|
||||
BaseRepo string `json:"base_repo"`
|
||||
BaseRevision string `json:"base_revision,omitempty"`
|
||||
Node string `json:"node"`
|
||||
GPUMemoryMiB int `json:"gpu_memory_mib"`
|
||||
Params string `json:"params,omitempty"`
|
||||
GGUF bool `json:"gguf"`
|
||||
Pairs int `json:"pairs"`
|
||||
State string `json:"state"`
|
||||
Step int `json:"step"`
|
||||
Steps int `json:"steps"`
|
||||
Loss float64 `json:"loss"`
|
||||
Message string `json:"message,omitempty"`
|
||||
PeakMiB int `json:"peak_mib"`
|
||||
CreatedAt time.Time `json:"created_at"`
|
||||
StartedAt time.Time `json:"started_at,omitzero"`
|
||||
FinishedAt time.Time `json:"finished_at,omitzero"`
|
||||
CreatedBy string `json:"created_by,omitempty"`
|
||||
// Artifacts are what it made: peft (a gzipped tar), gguf.
|
||||
Artifacts []string `json:"artifacts,omitempty"`
|
||||
}
|
||||
|
||||
func trainView(j store.TrainJob) TrainJobView {
|
||||
return TrainJobView{ID: j.ID, Name: j.Name, Model: j.Model, BaseRepo: j.BaseRepo, BaseRevision: j.BaseRevision,
|
||||
Node: j.Node, GPUMemoryMiB: j.GPUMemoryMiB, Params: j.Params, GGUF: j.GGUF, Pairs: j.Pairs, State: j.State,
|
||||
Step: j.Step, Steps: j.Steps, Loss: j.Loss, Message: j.Message, PeakMiB: j.PeakMiB, CreatedAt: j.CreatedAt,
|
||||
StartedAt: j.StartedAt, FinishedAt: j.FinishedAt, CreatedBy: j.CreatedBy}
|
||||
}
|
||||
|
||||
func (a *API) registerTrain(mux Mux) {
|
||||
mux.HandleFunc("POST /api/v1/train/jobs", a.trainCreate)
|
||||
mux.HandleFunc("GET /api/v1/train/jobs", a.trainList)
|
||||
mux.HandleFunc("GET /api/v1/train/jobs/{id}", a.trainGet)
|
||||
mux.HandleFunc("POST /api/v1/train/jobs/{id}/cancel", a.trainCancel)
|
||||
mux.HandleFunc("DELETE /api/v1/train/jobs/{id}", a.trainDelete)
|
||||
mux.HandleFunc("GET /api/v1/train/jobs/{id}/artifacts/{kind}", a.trainArtifact)
|
||||
}
|
||||
|
||||
func (a *API) trainReady(w http.ResponseWriter) bool {
|
||||
if a.train == nil {
|
||||
writeError(w, http.StatusNotImplemented, "training needs the store and the nodes' agents, which this control plane runs without")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// trainMaxData bounds a job's pairs.
|
||||
const trainMaxData = 64 << 20
|
||||
|
||||
var adapterName = regexp.MustCompile(`^[A-Za-z0-9][A-Za-z0-9._-]{0,99}$`)
|
||||
|
||||
// trainParams are the trainer's settings, with what each may be.
|
||||
var trainParams = map[string][2]float64{
|
||||
"rank": {1, 256}, "alpha": {1, 1024}, "dropout": {0, 0.9}, "epochs": {1, 100},
|
||||
"lr": {1e-7, 1e-2}, "batch": {1, 128}, "max_len": {16, 32768}, "seed": {0, 1 << 31},
|
||||
}
|
||||
|
||||
func (a *API) trainCreate(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.trainReady(w) {
|
||||
return
|
||||
}
|
||||
var in TrainRequest
|
||||
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, trainMaxData+1<<20)).Decode(&in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
bad := func(format string, args ...any) {
|
||||
writeError(w, http.StatusBadRequest, fmt.Sprintf(format, args...))
|
||||
}
|
||||
if !adapterName.MatchString(in.Name) {
|
||||
bad("an adapter's name is letters, digits, . _ and -, up to 100: %q is not", in.Name)
|
||||
return
|
||||
}
|
||||
m, ok := a.ctl.Catalog().Resolve(in.Model)
|
||||
if !ok {
|
||||
bad("no model %s in the catalog", in.Model)
|
||||
return
|
||||
}
|
||||
if _, _, _, taken := a.ctl.Catalog().ResolveAdapter(in.Name); taken || in.Name == m.Name {
|
||||
bad("%s is already a model's or an adapter's name", in.Name)
|
||||
return
|
||||
}
|
||||
if strings.Count(in.BaseRepo, "/") != 1 || strings.ContainsAny(in.BaseRepo, " :") {
|
||||
bad("base_repo is the weights' registry repository, owner/name: %q is not", in.BaseRepo)
|
||||
return
|
||||
}
|
||||
if _, ok := a.ctl.Inventory().Get(in.Node); !ok {
|
||||
bad("no node %s", in.Node)
|
||||
return
|
||||
}
|
||||
if in.GPUMemoryMiB <= 0 {
|
||||
bad("gpu_memory_mib is what the job needs of a card, in MiB")
|
||||
return
|
||||
}
|
||||
pairs, err := countPairs(in.Data)
|
||||
if err != nil {
|
||||
bad("%v", err)
|
||||
return
|
||||
}
|
||||
for k, v := range in.Params {
|
||||
lim, known := trainParams[k]
|
||||
f, isNum := v.(float64)
|
||||
if !known || !isNum || f < lim[0] || f > lim[1] {
|
||||
bad("params.%s: the trainer takes rank, alpha, dropout, epochs, lr, batch, max_len and seed, each a number in its range", k)
|
||||
return
|
||||
}
|
||||
}
|
||||
params := ""
|
||||
if len(in.Params) > 0 {
|
||||
b, _ := json.Marshal(in.Params)
|
||||
params = string(b)
|
||||
}
|
||||
gguf := false
|
||||
for _, p := range m.Profiles {
|
||||
if p.Engine == "llamacpp" {
|
||||
gguf = true
|
||||
}
|
||||
}
|
||||
if in.GGUF != nil {
|
||||
gguf = *in.GGUF
|
||||
}
|
||||
j := store.TrainJob{ID: store.NewID("train-"), Name: in.Name, Model: m.Name, BaseRepo: in.BaseRepo,
|
||||
BaseRevision: in.BaseRevision, Node: in.Node, GPUMemoryMiB: in.GPUMemoryMiB, Params: params, GGUF: gguf,
|
||||
Data: []byte(in.Data), Pairs: pairs, CreatedBy: actorOf(r)}
|
||||
if err := a.train.Store.CreateTrainJob(r.Context(), j); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
// At once rather than at the next look, when the node is here.
|
||||
go a.train.Service.Dispatch(context.WithoutCancel(r.Context()))
|
||||
j.State = store.TrainQueued
|
||||
writeJSON(w, http.StatusCreated, trainView(j))
|
||||
}
|
||||
|
||||
// countPairs checks a job's data: NDJSON of {"q", "a"}, at least one.
|
||||
func countPairs(data string) (int, error) {
|
||||
if len(data) > trainMaxData {
|
||||
return 0, fmt.Errorf("the data is over %d MiB", trainMaxData>>20)
|
||||
}
|
||||
n := 0
|
||||
sc := bufio.NewScanner(strings.NewReader(data))
|
||||
sc.Buffer(make([]byte, 64<<10), 1<<20)
|
||||
line := 0
|
||||
for sc.Scan() {
|
||||
line++
|
||||
b := bytes.TrimSpace(sc.Bytes())
|
||||
if len(b) == 0 {
|
||||
continue
|
||||
}
|
||||
var p struct{ Q, A string }
|
||||
if err := json.Unmarshal(b, &p); err != nil {
|
||||
return 0, fmt.Errorf("data line %d is not JSON: %v", line, err)
|
||||
}
|
||||
if strings.TrimSpace(p.Q) == "" || strings.TrimSpace(p.A) == "" {
|
||||
return 0, fmt.Errorf("data line %d has no q or no a", line)
|
||||
}
|
||||
n++
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if n == 0 {
|
||||
return 0, errors.New("the data holds no pairs: NDJSON of {\"q\", \"a\"}, as llmgw ctl kb dataset writes")
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (a *API) trainList(w http.ResponseWriter, r *http.Request) {
|
||||
if !a.trainReady(w) {
|
||||
return
|
||||
}
|
||||
limit := 50
|
||||
if n, err := strconv.Atoi(r.URL.Query().Get("limit")); err == nil && n > 0 && n <= 500 {
|
||||
limit = n
|
||||
}
|
||||
jobs, err := a.train.Store.TrainJobs(r.Context(), limit)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
out := []TrainJobView{}
|
||||
for _, j := range jobs {
|
||||
out = append(out, trainView(j))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, out)
|
||||
}
|
||||
|
||||
func (a *API) trainJob(w http.ResponseWriter, r *http.Request) (store.TrainJob, bool) {
|
||||
if !a.trainReady(w) {
|
||||
return store.TrainJob{}, false
|
||||
}
|
||||
j, err := a.train.Store.TrainJob(r.Context(), r.PathValue("id"))
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
writeError(w, http.StatusNotFound, "no training job "+r.PathValue("id"))
|
||||
return j, false
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return j, false
|
||||
}
|
||||
return j, true
|
||||
}
|
||||
|
||||
func (a *API) trainGet(w http.ResponseWriter, r *http.Request) {
|
||||
j, ok := a.trainJob(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
v := trainView(j)
|
||||
v.Artifacts, _ = a.train.Store.TrainArtifactKinds(r.Context(), j.ID)
|
||||
writeJSON(w, http.StatusOK, v)
|
||||
}
|
||||
|
||||
func (a *API) trainCancel(w http.ResponseWriter, r *http.Request) {
|
||||
j, ok := a.trainJob(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if err := a.train.Service.Cancel(r.Context(), j.ID); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
j, _ = a.train.Store.TrainJob(r.Context(), j.ID)
|
||||
writeJSON(w, http.StatusOK, trainView(j))
|
||||
}
|
||||
|
||||
func (a *API) trainDelete(w http.ResponseWriter, r *http.Request) {
|
||||
j, ok := a.trainJob(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if !store.TrainOver(j.State) {
|
||||
writeError(w, http.StatusConflict, "the job has not ended: cancel it first")
|
||||
return
|
||||
}
|
||||
if err := a.train.Store.DeleteTrainJob(r.Context(), j.ID); err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"id": j.ID, "deleted": true})
|
||||
}
|
||||
|
||||
func (a *API) trainArtifact(w http.ResponseWriter, r *http.Request) {
|
||||
j, ok := a.trainJob(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
kind := r.PathValue("kind")
|
||||
b, err := a.train.Store.TrainArtifact(r.Context(), j.ID, kind)
|
||||
if errors.Is(err, store.ErrNotFound) {
|
||||
writeError(w, http.StatusNotFound, "the job made no "+kind)
|
||||
return
|
||||
}
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
name := j.Name + ".tar.gz"
|
||||
w.Header().Set("Content-Type", "application/gzip")
|
||||
if kind == "gguf" {
|
||||
name = j.Name + ".gguf"
|
||||
w.Header().Set("Content-Type", "application/octet-stream")
|
||||
}
|
||||
w.Header().Set("Content-Disposition", `attachment; filename="`+name+`"`)
|
||||
w.Header().Set("Content-Length", strconv.Itoa(len(b)))
|
||||
_, _ = w.Write(b)
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
type noAgents struct{}
|
||||
|
||||
func (noAgents) Send(string, *llmgwv1.ControlMessage) error { return nil }
|
||||
func (noAgents) Connected(string) bool { return false }
|
||||
|
||||
func TestATrainingJobIsCheckedBeforeItIsQueued(t *testing.T) {
|
||||
models := append(testModels(), catalog.Model{Name: "small", Source: "/s",
|
||||
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
|
||||
a, _ := setup(t, models, gpuNode("cuda01", 1))
|
||||
s, err := store.OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "s.db"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
a.SetTrain(&TrainAdmin{Store: s, Service: train.New(s, noAgents{}, nil)})
|
||||
mux := http.NewServeMux()
|
||||
a.Register(mux)
|
||||
post := func(body string) (int, string) {
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/train/jobs", strings.NewReader(body)))
|
||||
return rec.Code, rec.Body.String()
|
||||
}
|
||||
good := map[string]any{"name": "tone", "model": "small", "base_repo": "Qwen/Qwen2.5-0.5B-Instruct", "node": "cuda01",
|
||||
"gpu_memory_mib": 2300, "data": "{\"q\":\"a\",\"a\":\"b\"}\n{\"q\":\"c\",\"a\":\"d\"}\n", "params": map[string]any{"epochs": 3}}
|
||||
with := func(k string, v any) string {
|
||||
m := map[string]any{}
|
||||
for kk, vv := range good {
|
||||
m[kk] = vv
|
||||
}
|
||||
m[k] = v
|
||||
b, _ := json.Marshal(m)
|
||||
return string(b)
|
||||
}
|
||||
for _, c := range []struct {
|
||||
field string
|
||||
value any
|
||||
says string
|
||||
}{
|
||||
{"name", "bad name", "letters, digits"},
|
||||
{"model", "nope", "no model nope"},
|
||||
{"name", "small", "already a model"},
|
||||
{"base_repo", "Qwen", "owner/name"},
|
||||
{"node", "gone", "no node gone"},
|
||||
{"data", "{\"q\":\"a\"}\n", "no q or no a"},
|
||||
{"data", "", "no pairs"},
|
||||
{"params", map[string]any{"epochs": 0}, "params.epochs"},
|
||||
{"params", map[string]any{"momentum": 0.9}, "params.momentum"},
|
||||
} {
|
||||
if code, body := post(with(c.field, c.value)); code != http.StatusBadRequest || !strings.Contains(body, c.says) {
|
||||
t.Errorf("%s=%v: %d %s", c.field, c.value, code, body)
|
||||
}
|
||||
}
|
||||
code, body := post(with("name", "tone"))
|
||||
var v TrainJobView
|
||||
_ = json.Unmarshal([]byte(body), &v)
|
||||
if code != http.StatusCreated || v.Pairs != 2 || !v.GGUF || v.State != "queued" || v.Params != `{"epochs":3}` {
|
||||
t.Fatalf("%d %s", code, body)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, httptest.NewRequest("DELETE", "/api/v1/train/jobs/"+v.ID, nil))
|
||||
if rec.Code != http.StatusConflict {
|
||||
t.Errorf("deleting a queued job: %d", rec.Code)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, httptest.NewRequest("GET", "/api/v1/train/jobs/"+v.ID+"/artifacts/peft", nil))
|
||||
if rec.Code != http.StatusNotFound {
|
||||
t.Errorf("an artifact not made: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user