Ilker ManapandClaude Opus 5.5
79b3e9f887
A job names the catalog model the adapter is for, the safetensors weights to train on, the node and the MiB of a card it needs, and question/answer pairs. The control plane sends it when the node's agent is there and nothing else trains on it; the agent trains with its installed vLLM's torch and a pinned overlay of peft and accelerate (no second environment), as the engines' user, on the card with that much free — beside an engine, if it leaves that much — converts the adapter to GGUF with the installed llama.cpp's own conversion code when the model is served by llama.cpp, and sends what it made back in pieces, kept in the store to download. Progress, loss and peak card memory are reported as it goes; a node that goes fails its job. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
87 lines
2.9 KiB
Go
87 lines
2.9 KiB
Go
package adminapi
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"path/filepath"
|
|
"strings"
|
|
"testing"
|
|
|
|
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/control/train"
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
|
)
|
|
|
|
type noAgents struct{}
|
|
|
|
func (noAgents) Send(string, *llmgwv1.ControlMessage) error { return nil }
|
|
func (noAgents) Connected(string) bool { return false }
|
|
|
|
func TestATrainingJobIsCheckedBeforeItIsQueued(t *testing.T) {
|
|
models := append(testModels(), catalog.Model{Name: "small", Source: "/s",
|
|
Profiles: []catalog.Profile{{Name: "cpu", Engine: "llamacpp"}}})
|
|
a, _ := setup(t, models, gpuNode("cuda01", 1))
|
|
s, err := store.OpenSQLite(context.Background(), filepath.Join(t.TempDir(), "s.db"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer s.Close()
|
|
a.SetTrain(&TrainAdmin{Store: s, Service: train.New(s, noAgents{}, nil)})
|
|
mux := http.NewServeMux()
|
|
a.Register(mux)
|
|
post := func(body string) (int, string) {
|
|
rec := httptest.NewRecorder()
|
|
mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/train/jobs", strings.NewReader(body)))
|
|
return rec.Code, rec.Body.String()
|
|
}
|
|
good := map[string]any{"name": "tone", "model": "small", "base_repo": "Qwen/Qwen2.5-0.5B-Instruct", "node": "cuda01",
|
|
"gpu_memory_mib": 2300, "data": "{\"q\":\"a\",\"a\":\"b\"}\n{\"q\":\"c\",\"a\":\"d\"}\n", "params": map[string]any{"epochs": 3}}
|
|
with := func(k string, v any) string {
|
|
m := map[string]any{}
|
|
for kk, vv := range good {
|
|
m[kk] = vv
|
|
}
|
|
m[k] = v
|
|
b, _ := json.Marshal(m)
|
|
return string(b)
|
|
}
|
|
for _, c := range []struct {
|
|
field string
|
|
value any
|
|
says string
|
|
}{
|
|
{"name", "bad name", "letters, digits"},
|
|
{"model", "nope", "no model nope"},
|
|
{"name", "small", "already a model"},
|
|
{"base_repo", "Qwen", "owner/name"},
|
|
{"node", "gone", "no node gone"},
|
|
{"data", "{\"q\":\"a\"}\n", "no q or no a"},
|
|
{"data", "", "no pairs"},
|
|
{"params", map[string]any{"epochs": 0}, "params.epochs"},
|
|
{"params", map[string]any{"momentum": 0.9}, "params.momentum"},
|
|
} {
|
|
if code, body := post(with(c.field, c.value)); code != http.StatusBadRequest || !strings.Contains(body, c.says) {
|
|
t.Errorf("%s=%v: %d %s", c.field, c.value, code, body)
|
|
}
|
|
}
|
|
code, body := post(with("name", "tone"))
|
|
var v TrainJobView
|
|
_ = json.Unmarshal([]byte(body), &v)
|
|
if code != http.StatusCreated || v.Pairs != 2 || !v.GGUF || v.State != "queued" || v.Params != `{"epochs":3}` {
|
|
t.Fatalf("%d %s", code, body)
|
|
}
|
|
rec := httptest.NewRecorder()
|
|
mux.ServeHTTP(rec, httptest.NewRequest("DELETE", "/api/v1/train/jobs/"+v.ID, nil))
|
|
if rec.Code != http.StatusConflict {
|
|
t.Errorf("deleting a queued job: %d", rec.Code)
|
|
}
|
|
rec = httptest.NewRecorder()
|
|
mux.ServeHTTP(rec, httptest.NewRequest("GET", "/api/v1/train/jobs/"+v.ID+"/artifacts/peft", nil))
|
|
if rec.Code != http.StatusNotFound {
|
|
t.Errorf("an artifact not made: %d", rec.Code)
|
|
}
|
|
}
|