Training data from a collection: llmgw ctl kb dataset

A chat model of the cluster reads each chunk and writes questions a person
might ask of it with the answers it holds, streamed as NDJSON a chunk at a
time, for people to correct before an adapter is trained on it. A chunk that
gave no pairs says so on its own line.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 20:04:37 +02:00
1 parent 0a6a586dfe
commit ae95ac8261
8 files changed
+455

No files matched your search

@@ -0,0 +1,91 @@
package adminapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestPairsAreReadOutOfWhatAModelWrote(t *testing.T) {
completion := `{"choices":[{"message":{"content":"<think>let me see</think>\n{\"q\": \"Babalık izni kaç gün?\", \"a\": \"10 iş günü.\"},\nnot json\n{\"q\": \"\", \"a\": \"empty question\"}\n{\"q\": \"Ne zaman?\", \"a\": \"Doğumdan sonra.\"}"}}]}`
got := datasetPairs([]byte(completion))
if len(got) != 2 || got[0].Q != "Babalık izni kaç gün?" || got[1].A != "Doğumdan sonra." {
t.Errorf("pairs %+v", got)
}
if datasetPairs([]byte(`{"choices":[]}`)) != nil {
t.Error("pairs from no choice")
}
}
func TestADatasetIsWrittenChunkByChunk(t *testing.T) {
dsn := os.Getenv("LLMGW_TEST_POSTGRES")
if dsn == "" {
t.Skip("set LLMGW_TEST_POSTGRES to a pgvector database to run this")
}
ctx := context.Background()
s, err := store.OpenPostgres(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer s.Close()
if s.KnowledgeAvailable() != nil {
t.Skip("no pgvector")
}
models := append(kbModels(), catalog.Model{Name: "writer", Source: "/w",
Profiles: []catalog.Profile{{Name: "cpu", GPUClass: "h100-80g", TP: 1}}})
a, _ := setup(t, models)
a.SetKB(&KBAdmin{Store: s})
asked := 0
a.SetReplayer(func(_ context.Context, path, model string, body []byte) (int, []byte, time.Duration, error) {
asked++
if path != "/v1/chat/completions" || model != "writer" || !strings.Contains(string(body), "Write in the passage") {
t.Errorf("asked %s %s %s", path, model, body)
}
if strings.Contains(string(body), "bozuk") {
return 200, []byte(`{"choices":[{"message":{"content":"sorry"}}]}`), 0, nil
}
return 200, []byte(`{"choices":[{"message":{"content":"{\"q\":\"Soru?\",\"a\":\"Cevap.\"}"}}]}`), 0, nil
})
org, _ := s.CreateOrg(ctx, "ds-"+strings.ToLower(store.NewID("")))
c, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeOrg, OwnerID: org, Name: "d", EmbedModel: "bge-m3", Dims: 3})
if err != nil {
t.Fatal(err)
}
for _, d := range []string{"a.md", "b.md"} {
body := "iyi metin"
if d == "b.md" {
body = "bozuk metin"
}
if _, _, err := s.PutDocument(ctx, c, store.KBDocument{Name: d, Digest: d}, []store.KBChunk{{Body: body, Embedding: []float32{1, 0, 0}}}); err != nil {
t.Fatal(err)
}
}
mux := http.NewServeMux()
a.Register(mux)
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/kb/collections/"+c.ID+"/dataset", strings.NewReader(`{"model":"writer","per_chunk":3}`)))
var lines []KBDatasetLine
for _, l := range strings.Split(strings.TrimSpace(rec.Body.String()), "\n") {
var x KBDatasetLine
_ = json.Unmarshal([]byte(l), &x)
lines = append(lines, x)
}
if rec.Code != 200 || asked != 2 || len(lines) != 2 || lines[0].Q != "Soru?" || lines[0].Doc != "a.md" ||
lines[1].Doc != "b.md" || lines[1].Error == "" {
t.Errorf("%d asked %d: %+v", rec.Code, asked, lines)
}
// An embedding model writes no questions.
rec = httptest.NewRecorder()
mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/kb/collections/"+c.ID+"/dataset", strings.NewReader(`{"model":"bge-m3"}`)))
if rec.Code != http.StatusBadRequest {
t.Errorf("an embedding model: %d", rec.Code)
}
}