Training data from a collection: llmgw ctl kb dataset

A chat model of the cluster reads each chunk and writes questions a person
might ask of it with the answers it holds, streamed as NDJSON a chunk at a
time, for people to correct before an adapter is trained on it. A chunk that
gave no pairs says so on its own line.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 20:04:37 +02:00
1 parent 0a6a586dfe
commit ae95ac8261
8 files changed
+455

No files matched your search

+2
View File
@@ -30,6 +30,7 @@ type KBStore interface {
DeleteDocument(ctx context.Context, collection, name string) (bool, error)
SearchChunks(ctx context.Context, q store.KBSearch) ([]store.KBHit, error)
SetKBDefault(ctx context.Context, d store.KBDefault) error
CollectionChunks(ctx context.Context, c store.KBCollection, limit int) ([]store.KBHit, error)
KBDefaults(ctx context.Context) ([]store.KBDefault, error)
}
@@ -161,6 +162,7 @@ func (a *API) registerKB(mux Mux) {
mux.HandleFunc("DELETE /api/v1/kb/grants", a.kbGrant(false))
mux.HandleFunc("POST /api/v1/kb/search", a.kbSearch)
mux.HandleFunc("POST /api/v1/kb/defaults", a.kbSetDefault)
mux.HandleFunc("POST /api/v1/kb/collections/{id}/dataset", a.kbDataset)
}
// kbReady answers for a gateway without knowledge bases, and says whether
+144
View File
@@ -0,0 +1,144 @@
package adminapi
import (
"encoding/json"
"fmt"
"net/http"
"regexp"
"strings"
)
// Training data from a collection (BACKLOG F4): questions a person might
// ask of each chunk and the answers it holds, written by a model of the
// cluster, for people to read through before an adapter is trained on them.
// What an adapter learns well is how the organisation's answers sound; what
// they say is still the knowledge base's to give.
// KBDatasetRequest asks for training data from a collection.
type KBDatasetRequest struct {
// Model writes the questions and answers: a chat model of the catalog.
Model string `json:"model"`
// PerChunk is how many pairs each chunk gives; 8 when 0.
PerChunk int `json:"per_chunk,omitempty"`
// MaxChunks bounds how many chunks are read; 500 when 0.
MaxChunks int `json:"max_chunks,omitempty"`
}
// KBDatasetLine is one line of the answer: a pair, or why a chunk gave none.
type KBDatasetLine struct {
Q string `json:"q,omitempty"`
A string `json:"a,omitempty"`
Doc string `json:"doc"`
Ord int `json:"ord"`
Error string `json:"error,omitempty"`
}
const kbDatasetPrompt = `From the passage of an organisation's document below, write %d different questions an employee might ask, each with a short, correct answer.
Rules: use only what the passage says. Phrase the questions differently from each other (formal, everyday, short, indirect). Answers are one or two sentences and keep the passage's numbers exactly. Leave out e-mail addresses and people's names. Write in the passage's own language.
Output: one JSON object per line, {"q": "...", "a": "..."}, and nothing else.
DOCUMENT %s:
%s`
var thinkBlock = regexp.MustCompile(`(?s)<think>.*?</think>`)
func (a *API) kbDataset(w http.ResponseWriter, r *http.Request) {
c, ok := a.kbCollection(w, r)
if !ok {
return
}
if a.replay == nil {
writeError(w, http.StatusNotImplemented, "no router here to ask a model")
return
}
var in KBDatasetRequest
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
m, found := a.ctl.Catalog().Resolve(in.Model)
if !found {
writeError(w, http.StatusBadRequest, "no model "+in.Model+" in the catalog")
return
}
if m.Capabilities.Embeddings {
writeError(w, http.StatusBadRequest, m.Name+" embeds; questions are written by a chat model")
return
}
per := in.PerChunk
if per <= 0 || per > 30 {
per = 8
}
limit := in.MaxChunks
if limit <= 0 || limit > 5000 {
limit = 500
}
chunks, err := a.kb.Store.CollectionChunks(r.Context(), c, limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/x-ndjson")
w.WriteHeader(http.StatusOK)
enc := json.NewEncoder(w)
flush := func() {
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}
for _, ch := range chunks {
if r.Context().Err() != nil {
return
}
body, _ := json.Marshal(map[string]any{"model": m.Name, "temperature": 0.7, "max_tokens": 200 * per,
"messages": []map[string]string{{"role": "user", "content": fmt.Sprintf(kbDatasetPrompt, per, ch.Document, ch.Body)}}})
status, out, _, err := a.replay(r.Context(), "/v1/chat/completions", m.Name, body)
line := KBDatasetLine{Doc: ch.Document, Ord: ch.Ord}
if err == nil && status != http.StatusOK {
err = fmt.Errorf("%d: %s", status, strings.TrimSpace(string(out)))
}
if err != nil {
line.Error = err.Error()
_ = enc.Encode(line)
flush()
continue
}
pairs := datasetPairs(out)
if len(pairs) == 0 {
line.Error = "the model gave no pairs it could read"
_ = enc.Encode(line)
}
for _, p := range pairs {
p.Doc, p.Ord = ch.Document, ch.Ord
_ = enc.Encode(p)
}
flush()
}
}
// datasetPairs reads the {"q","a"} lines out of a chat completion.
func datasetPairs(completion []byte) []KBDatasetLine {
var res struct {
Choices []struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
} `json:"choices"`
}
if json.Unmarshal(completion, &res) != nil || len(res.Choices) == 0 {
return nil
}
text := thinkBlock.ReplaceAllString(res.Choices[0].Message.Content, "")
var out []KBDatasetLine
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSuffix(strings.TrimSpace(line), ",")
if !strings.HasPrefix(line, "{") || !strings.HasSuffix(line, "}") {
continue
}
var p struct{ Q, A string }
if json.Unmarshal([]byte(line), &p) == nil && strings.TrimSpace(p.Q) != "" && strings.TrimSpace(p.A) != "" {
out = append(out, KBDatasetLine{Q: strings.TrimSpace(p.Q), A: strings.TrimSpace(p.A)})
}
}
return out
}
@@ -0,0 +1,91 @@
package adminapi
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestPairsAreReadOutOfWhatAModelWrote(t *testing.T) {
completion := `{"choices":[{"message":{"content":"<think>let me see</think>\n{\"q\": \"Babalık izni kaç gün?\", \"a\": \"10 iş günü.\"},\nnot json\n{\"q\": \"\", \"a\": \"empty question\"}\n{\"q\": \"Ne zaman?\", \"a\": \"Doğumdan sonra.\"}"}}]}`
got := datasetPairs([]byte(completion))
if len(got) != 2 || got[0].Q != "Babalık izni kaç gün?" || got[1].A != "Doğumdan sonra." {
t.Errorf("pairs %+v", got)
}
if datasetPairs([]byte(`{"choices":[]}`)) != nil {
t.Error("pairs from no choice")
}
}
func TestADatasetIsWrittenChunkByChunk(t *testing.T) {
dsn := os.Getenv("LLMGW_TEST_POSTGRES")
if dsn == "" {
t.Skip("set LLMGW_TEST_POSTGRES to a pgvector database to run this")
}
ctx := context.Background()
s, err := store.OpenPostgres(ctx, dsn)
if err != nil {
t.Fatal(err)
}
defer s.Close()
if s.KnowledgeAvailable() != nil {
t.Skip("no pgvector")
}
models := append(kbModels(), catalog.Model{Name: "writer", Source: "/w",
Profiles: []catalog.Profile{{Name: "cpu", GPUClass: "h100-80g", TP: 1}}})
a, _ := setup(t, models)
a.SetKB(&KBAdmin{Store: s})
asked := 0
a.SetReplayer(func(_ context.Context, path, model string, body []byte) (int, []byte, time.Duration, error) {
asked++
if path != "/v1/chat/completions" || model != "writer" || !strings.Contains(string(body), "Write in the passage") {
t.Errorf("asked %s %s %s", path, model, body)
}
if strings.Contains(string(body), "bozuk") {
return 200, []byte(`{"choices":[{"message":{"content":"sorry"}}]}`), 0, nil
}
return 200, []byte(`{"choices":[{"message":{"content":"{\"q\":\"Soru?\",\"a\":\"Cevap.\"}"}}]}`), 0, nil
})
org, _ := s.CreateOrg(ctx, "ds-"+strings.ToLower(store.NewID("")))
c, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeOrg, OwnerID: org, Name: "d", EmbedModel: "bge-m3", Dims: 3})
if err != nil {
t.Fatal(err)
}
for _, d := range []string{"a.md", "b.md"} {
body := "iyi metin"
if d == "b.md" {
body = "bozuk metin"
}
if _, _, err := s.PutDocument(ctx, c, store.KBDocument{Name: d, Digest: d}, []store.KBChunk{{Body: body, Embedding: []float32{1, 0, 0}}}); err != nil {
t.Fatal(err)
}
}
mux := http.NewServeMux()
a.Register(mux)
rec := httptest.NewRecorder()
mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/kb/collections/"+c.ID+"/dataset", strings.NewReader(`{"model":"writer","per_chunk":3}`)))
var lines []KBDatasetLine
for _, l := range strings.Split(strings.TrimSpace(rec.Body.String()), "\n") {
var x KBDatasetLine
_ = json.Unmarshal([]byte(l), &x)
lines = append(lines, x)
}
if rec.Code != 200 || asked != 2 || len(lines) != 2 || lines[0].Q != "Soru?" || lines[0].Doc != "a.md" ||
lines[1].Doc != "b.md" || lines[1].Error == "" {
t.Errorf("%d asked %d: %+v", rec.Code, asked, lines)
}
// An embedding model writes no questions.
rec = httptest.NewRecorder()
mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/kb/collections/"+c.ID+"/dataset", strings.NewReader(`{"model":"bge-m3"}`)))
if rec.Code != http.StatusBadRequest {
t.Errorf("an embedding model: %d", rec.Code)
}
}
+3
View File
@@ -288,6 +288,9 @@ var Routes = []Route{
Description: "On an organization, team, project or key; the narrowest one with a default wins. No collections clears it. " +
"A request's own X-Llmgw-Knowledge overrides it, and X-Llmgw-Knowledge: none turns it off.",
Request: KBDefaultRequest{}, Response: KBDefaultView{}},
{Method: "POST", Path: "/api/v1/kb/collections/{id}/dataset", Summary: "Training data from a collection: questions and answers a chat model writes from each chunk",
Description: "NDJSON, a line per pair as each chunk is read, or a line with error for a chunk that gave none. For people to correct before an adapter is trained on it.",
Request: KBDatasetRequest{}, Response: KBDatasetLine{}},
{Method: "POST", Path: "/api/v1/kb/search", Summary: "Try a search of a collection, as an administrator",
Description: "No tenancy applies: this is for checking what a collection holds and finds.",
Request: KBSearchRequest{}, Response: []KBHitView{}},