diff --git a/docs/admin-api.openapi.json b/docs/admin-api.openapi.json
index 398f890..8b24cfb 100644
--- a/docs/admin-api.openapi.json
+++ b/docs/admin-api.openapi.json
@@ -1185,6 +1185,58 @@
}
}
},
+ "/api/v1/kb/collections/{id}/dataset": {
+ "post": {
+ "operationId": "postKbCollectionsIdDataset",
+ "summary": "Training data from a collection: questions and answers a chat model writes from each chunk",
+ "description": "NDJSON, a line per pair as each chunk is read, or a line with error for a chunk that gave none. For people to correct before an adapter is trained on it.",
+ "tags": [
+ "kb"
+ ],
+ "parameters": [
+ {
+ "name": "id",
+ "in": "path",
+ "required": true,
+ "description": "The collection's ID, its name when that is unique, or ORG/NAME.",
+ "schema": {
+ "type": "string"
+ }
+ },
+ {
+ "$ref": "#/components/parameters/Actor"
+ }
+ ],
+ "requestBody": {
+ "required": true,
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/KBDatasetRequest"
+ }
+ }
+ }
+ },
+ "responses": {
+ "200": {
+ "description": "OK",
+ "content": {
+ "application/json": {
+ "schema": {
+ "$ref": "#/components/schemas/KBDatasetLine"
+ }
+ }
+ }
+ },
+ "401": {
+ "$ref": "#/components/responses/Unauthorized"
+ },
+ "default": {
+ "$ref": "#/components/responses/Error"
+ }
+ }
+ }
+ },
"/api/v1/kb/collections/{id}/documents": {
"get": {
"operationId": "getKbCollectionsIdDocuments",
@@ -5059,6 +5111,45 @@
}
}
},
+ "KBDatasetLine": {
+ "type": "object",
+ "description": "KBDatasetLine is one line of the answer: a pair, or why a chunk gave none.",
+ "properties": {
+ "q": {
+ "type": "string"
+ },
+ "a": {
+ "type": "string"
+ },
+ "doc": {
+ "type": "string"
+ },
+ "ord": {
+ "type": "integer"
+ },
+ "error": {
+ "type": "string"
+ }
+ }
+ },
+ "KBDatasetRequest": {
+ "type": "object",
+ "description": "KBDatasetRequest asks for training data from a collection.",
+ "properties": {
+ "model": {
+ "type": "string",
+ "description": "Model writes the questions and answers: a chat model of the catalog."
+ },
+ "per_chunk": {
+ "type": "integer",
+ "description": "PerChunk is how many pairs each chunk gives; 8 when 0."
+ },
+ "max_chunks": {
+ "type": "integer",
+ "description": "MaxChunks bounds how many chunks are read; 500 when 0."
+ }
+ }
+ },
"KBDefaultRequest": {
"type": "object",
"description": "KBDefaultRequest sets the collections a part of the tree answers from when a request names none; none clears it.",
diff --git a/internal/cli/ctl.go b/internal/cli/ctl.go
index 8474b00..0f51844 100644
--- a/internal/cli/ctl.go
+++ b/internal/cli/ctl.go
@@ -116,6 +116,11 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
if out == nil {
return nil
}
+ // A stream, as it comes: NDJSON a long run writes line by line.
+ if w, ok := out.(io.Writer); ok {
+ _, err := io.Copy(w, resp.Body)
+ return err
+ }
// An older control plane answers a path it does not know with its
// dashboard: a page, which read as JSON said only "invalid character '<'".
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
diff --git a/internal/cli/kb.go b/internal/cli/kb.go
index 02bee7f..4bfc169 100644
--- a/internal/cli/kb.go
+++ b/internal/cli/kb.go
@@ -1,10 +1,14 @@
package cli
import (
+ "bytes"
"encoding/json"
"fmt"
+ "io"
"net/url"
+ "os"
"strings"
+ "time"
"github.com/spf13/cobra"
@@ -212,6 +216,45 @@ func newKBCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
def.Flags().BoolVar(&clear, "clear", false, "no default any more")
c.AddCommand(def)
+ var dsModel, dsOut string
+ var dsPer, dsMax int
+ dataset := &cobra.Command{
+ Use: "dataset COLLECTION --model MODEL",
+ Short: "Write training data from a collection: questions and answers, for an adapter",
+ Long: "A chat model of the cluster reads each chunk of the collection and writes\n" +
+ "questions a person might ask of it, with the answers it holds, one JSON object\n" +
+ "a line ({\"q\", \"a\", \"doc\", \"ord\"}). Read it through and correct it before\n" +
+ "training on it (llmgw ctl train): what the model got wrong, an adapter learns.\n\n" +
+ "An adapter learns how answers sound — the organisation's words, format and\n" +
+ "tone — far better than what they say; the facts stay the knowledge base's.",
+ Example: " llmgw ctl kb dataset Acme/policies --model qwen2.5-coder-14b-instruct --out policies.jsonl",
+ Args: cobra.ExactArgs(1),
+ RunE: func(cmd *cobra.Command, args []string) error {
+ w := cmd.OutOrStdout()
+ if dsOut != "" {
+ f, err := os.Create(dsOut)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ w = f
+ }
+ lw := &datasetWriter{pairs: w, problems: cmd.ErrOrStderr()}
+ body, _ := json.Marshal(adminapi.KBDatasetRequest{Model: dsModel, PerChunk: dsPer, MaxChunks: dsMax})
+ err := client().patient(6*time.Hour).do(cmd.Context(), "POST",
+ "/api/v1/kb/collections/"+url.PathEscape(args[0])+"/dataset", json.RawMessage(body), lw)
+ lw.flush()
+ fmt.Fprintf(cmd.ErrOrStderr(), "%d pairs from %d chunks; %d chunks gave none\n", lw.n, len(lw.chunks), lw.failed)
+ return err
+ },
+ }
+ dataset.Flags().StringVar(&dsModel, "model", "", "the chat model that writes the questions and answers")
+ dataset.Flags().StringVar(&dsOut, "out", "", "the file to write (default: standard output)")
+ dataset.Flags().IntVar(&dsPer, "per-chunk", 8, "pairs from each chunk")
+ dataset.Flags().IntVar(&dsMax, "max-chunks", 500, "the most chunks read")
+ _ = dataset.MarkFlagRequired("model")
+ c.AddCommand(dataset)
+
var k int
search := &cobra.Command{
Use: "search COLLECTION QUERY...",
@@ -240,3 +283,50 @@ func newKBCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
c.AddCommand(search)
return c
}
+
+// datasetWriter splits the dataset stream: pairs to the file, problems to
+// the terminal, counted as they pass.
+type datasetWriter struct {
+ pairs, problems io.Writer
+ buf []byte
+ n, failed int
+ chunks map[string]bool
+}
+
+func (d *datasetWriter) Write(p []byte) (int, error) {
+ d.buf = append(d.buf, p...)
+ for {
+ i := bytes.IndexByte(d.buf, '\n')
+ if i < 0 {
+ return len(p), nil
+ }
+ d.line(d.buf[:i])
+ d.buf = d.buf[i+1:]
+ }
+}
+
+func (d *datasetWriter) flush() {
+ if len(bytes.TrimSpace(d.buf)) > 0 {
+ d.line(d.buf)
+ }
+ d.buf = nil
+}
+
+func (d *datasetWriter) line(b []byte) {
+ var l adminapi.KBDatasetLine
+ if json.Unmarshal(b, &l) != nil {
+ return
+ }
+ if d.chunks == nil {
+ d.chunks = map[string]bool{}
+ }
+ d.chunks[fmt.Sprintf("%s#%d", l.Doc, l.Ord)] = true
+ if l.Error != "" {
+ d.failed++
+ fmt.Fprintf(d.problems, " %s #%d: %s\n", l.Doc, l.Ord, l.Error)
+ return
+ }
+ d.n++
+ out, _ := json.Marshal(l)
+ fmt.Fprintln(d.pairs, string(out))
+}
diff --git a/internal/control/adminapi/kb.go b/internal/control/adminapi/kb.go
index 215f2b7..ee4f528 100644
--- a/internal/control/adminapi/kb.go
+++ b/internal/control/adminapi/kb.go
@@ -30,6 +30,7 @@ type KBStore interface {
DeleteDocument(ctx context.Context, collection, name string) (bool, error)
SearchChunks(ctx context.Context, q store.KBSearch) ([]store.KBHit, error)
SetKBDefault(ctx context.Context, d store.KBDefault) error
+ CollectionChunks(ctx context.Context, c store.KBCollection, limit int) ([]store.KBHit, error)
KBDefaults(ctx context.Context) ([]store.KBDefault, error)
}
@@ -161,6 +162,7 @@ func (a *API) registerKB(mux Mux) {
mux.HandleFunc("DELETE /api/v1/kb/grants", a.kbGrant(false))
mux.HandleFunc("POST /api/v1/kb/search", a.kbSearch)
mux.HandleFunc("POST /api/v1/kb/defaults", a.kbSetDefault)
+ mux.HandleFunc("POST /api/v1/kb/collections/{id}/dataset", a.kbDataset)
}
// kbReady answers for a gateway without knowledge bases, and says whether
diff --git a/internal/control/adminapi/kbdataset.go b/internal/control/adminapi/kbdataset.go
new file mode 100644
index 0000000..61c0f68
--- /dev/null
+++ b/internal/control/adminapi/kbdataset.go
@@ -0,0 +1,144 @@
+package adminapi
+
+import (
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strings"
+)
+
+// Training data from a collection (BACKLOG F4): questions a person might
+// ask of each chunk and the answers it holds, written by a model of the
+// cluster, for people to read through before an adapter is trained on them.
+// What an adapter learns well is how the organisation's answers sound; what
+// they say is still the knowledge base's to give.
+
+// KBDatasetRequest asks for training data from a collection.
+type KBDatasetRequest struct {
+ // Model writes the questions and answers: a chat model of the catalog.
+ Model string `json:"model"`
+ // PerChunk is how many pairs each chunk gives; 8 when 0.
+ PerChunk int `json:"per_chunk,omitempty"`
+ // MaxChunks bounds how many chunks are read; 500 when 0.
+ MaxChunks int `json:"max_chunks,omitempty"`
+}
+
+// KBDatasetLine is one line of the answer: a pair, or why a chunk gave none.
+type KBDatasetLine struct {
+ Q string `json:"q,omitempty"`
+ A string `json:"a,omitempty"`
+ Doc string `json:"doc"`
+ Ord int `json:"ord"`
+ Error string `json:"error,omitempty"`
+}
+
+const kbDatasetPrompt = `From the passage of an organisation's document below, write %d different questions an employee might ask, each with a short, correct answer.
+Rules: use only what the passage says. Phrase the questions differently from each other (formal, everyday, short, indirect). Answers are one or two sentences and keep the passage's numbers exactly. Leave out e-mail addresses and people's names. Write in the passage's own language.
+Output: one JSON object per line, {"q": "...", "a": "..."}, and nothing else.
+
+DOCUMENT %s:
+%s`
+
+var thinkBlock = regexp.MustCompile(`(?s).*?`)
+
+func (a *API) kbDataset(w http.ResponseWriter, r *http.Request) {
+ c, ok := a.kbCollection(w, r)
+ if !ok {
+ return
+ }
+ if a.replay == nil {
+ writeError(w, http.StatusNotImplemented, "no router here to ask a model")
+ return
+ }
+ var in KBDatasetRequest
+ if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
+ writeError(w, http.StatusBadRequest, err.Error())
+ return
+ }
+ m, found := a.ctl.Catalog().Resolve(in.Model)
+ if !found {
+ writeError(w, http.StatusBadRequest, "no model "+in.Model+" in the catalog")
+ return
+ }
+ if m.Capabilities.Embeddings {
+ writeError(w, http.StatusBadRequest, m.Name+" embeds; questions are written by a chat model")
+ return
+ }
+ per := in.PerChunk
+ if per <= 0 || per > 30 {
+ per = 8
+ }
+ limit := in.MaxChunks
+ if limit <= 0 || limit > 5000 {
+ limit = 500
+ }
+ chunks, err := a.kb.Store.CollectionChunks(r.Context(), c, limit)
+ if err != nil {
+ writeError(w, http.StatusInternalServerError, err.Error())
+ return
+ }
+ w.Header().Set("Content-Type", "application/x-ndjson")
+ w.WriteHeader(http.StatusOK)
+ enc := json.NewEncoder(w)
+ flush := func() {
+ if f, ok := w.(http.Flusher); ok {
+ f.Flush()
+ }
+ }
+ for _, ch := range chunks {
+ if r.Context().Err() != nil {
+ return
+ }
+ body, _ := json.Marshal(map[string]any{"model": m.Name, "temperature": 0.7, "max_tokens": 200 * per,
+ "messages": []map[string]string{{"role": "user", "content": fmt.Sprintf(kbDatasetPrompt, per, ch.Document, ch.Body)}}})
+ status, out, _, err := a.replay(r.Context(), "/v1/chat/completions", m.Name, body)
+ line := KBDatasetLine{Doc: ch.Document, Ord: ch.Ord}
+ if err == nil && status != http.StatusOK {
+ err = fmt.Errorf("%d: %s", status, strings.TrimSpace(string(out)))
+ }
+ if err != nil {
+ line.Error = err.Error()
+ _ = enc.Encode(line)
+ flush()
+ continue
+ }
+ pairs := datasetPairs(out)
+ if len(pairs) == 0 {
+ line.Error = "the model gave no pairs it could read"
+ _ = enc.Encode(line)
+ }
+ for _, p := range pairs {
+ p.Doc, p.Ord = ch.Document, ch.Ord
+ _ = enc.Encode(p)
+ }
+ flush()
+ }
+}
+
+// datasetPairs reads the {"q","a"} lines out of a chat completion.
+func datasetPairs(completion []byte) []KBDatasetLine {
+ var res struct {
+ Choices []struct {
+ Message struct {
+ Content string `json:"content"`
+ } `json:"message"`
+ } `json:"choices"`
+ }
+ if json.Unmarshal(completion, &res) != nil || len(res.Choices) == 0 {
+ return nil
+ }
+ text := thinkBlock.ReplaceAllString(res.Choices[0].Message.Content, "")
+ var out []KBDatasetLine
+ for _, line := range strings.Split(text, "\n") {
+ line = strings.TrimSuffix(strings.TrimSpace(line), ",")
+ if !strings.HasPrefix(line, "{") || !strings.HasSuffix(line, "}") {
+ continue
+ }
+ var p struct{ Q, A string }
+ if json.Unmarshal([]byte(line), &p) == nil && strings.TrimSpace(p.Q) != "" && strings.TrimSpace(p.A) != "" {
+ out = append(out, KBDatasetLine{Q: strings.TrimSpace(p.Q), A: strings.TrimSpace(p.A)})
+ }
+ }
+ return out
+}
diff --git a/internal/control/adminapi/kbdataset_test.go b/internal/control/adminapi/kbdataset_test.go
new file mode 100644
index 0000000..23edf80
--- /dev/null
+++ b/internal/control/adminapi/kbdataset_test.go
@@ -0,0 +1,91 @@
+package adminapi
+
+import (
+ "context"
+ "encoding/json"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strings"
+ "testing"
+ "time"
+
+ "gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
+ "gitea.mynodes.xyz/ilker/llmgw/internal/store"
+)
+
+func TestPairsAreReadOutOfWhatAModelWrote(t *testing.T) {
+ completion := `{"choices":[{"message":{"content":"let me see\n{\"q\": \"Babalık izni kaç gün?\", \"a\": \"10 iş günü.\"},\nnot json\n{\"q\": \"\", \"a\": \"empty question\"}\n{\"q\": \"Ne zaman?\", \"a\": \"Doğumdan sonra.\"}"}}]}`
+ got := datasetPairs([]byte(completion))
+ if len(got) != 2 || got[0].Q != "Babalık izni kaç gün?" || got[1].A != "Doğumdan sonra." {
+ t.Errorf("pairs %+v", got)
+ }
+ if datasetPairs([]byte(`{"choices":[]}`)) != nil {
+ t.Error("pairs from no choice")
+ }
+}
+
+func TestADatasetIsWrittenChunkByChunk(t *testing.T) {
+ dsn := os.Getenv("LLMGW_TEST_POSTGRES")
+ if dsn == "" {
+ t.Skip("set LLMGW_TEST_POSTGRES to a pgvector database to run this")
+ }
+ ctx := context.Background()
+ s, err := store.OpenPostgres(ctx, dsn)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer s.Close()
+ if s.KnowledgeAvailable() != nil {
+ t.Skip("no pgvector")
+ }
+ models := append(kbModels(), catalog.Model{Name: "writer", Source: "/w",
+ Profiles: []catalog.Profile{{Name: "cpu", GPUClass: "h100-80g", TP: 1}}})
+ a, _ := setup(t, models)
+ a.SetKB(&KBAdmin{Store: s})
+ asked := 0
+ a.SetReplayer(func(_ context.Context, path, model string, body []byte) (int, []byte, time.Duration, error) {
+ asked++
+ if path != "/v1/chat/completions" || model != "writer" || !strings.Contains(string(body), "Write in the passage") {
+ t.Errorf("asked %s %s %s", path, model, body)
+ }
+ if strings.Contains(string(body), "bozuk") {
+ return 200, []byte(`{"choices":[{"message":{"content":"sorry"}}]}`), 0, nil
+ }
+ return 200, []byte(`{"choices":[{"message":{"content":"{\"q\":\"Soru?\",\"a\":\"Cevap.\"}"}}]}`), 0, nil
+ })
+ org, _ := s.CreateOrg(ctx, "ds-"+strings.ToLower(store.NewID("")))
+ c, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeOrg, OwnerID: org, Name: "d", EmbedModel: "bge-m3", Dims: 3})
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, d := range []string{"a.md", "b.md"} {
+ body := "iyi metin"
+ if d == "b.md" {
+ body = "bozuk metin"
+ }
+ if _, _, err := s.PutDocument(ctx, c, store.KBDocument{Name: d, Digest: d}, []store.KBChunk{{Body: body, Embedding: []float32{1, 0, 0}}}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ mux := http.NewServeMux()
+ a.Register(mux)
+ rec := httptest.NewRecorder()
+ mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/kb/collections/"+c.ID+"/dataset", strings.NewReader(`{"model":"writer","per_chunk":3}`)))
+ var lines []KBDatasetLine
+ for _, l := range strings.Split(strings.TrimSpace(rec.Body.String()), "\n") {
+ var x KBDatasetLine
+ _ = json.Unmarshal([]byte(l), &x)
+ lines = append(lines, x)
+ }
+ if rec.Code != 200 || asked != 2 || len(lines) != 2 || lines[0].Q != "Soru?" || lines[0].Doc != "a.md" ||
+ lines[1].Doc != "b.md" || lines[1].Error == "" {
+ t.Errorf("%d asked %d: %+v", rec.Code, asked, lines)
+ }
+ // An embedding model writes no questions.
+ rec = httptest.NewRecorder()
+ mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/kb/collections/"+c.ID+"/dataset", strings.NewReader(`{"model":"bge-m3"}`)))
+ if rec.Code != http.StatusBadRequest {
+ t.Errorf("an embedding model: %d", rec.Code)
+ }
+}
diff --git a/internal/control/adminapi/routes.go b/internal/control/adminapi/routes.go
index c30b86b..72ca856 100644
--- a/internal/control/adminapi/routes.go
+++ b/internal/control/adminapi/routes.go
@@ -288,6 +288,9 @@ var Routes = []Route{
Description: "On an organization, team, project or key; the narrowest one with a default wins. No collections clears it. " +
"A request's own X-Llmgw-Knowledge overrides it, and X-Llmgw-Knowledge: none turns it off.",
Request: KBDefaultRequest{}, Response: KBDefaultView{}},
+ {Method: "POST", Path: "/api/v1/kb/collections/{id}/dataset", Summary: "Training data from a collection: questions and answers a chat model writes from each chunk",
+ Description: "NDJSON, a line per pair as each chunk is read, or a line with error for a chunk that gave none. For people to correct before an adapter is trained on it.",
+ Request: KBDatasetRequest{}, Response: KBDatasetLine{}},
{Method: "POST", Path: "/api/v1/kb/search", Summary: "Try a search of a collection, as an administrator",
Description: "No tenancy applies: this is for checking what a collection holds and finds.",
Request: KBSearchRequest{}, Response: []KBHitView{}},
diff --git a/internal/store/knowledge.go b/internal/store/knowledge.go
index 50b8257..515353b 100644
--- a/internal/store/knowledge.go
+++ b/internal/store/knowledge.go
@@ -1026,3 +1026,32 @@ func (s *DB) KBDefaultFor(ctx context.Context, e Effective) ([]string, error) {
}
return nil, nil
}
+
+// CollectionChunks lists a collection's chunks in document order, for
+// building training data from them (BACKLOG F4). At most limit.
+func (s *DB) CollectionChunks(ctx context.Context, c KBCollection, limit int) ([]KBHit, error) {
+ if err := s.KnowledgeAvailable(); err != nil {
+ return nil, err
+ }
+ if !kbIDPattern.MatchString(c.ID) {
+ return nil, ErrNotFound
+ }
+ var out []KBHit
+ err := s.inTx(ctx, func(tx conn) error {
+ if err := kbScope(ctx, tx, c.OrgID, []string{c.ID}); err != nil {
+ return err
+ }
+ var err error
+ out, err = queryAll(ctx, tx, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body
+ FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id
+ WHERE k.org_id = ? AND k.collection = `+sqlLiteral(c.ID)+`
+ ORDER BY d.name, k.ord LIMIT ?`, []any{c.OrgID, limit},
+ func(r *sql.Rows) (KBHit, error) {
+ var h KBHit
+ err := r.Scan(&h.Collection, &h.DocID, &h.Document, &h.Ord, &h.Body)
+ return h, err
+ })
+ return err
+ })
+ return out, err
+}