Training data from a collection: llmgw ctl kb dataset
A chat model of the cluster reads each chunk and writes questions a person might ask of it with the answers it holds, streamed as NDJSON a chunk at a time, for people to correct before an adapter is trained on it. A chunk that gave no pairs says so on its own line. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0a6a586dfe
commit
ae95ac8261
8 files changed
+455
No files matched your search
@@ -30,6 +30,7 @@ type KBStore interface {
|
||||
DeleteDocument(ctx context.Context, collection, name string) (bool, error)
|
||||
SearchChunks(ctx context.Context, q store.KBSearch) ([]store.KBHit, error)
|
||||
SetKBDefault(ctx context.Context, d store.KBDefault) error
|
||||
CollectionChunks(ctx context.Context, c store.KBCollection, limit int) ([]store.KBHit, error)
|
||||
KBDefaults(ctx context.Context) ([]store.KBDefault, error)
|
||||
}
|
||||
|
||||
@@ -161,6 +162,7 @@ func (a *API) registerKB(mux Mux) {
|
||||
mux.HandleFunc("DELETE /api/v1/kb/grants", a.kbGrant(false))
|
||||
mux.HandleFunc("POST /api/v1/kb/search", a.kbSearch)
|
||||
mux.HandleFunc("POST /api/v1/kb/defaults", a.kbSetDefault)
|
||||
mux.HandleFunc("POST /api/v1/kb/collections/{id}/dataset", a.kbDataset)
|
||||
}
|
||||
|
||||
// kbReady answers for a gateway without knowledge bases, and says whether
|
||||
|
||||
@@ -0,0 +1,144 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Training data from a collection (BACKLOG F4): questions a person might
|
||||
// ask of each chunk and the answers it holds, written by a model of the
|
||||
// cluster, for people to read through before an adapter is trained on them.
|
||||
// What an adapter learns well is how the organisation's answers sound; what
|
||||
// they say is still the knowledge base's to give.
|
||||
|
||||
// KBDatasetRequest asks for training data from a collection.
|
||||
type KBDatasetRequest struct {
|
||||
// Model writes the questions and answers: a chat model of the catalog.
|
||||
Model string `json:"model"`
|
||||
// PerChunk is how many pairs each chunk gives; 8 when 0.
|
||||
PerChunk int `json:"per_chunk,omitempty"`
|
||||
// MaxChunks bounds how many chunks are read; 500 when 0.
|
||||
MaxChunks int `json:"max_chunks,omitempty"`
|
||||
}
|
||||
|
||||
// KBDatasetLine is one line of the answer: a pair, or why a chunk gave none.
|
||||
type KBDatasetLine struct {
|
||||
Q string `json:"q,omitempty"`
|
||||
A string `json:"a,omitempty"`
|
||||
Doc string `json:"doc"`
|
||||
Ord int `json:"ord"`
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
const kbDatasetPrompt = `From the passage of an organisation's document below, write %d different questions an employee might ask, each with a short, correct answer.
|
||||
Rules: use only what the passage says. Phrase the questions differently from each other (formal, everyday, short, indirect). Answers are one or two sentences and keep the passage's numbers exactly. Leave out e-mail addresses and people's names. Write in the passage's own language.
|
||||
Output: one JSON object per line, {"q": "...", "a": "..."}, and nothing else.
|
||||
|
||||
DOCUMENT %s:
|
||||
%s`
|
||||
|
||||
var thinkBlock = regexp.MustCompile(`(?s)<think>.*?</think>`)
|
||||
|
||||
func (a *API) kbDataset(w http.ResponseWriter, r *http.Request) {
|
||||
c, ok := a.kbCollection(w, r)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if a.replay == nil {
|
||||
writeError(w, http.StatusNotImplemented, "no router here to ask a model")
|
||||
return
|
||||
}
|
||||
var in KBDatasetRequest
|
||||
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
|
||||
writeError(w, http.StatusBadRequest, err.Error())
|
||||
return
|
||||
}
|
||||
m, found := a.ctl.Catalog().Resolve(in.Model)
|
||||
if !found {
|
||||
writeError(w, http.StatusBadRequest, "no model "+in.Model+" in the catalog")
|
||||
return
|
||||
}
|
||||
if m.Capabilities.Embeddings {
|
||||
writeError(w, http.StatusBadRequest, m.Name+" embeds; questions are written by a chat model")
|
||||
return
|
||||
}
|
||||
per := in.PerChunk
|
||||
if per <= 0 || per > 30 {
|
||||
per = 8
|
||||
}
|
||||
limit := in.MaxChunks
|
||||
if limit <= 0 || limit > 5000 {
|
||||
limit = 500
|
||||
}
|
||||
chunks, err := a.kb.Store.CollectionChunks(r.Context(), c, limit)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
w.Header().Set("Content-Type", "application/x-ndjson")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
enc := json.NewEncoder(w)
|
||||
flush := func() {
|
||||
if f, ok := w.(http.Flusher); ok {
|
||||
f.Flush()
|
||||
}
|
||||
}
|
||||
for _, ch := range chunks {
|
||||
if r.Context().Err() != nil {
|
||||
return
|
||||
}
|
||||
body, _ := json.Marshal(map[string]any{"model": m.Name, "temperature": 0.7, "max_tokens": 200 * per,
|
||||
"messages": []map[string]string{{"role": "user", "content": fmt.Sprintf(kbDatasetPrompt, per, ch.Document, ch.Body)}}})
|
||||
status, out, _, err := a.replay(r.Context(), "/v1/chat/completions", m.Name, body)
|
||||
line := KBDatasetLine{Doc: ch.Document, Ord: ch.Ord}
|
||||
if err == nil && status != http.StatusOK {
|
||||
err = fmt.Errorf("%d: %s", status, strings.TrimSpace(string(out)))
|
||||
}
|
||||
if err != nil {
|
||||
line.Error = err.Error()
|
||||
_ = enc.Encode(line)
|
||||
flush()
|
||||
continue
|
||||
}
|
||||
pairs := datasetPairs(out)
|
||||
if len(pairs) == 0 {
|
||||
line.Error = "the model gave no pairs it could read"
|
||||
_ = enc.Encode(line)
|
||||
}
|
||||
for _, p := range pairs {
|
||||
p.Doc, p.Ord = ch.Document, ch.Ord
|
||||
_ = enc.Encode(p)
|
||||
}
|
||||
flush()
|
||||
}
|
||||
}
|
||||
|
||||
// datasetPairs reads the {"q","a"} lines out of a chat completion.
|
||||
func datasetPairs(completion []byte) []KBDatasetLine {
|
||||
var res struct {
|
||||
Choices []struct {
|
||||
Message struct {
|
||||
Content string `json:"content"`
|
||||
} `json:"message"`
|
||||
} `json:"choices"`
|
||||
}
|
||||
if json.Unmarshal(completion, &res) != nil || len(res.Choices) == 0 {
|
||||
return nil
|
||||
}
|
||||
text := thinkBlock.ReplaceAllString(res.Choices[0].Message.Content, "")
|
||||
var out []KBDatasetLine
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
line = strings.TrimSuffix(strings.TrimSpace(line), ",")
|
||||
if !strings.HasPrefix(line, "{") || !strings.HasSuffix(line, "}") {
|
||||
continue
|
||||
}
|
||||
var p struct{ Q, A string }
|
||||
if json.Unmarshal([]byte(line), &p) == nil && strings.TrimSpace(p.Q) != "" && strings.TrimSpace(p.A) != "" {
|
||||
out = append(out, KBDatasetLine{Q: strings.TrimSpace(p.Q), A: strings.TrimSpace(p.A)})
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package adminapi
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func TestPairsAreReadOutOfWhatAModelWrote(t *testing.T) {
|
||||
completion := `{"choices":[{"message":{"content":"<think>let me see</think>\n{\"q\": \"Babalık izni kaç gün?\", \"a\": \"10 iş günü.\"},\nnot json\n{\"q\": \"\", \"a\": \"empty question\"}\n{\"q\": \"Ne zaman?\", \"a\": \"Doğumdan sonra.\"}"}}]}`
|
||||
got := datasetPairs([]byte(completion))
|
||||
if len(got) != 2 || got[0].Q != "Babalık izni kaç gün?" || got[1].A != "Doğumdan sonra." {
|
||||
t.Errorf("pairs %+v", got)
|
||||
}
|
||||
if datasetPairs([]byte(`{"choices":[]}`)) != nil {
|
||||
t.Error("pairs from no choice")
|
||||
}
|
||||
}
|
||||
|
||||
func TestADatasetIsWrittenChunkByChunk(t *testing.T) {
|
||||
dsn := os.Getenv("LLMGW_TEST_POSTGRES")
|
||||
if dsn == "" {
|
||||
t.Skip("set LLMGW_TEST_POSTGRES to a pgvector database to run this")
|
||||
}
|
||||
ctx := context.Background()
|
||||
s, err := store.OpenPostgres(ctx, dsn)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer s.Close()
|
||||
if s.KnowledgeAvailable() != nil {
|
||||
t.Skip("no pgvector")
|
||||
}
|
||||
models := append(kbModels(), catalog.Model{Name: "writer", Source: "/w",
|
||||
Profiles: []catalog.Profile{{Name: "cpu", GPUClass: "h100-80g", TP: 1}}})
|
||||
a, _ := setup(t, models)
|
||||
a.SetKB(&KBAdmin{Store: s})
|
||||
asked := 0
|
||||
a.SetReplayer(func(_ context.Context, path, model string, body []byte) (int, []byte, time.Duration, error) {
|
||||
asked++
|
||||
if path != "/v1/chat/completions" || model != "writer" || !strings.Contains(string(body), "Write in the passage") {
|
||||
t.Errorf("asked %s %s %s", path, model, body)
|
||||
}
|
||||
if strings.Contains(string(body), "bozuk") {
|
||||
return 200, []byte(`{"choices":[{"message":{"content":"sorry"}}]}`), 0, nil
|
||||
}
|
||||
return 200, []byte(`{"choices":[{"message":{"content":"{\"q\":\"Soru?\",\"a\":\"Cevap.\"}"}}]}`), 0, nil
|
||||
})
|
||||
org, _ := s.CreateOrg(ctx, "ds-"+strings.ToLower(store.NewID("")))
|
||||
c, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeOrg, OwnerID: org, Name: "d", EmbedModel: "bge-m3", Dims: 3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
for _, d := range []string{"a.md", "b.md"} {
|
||||
body := "iyi metin"
|
||||
if d == "b.md" {
|
||||
body = "bozuk metin"
|
||||
}
|
||||
if _, _, err := s.PutDocument(ctx, c, store.KBDocument{Name: d, Digest: d}, []store.KBChunk{{Body: body, Embedding: []float32{1, 0, 0}}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
mux := http.NewServeMux()
|
||||
a.Register(mux)
|
||||
rec := httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/kb/collections/"+c.ID+"/dataset", strings.NewReader(`{"model":"writer","per_chunk":3}`)))
|
||||
var lines []KBDatasetLine
|
||||
for _, l := range strings.Split(strings.TrimSpace(rec.Body.String()), "\n") {
|
||||
var x KBDatasetLine
|
||||
_ = json.Unmarshal([]byte(l), &x)
|
||||
lines = append(lines, x)
|
||||
}
|
||||
if rec.Code != 200 || asked != 2 || len(lines) != 2 || lines[0].Q != "Soru?" || lines[0].Doc != "a.md" ||
|
||||
lines[1].Doc != "b.md" || lines[1].Error == "" {
|
||||
t.Errorf("%d asked %d: %+v", rec.Code, asked, lines)
|
||||
}
|
||||
// An embedding model writes no questions.
|
||||
rec = httptest.NewRecorder()
|
||||
mux.ServeHTTP(rec, httptest.NewRequest("POST", "/api/v1/kb/collections/"+c.ID+"/dataset", strings.NewReader(`{"model":"bge-m3"}`)))
|
||||
if rec.Code != http.StatusBadRequest {
|
||||
t.Errorf("an embedding model: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
@@ -288,6 +288,9 @@ var Routes = []Route{
|
||||
Description: "On an organization, team, project or key; the narrowest one with a default wins. No collections clears it. " +
|
||||
"A request's own X-Llmgw-Knowledge overrides it, and X-Llmgw-Knowledge: none turns it off.",
|
||||
Request: KBDefaultRequest{}, Response: KBDefaultView{}},
|
||||
{Method: "POST", Path: "/api/v1/kb/collections/{id}/dataset", Summary: "Training data from a collection: questions and answers a chat model writes from each chunk",
|
||||
Description: "NDJSON, a line per pair as each chunk is read, or a line with error for a chunk that gave none. For people to correct before an adapter is trained on it.",
|
||||
Request: KBDatasetRequest{}, Response: KBDatasetLine{}},
|
||||
{Method: "POST", Path: "/api/v1/kb/search", Summary: "Try a search of a collection, as an administrator",
|
||||
Description: "No tenancy applies: this is for checking what a collection holds and finds.",
|
||||
Request: KBSearchRequest{}, Response: []KBHitView{}},
|
||||
|
||||
Reference in new issue
Block a user