Files
llmgw/internal/control/adminapi/kbdataset.go
T
Ilker ManapandClaude Opus 5.5 ae95ac8261 Training data from a collection: llmgw ctl kb dataset
A chat model of the cluster reads each chunk and writes questions a person
might ask of it with the answers it holds, streamed as NDJSON a chunk at a
time, for people to correct before an adapter is trained on it. A chunk that
gave no pairs says so on its own line.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-08 20:04:37 +02:00

145 lines
4.6 KiB
Go

package adminapi
import (
"encoding/json"
"fmt"
"net/http"
"regexp"
"strings"
)
// Training data from a collection (BACKLOG F4): questions a person might
// ask of each chunk and the answers it holds, written by a model of the
// cluster, for people to read through before an adapter is trained on them.
// What an adapter learns well is how the organisation's answers sound; what
// they say is still the knowledge base's to give.
// KBDatasetRequest asks for training data from a collection.
type KBDatasetRequest struct {
// Model writes the questions and answers: a chat model of the catalog.
Model string `json:"model"`
// PerChunk is how many pairs each chunk gives; 8 when 0.
PerChunk int `json:"per_chunk,omitempty"`
// MaxChunks bounds how many chunks are read; 500 when 0.
MaxChunks int `json:"max_chunks,omitempty"`
}
// KBDatasetLine is one line of the answer: a pair, or why a chunk gave none.
type KBDatasetLine struct {
Q string `json:"q,omitempty"`
A string `json:"a,omitempty"`
Doc string `json:"doc"`
Ord int `json:"ord"`
Error string `json:"error,omitempty"`
}
const kbDatasetPrompt = `From the passage of an organisation's document below, write %d different questions an employee might ask, each with a short, correct answer.
Rules: use only what the passage says. Phrase the questions differently from each other (formal, everyday, short, indirect). Answers are one or two sentences and keep the passage's numbers exactly. Leave out e-mail addresses and people's names. Write in the passage's own language.
Output: one JSON object per line, {"q": "...", "a": "..."}, and nothing else.
DOCUMENT %s:
%s`
var thinkBlock = regexp.MustCompile(`(?s)<think>.*?</think>`)
func (a *API) kbDataset(w http.ResponseWriter, r *http.Request) {
c, ok := a.kbCollection(w, r)
if !ok {
return
}
if a.replay == nil {
writeError(w, http.StatusNotImplemented, "no router here to ask a model")
return
}
var in KBDatasetRequest
if err := json.NewDecoder(r.Body).Decode(&in); err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
m, found := a.ctl.Catalog().Resolve(in.Model)
if !found {
writeError(w, http.StatusBadRequest, "no model "+in.Model+" in the catalog")
return
}
if m.Capabilities.Embeddings {
writeError(w, http.StatusBadRequest, m.Name+" embeds; questions are written by a chat model")
return
}
per := in.PerChunk
if per <= 0 || per > 30 {
per = 8
}
limit := in.MaxChunks
if limit <= 0 || limit > 5000 {
limit = 500
}
chunks, err := a.kb.Store.CollectionChunks(r.Context(), c, limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
w.Header().Set("Content-Type", "application/x-ndjson")
w.WriteHeader(http.StatusOK)
enc := json.NewEncoder(w)
flush := func() {
if f, ok := w.(http.Flusher); ok {
f.Flush()
}
}
for _, ch := range chunks {
if r.Context().Err() != nil {
return
}
body, _ := json.Marshal(map[string]any{"model": m.Name, "temperature": 0.7, "max_tokens": 200 * per,
"messages": []map[string]string{{"role": "user", "content": fmt.Sprintf(kbDatasetPrompt, per, ch.Document, ch.Body)}}})
status, out, _, err := a.replay(r.Context(), "/v1/chat/completions", m.Name, body)
line := KBDatasetLine{Doc: ch.Document, Ord: ch.Ord}
if err == nil && status != http.StatusOK {
err = fmt.Errorf("%d: %s", status, strings.TrimSpace(string(out)))
}
if err != nil {
line.Error = err.Error()
_ = enc.Encode(line)
flush()
continue
}
pairs := datasetPairs(out)
if len(pairs) == 0 {
line.Error = "the model gave no pairs it could read"
_ = enc.Encode(line)
}
for _, p := range pairs {
p.Doc, p.Ord = ch.Document, ch.Ord
_ = enc.Encode(p)
}
flush()
}
}
// datasetPairs reads the {"q","a"} lines out of a chat completion.
func datasetPairs(completion []byte) []KBDatasetLine {
var res struct {
Choices []struct {
Message struct {
Content string `json:"content"`
} `json:"message"`
} `json:"choices"`
}
if json.Unmarshal(completion, &res) != nil || len(res.Choices) == 0 {
return nil
}
text := thinkBlock.ReplaceAllString(res.Choices[0].Message.Content, "")
var out []KBDatasetLine
for _, line := range strings.Split(text, "\n") {
line = strings.TrimSuffix(strings.TrimSpace(line), ",")
if !strings.HasPrefix(line, "{") || !strings.HasSuffix(line, "}") {
continue
}
var p struct{ Q, A string }
if json.Unmarshal([]byte(line), &p) == nil && strings.TrimSpace(p.Q) != "" && strings.TrimSpace(p.A) != "" {
out = append(out, KBDatasetLine{Q: strings.TrimSpace(p.Q), A: strings.TrimSpace(p.A)})
}
}
return out
}