Training data from a collection: llmgw ctl kb dataset

A chat model of the cluster reads each chunk and writes questions a person
might ask of it with the answers it holds, streamed as NDJSON a chunk at a
time, for people to correct before an adapter is trained on it. A chunk that
gave no pairs says so on its own line.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 20:04:37 +02:00
1 parent 0a6a586dfe
commit ae95ac8261
8 files changed
+455

No files matched your search

+5
View File
@@ -116,6 +116,11 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
if out == nil {
return nil
}
// A stream, as it comes: NDJSON a long run writes line by line.
if w, ok := out.(io.Writer); ok {
_, err := io.Copy(w, resp.Body)
return err
}
// An older control plane answers a path it does not know with its
// dashboard: a page, which read as JSON said only "invalid character '<'".
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
+90
View File
@@ -1,10 +1,14 @@
package cli
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/url"
"os"
"strings"
"time"
"github.com/spf13/cobra"
@@ -212,6 +216,45 @@ func newKBCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
def.Flags().BoolVar(&clear, "clear", false, "no default any more")
c.AddCommand(def)
var dsModel, dsOut string
var dsPer, dsMax int
dataset := &cobra.Command{
Use: "dataset COLLECTION --model MODEL",
Short: "Write training data from a collection: questions and answers, for an adapter",
Long: "A chat model of the cluster reads each chunk of the collection and writes\n" +
"questions a person might ask of it, with the answers it holds, one JSON object\n" +
"a line ({\"q\", \"a\", \"doc\", \"ord\"}). Read it through and correct it before\n" +
"training on it (llmgw ctl train): what the model got wrong, an adapter learns.\n\n" +
"An adapter learns how answers sound — the organisation's words, format and\n" +
"tone — far better than what they say; the facts stay the knowledge base's.",
Example: " llmgw ctl kb dataset Acme/policies --model qwen2.5-coder-14b-instruct --out policies.jsonl",
Args: cobra.ExactArgs(1),
RunE: func(cmd *cobra.Command, args []string) error {
w := cmd.OutOrStdout()
if dsOut != "" {
f, err := os.Create(dsOut)
if err != nil {
return err
}
defer f.Close()
w = f
}
lw := &datasetWriter{pairs: w, problems: cmd.ErrOrStderr()}
body, _ := json.Marshal(adminapi.KBDatasetRequest{Model: dsModel, PerChunk: dsPer, MaxChunks: dsMax})
err := client().patient(6*time.Hour).do(cmd.Context(), "POST",
"/api/v1/kb/collections/"+url.PathEscape(args[0])+"/dataset", json.RawMessage(body), lw)
lw.flush()
fmt.Fprintf(cmd.ErrOrStderr(), "%d pairs from %d chunks; %d chunks gave none\n", lw.n, len(lw.chunks), lw.failed)
return err
},
}
dataset.Flags().StringVar(&dsModel, "model", "", "the chat model that writes the questions and answers")
dataset.Flags().StringVar(&dsOut, "out", "", "the file to write (default: standard output)")
dataset.Flags().IntVar(&dsPer, "per-chunk", 8, "pairs from each chunk")
dataset.Flags().IntVar(&dsMax, "max-chunks", 500, "the most chunks read")
_ = dataset.MarkFlagRequired("model")
c.AddCommand(dataset)
var k int
search := &cobra.Command{
Use: "search COLLECTION QUERY...",
@@ -240,3 +283,50 @@ func newKBCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
c.AddCommand(search)
return c
}
// datasetWriter splits the dataset stream: pairs to the file, problems to
// the terminal, counted as they pass.
type datasetWriter struct {
pairs, problems io.Writer
buf []byte
n, failed int
chunks map[string]bool
}
func (d *datasetWriter) Write(p []byte) (int, error) {
d.buf = append(d.buf, p...)
for {
i := bytes.IndexByte(d.buf, '\n')
if i < 0 {
return len(p), nil
}
d.line(d.buf[:i])
d.buf = d.buf[i+1:]
}
}
func (d *datasetWriter) flush() {
if len(bytes.TrimSpace(d.buf)) > 0 {
d.line(d.buf)
}
d.buf = nil
}
func (d *datasetWriter) line(b []byte) {
var l adminapi.KBDatasetLine
if json.Unmarshal(b, &l) != nil {
return
}
if d.chunks == nil {
d.chunks = map[string]bool{}
}
d.chunks[fmt.Sprintf("%s#%d", l.Doc, l.Ord)] = true
if l.Error != "" {
d.failed++
fmt.Fprintf(d.problems, " %s #%d: %s\n", l.Doc, l.Ord, l.Error)
return
}
d.n++
out, _ := json.Marshal(l)
fmt.Fprintln(d.pairs, string(out))
}