Training data from a collection: llmgw ctl kb dataset
A chat model of the cluster reads each chunk and writes questions a person might ask of it with the answers it holds, streamed as NDJSON a chunk at a time, for people to correct before an adapter is trained on it. A chunk that gave no pairs says so on its own line. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0a6a586dfe
commit
ae95ac8261
8 files changed
+455
No files matched your search
@@ -116,6 +116,11 @@ func (c *ctlClient) do(ctx context.Context, method, path string, body, out any)
|
||||
if out == nil {
|
||||
return nil
|
||||
}
|
||||
// A stream, as it comes: NDJSON a long run writes line by line.
|
||||
if w, ok := out.(io.Writer); ok {
|
||||
_, err := io.Copy(w, resp.Body)
|
||||
return err
|
||||
}
|
||||
// An older control plane answers a path it does not know with its
|
||||
// dashboard: a page, which read as JSON said only "invalid character '<'".
|
||||
if strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") {
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
|
||||
@@ -212,6 +216,45 @@ func newKBCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
def.Flags().BoolVar(&clear, "clear", false, "no default any more")
|
||||
c.AddCommand(def)
|
||||
|
||||
var dsModel, dsOut string
|
||||
var dsPer, dsMax int
|
||||
dataset := &cobra.Command{
|
||||
Use: "dataset COLLECTION --model MODEL",
|
||||
Short: "Write training data from a collection: questions and answers, for an adapter",
|
||||
Long: "A chat model of the cluster reads each chunk of the collection and writes\n" +
|
||||
"questions a person might ask of it, with the answers it holds, one JSON object\n" +
|
||||
"a line ({\"q\", \"a\", \"doc\", \"ord\"}). Read it through and correct it before\n" +
|
||||
"training on it (llmgw ctl train): what the model got wrong, an adapter learns.\n\n" +
|
||||
"An adapter learns how answers sound — the organisation's words, format and\n" +
|
||||
"tone — far better than what they say; the facts stay the knowledge base's.",
|
||||
Example: " llmgw ctl kb dataset Acme/policies --model qwen2.5-coder-14b-instruct --out policies.jsonl",
|
||||
Args: cobra.ExactArgs(1),
|
||||
RunE: func(cmd *cobra.Command, args []string) error {
|
||||
w := cmd.OutOrStdout()
|
||||
if dsOut != "" {
|
||||
f, err := os.Create(dsOut)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
w = f
|
||||
}
|
||||
lw := &datasetWriter{pairs: w, problems: cmd.ErrOrStderr()}
|
||||
body, _ := json.Marshal(adminapi.KBDatasetRequest{Model: dsModel, PerChunk: dsPer, MaxChunks: dsMax})
|
||||
err := client().patient(6*time.Hour).do(cmd.Context(), "POST",
|
||||
"/api/v1/kb/collections/"+url.PathEscape(args[0])+"/dataset", json.RawMessage(body), lw)
|
||||
lw.flush()
|
||||
fmt.Fprintf(cmd.ErrOrStderr(), "%d pairs from %d chunks; %d chunks gave none\n", lw.n, len(lw.chunks), lw.failed)
|
||||
return err
|
||||
},
|
||||
}
|
||||
dataset.Flags().StringVar(&dsModel, "model", "", "the chat model that writes the questions and answers")
|
||||
dataset.Flags().StringVar(&dsOut, "out", "", "the file to write (default: standard output)")
|
||||
dataset.Flags().IntVar(&dsPer, "per-chunk", 8, "pairs from each chunk")
|
||||
dataset.Flags().IntVar(&dsMax, "max-chunks", 500, "the most chunks read")
|
||||
_ = dataset.MarkFlagRequired("model")
|
||||
c.AddCommand(dataset)
|
||||
|
||||
var k int
|
||||
search := &cobra.Command{
|
||||
Use: "search COLLECTION QUERY...",
|
||||
@@ -240,3 +283,50 @@ func newKBCmd(client func() *ctlClient, asJSON *bool) *cobra.Command {
|
||||
c.AddCommand(search)
|
||||
return c
|
||||
}
|
||||
|
||||
// datasetWriter splits the dataset stream: pairs to the file, problems to
|
||||
// the terminal, counted as they pass.
|
||||
type datasetWriter struct {
|
||||
pairs, problems io.Writer
|
||||
buf []byte
|
||||
n, failed int
|
||||
chunks map[string]bool
|
||||
}
|
||||
|
||||
func (d *datasetWriter) Write(p []byte) (int, error) {
|
||||
d.buf = append(d.buf, p...)
|
||||
for {
|
||||
i := bytes.IndexByte(d.buf, '\n')
|
||||
if i < 0 {
|
||||
return len(p), nil
|
||||
}
|
||||
d.line(d.buf[:i])
|
||||
d.buf = d.buf[i+1:]
|
||||
}
|
||||
}
|
||||
|
||||
func (d *datasetWriter) flush() {
|
||||
if len(bytes.TrimSpace(d.buf)) > 0 {
|
||||
d.line(d.buf)
|
||||
}
|
||||
d.buf = nil
|
||||
}
|
||||
|
||||
func (d *datasetWriter) line(b []byte) {
|
||||
var l adminapi.KBDatasetLine
|
||||
if json.Unmarshal(b, &l) != nil {
|
||||
return
|
||||
}
|
||||
if d.chunks == nil {
|
||||
d.chunks = map[string]bool{}
|
||||
}
|
||||
d.chunks[fmt.Sprintf("%s#%d", l.Doc, l.Ord)] = true
|
||||
if l.Error != "" {
|
||||
d.failed++
|
||||
fmt.Fprintf(d.problems, " %s #%d: %s\n", l.Doc, l.Ord, l.Error)
|
||||
return
|
||||
}
|
||||
d.n++
|
||||
out, _ := json.Marshal(l)
|
||||
fmt.Fprintln(d.pairs, string(out))
|
||||
}
|
||||
Reference in new issue
Block a user