Training data from a collection: llmgw ctl kb dataset

A chat model of the cluster reads each chunk and writes questions a person
might ask of it with the answers it holds, streamed as NDJSON a chunk at a
time, for people to correct before an adapter is trained on it. A chunk that
gave no pairs says so on its own line.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 20:04:37 +02:00
1 parent 0a6a586dfe
commit ae95ac8261
8 files changed
+455

No files matched your search

+29
View File
@@ -1026,3 +1026,32 @@ func (s *DB) KBDefaultFor(ctx context.Context, e Effective) ([]string, error) {
}
return nil, nil
}
// CollectionChunks lists a collection's chunks in document order, for
// building training data from them (BACKLOG F4). At most limit.
func (s *DB) CollectionChunks(ctx context.Context, c KBCollection, limit int) ([]KBHit, error) {
if err := s.KnowledgeAvailable(); err != nil {
return nil, err
}
if !kbIDPattern.MatchString(c.ID) {
return nil, ErrNotFound
}
var out []KBHit
err := s.inTx(ctx, func(tx conn) error {
if err := kbScope(ctx, tx, c.OrgID, []string{c.ID}); err != nil {
return err
}
var err error
out, err = queryAll(ctx, tx, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id
WHERE k.org_id = ? AND k.collection = `+sqlLiteral(c.ID)+`
ORDER BY d.name, k.ord LIMIT ?`, []any{c.OrgID, limit},
func(r *sql.Rows) (KBHit, error) {
var h KBHit
err := r.Scan(&h.Collection, &h.DocID, &h.Document, &h.Ord, &h.Body)
return h, err
})
return err
})
return out, err
}