Training data from a collection: llmgw ctl kb dataset
A chat model of the cluster reads each chunk and writes questions a person might ask of it with the answers it holds, streamed as NDJSON a chunk at a time, for people to correct before an adapter is trained on it. A chunk that gave no pairs says so on its own line. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0a6a586dfe
commit
ae95ac8261
8 files changed
+455
No files matched your search
@@ -1026,3 +1026,32 @@ func (s *DB) KBDefaultFor(ctx context.Context, e Effective) ([]string, error) {
|
||||
}
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// CollectionChunks lists a collection's chunks in document order, for
|
||||
// building training data from them (BACKLOG F4). At most limit.
|
||||
func (s *DB) CollectionChunks(ctx context.Context, c KBCollection, limit int) ([]KBHit, error) {
|
||||
if err := s.KnowledgeAvailable(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !kbIDPattern.MatchString(c.ID) {
|
||||
return nil, ErrNotFound
|
||||
}
|
||||
var out []KBHit
|
||||
err := s.inTx(ctx, func(tx conn) error {
|
||||
if err := kbScope(ctx, tx, c.OrgID, []string{c.ID}); err != nil {
|
||||
return err
|
||||
}
|
||||
var err error
|
||||
out, err = queryAll(ctx, tx, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body
|
||||
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id
|
||||
WHERE k.org_id = ? AND k.collection = `+sqlLiteral(c.ID)+`
|
||||
ORDER BY d.name, k.ord LIMIT ?`, []any{c.OrgID, limit},
|
||||
func(r *sql.Rows) (KBHit, error) {
|
||||
var h KBHit
|
||||
err := r.Scan(&h.Collection, &h.DocID, &h.Document, &h.Ord, &h.Body)
|
||||
return h, err
|
||||
})
|
||||
return err
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
Reference in new issue
Block a user