Knowledge bases at scale: halfvec, a text index per collection, ef_search raised to the candidates wanted
A search asking for more than 10 passages got at most 40 candidates from HNSW, pgvector's default ef_search. The full-text arm scored every chunk holding a word of the question before limiting, over the whole organization's GIN index; it now takes candidates by ts_rank from the collection's own partial index and scores those. New collections index halfvec on pgvector 0.7 and later (up to 4000 dimensions). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
42d3b7ea22
commit
9c675e7293
7 files changed
+223
-26
No files matched your search
+113
-23
@@ -29,9 +29,13 @@ import (
|
||||
// ErrKnowledgeOff is a store that cannot hold knowledge bases.
|
||||
var ErrKnowledgeOff = errors.New("knowledge bases are off")
|
||||
|
||||
// MaxEmbeddingDims is the most dimensions pgvector can index (HNSW on vector).
|
||||
// MaxEmbeddingDims is the most dimensions pgvector can index with HNSW: on
|
||||
// vector; halfvec takes up to MaxHalfvecDims.
|
||||
const MaxEmbeddingDims = 2000
|
||||
|
||||
// MaxHalfvecDims is the most dimensions an HNSW index on halfvec takes.
|
||||
const MaxHalfvecDims = 4000
|
||||
|
||||
// KBPerm is what a grant allows.
|
||||
type KBPerm string
|
||||
|
||||
@@ -50,8 +54,11 @@ type KBCollection struct {
|
||||
Name string
|
||||
EmbedModel string
|
||||
Dims int
|
||||
CreatedAt time.Time
|
||||
CreatedBy string
|
||||
// VecType is how its vectors are indexed: vector, or halfvec where
|
||||
// pgvector has it (0.7 and later), at half the index's memory.
|
||||
VecType string
|
||||
CreatedAt time.Time
|
||||
CreatedBy string
|
||||
}
|
||||
|
||||
// KBGrant lets a scope, and everything below it, read or write a collection.
|
||||
@@ -133,11 +140,16 @@ func (s *DB) ensureKnowledge(ctx context.Context) error {
|
||||
"as postgres, run CREATE EXTENSION vector in it (%v)", ErrKnowledgeOff, err)
|
||||
}
|
||||
}
|
||||
var version string
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT extversion FROM pg_extension WHERE extname = 'vector'`).Scan(&version); err == nil {
|
||||
s.kbHalfvec = versionAtLeast(version, 0, 7)
|
||||
}
|
||||
var exists bool
|
||||
if err := s.db.QueryRowContext(ctx, `SELECT to_regclass('kb_chunks') IS NOT NULL`).Scan(&exists); err != nil {
|
||||
return fmt.Errorf("%w: %v", ErrKnowledgeOff, err)
|
||||
}
|
||||
if exists {
|
||||
s.kbBackfillIndexes(ctx)
|
||||
return nil
|
||||
}
|
||||
err := s.inTx(ctx, func(c conn) error {
|
||||
@@ -197,6 +209,52 @@ func kbVectorIndex(collection string) string {
|
||||
return "kb_vec_" + strings.TrimPrefix(collection, "kb_")
|
||||
}
|
||||
|
||||
// kbTextIndex is the name of a collection's full-text index.
|
||||
func kbTextIndex(collection string) string { return "kb_tsv_" + strings.TrimPrefix(collection, "kb_") }
|
||||
|
||||
// cast is the type a collection's vectors are compared as, which its index
|
||||
// is built on: the query must say the same for the planner to use it.
|
||||
func (c KBCollection) cast() string {
|
||||
t := c.VecType
|
||||
if t != "halfvec" {
|
||||
t = "vector"
|
||||
}
|
||||
return "::" + t + "(" + strconv.Itoa(c.Dims) + ")"
|
||||
}
|
||||
|
||||
// versionAtLeast compares an extension's "major.minor.patch".
|
||||
func versionAtLeast(v string, major, minor int) bool {
|
||||
parts := strings.SplitN(v, ".", 3)
|
||||
if len(parts) < 2 {
|
||||
return false
|
||||
}
|
||||
ma, err1 := strconv.Atoi(parts[0])
|
||||
mi, err2 := strconv.Atoi(parts[1])
|
||||
if err1 != nil || err2 != nil {
|
||||
return false
|
||||
}
|
||||
return ma > major || ma == major && mi >= minor
|
||||
}
|
||||
|
||||
// kbBackfillIndexes gives collections made before their full-text index
|
||||
// existed one. Best effort, at open: a search still works without it.
|
||||
func (s *DB) kbBackfillIndexes(ctx context.Context) {
|
||||
cs, err := queryAll(ctx, s.db, `SELECT id, org_id FROM kb_collections`, nil, func(r *sql.Rows) ([2]string, error) {
|
||||
var x [2]string
|
||||
return x, r.Scan(&x[0], &x[1])
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
for _, c := range cs {
|
||||
if !kbIDPattern.MatchString(c[0]) {
|
||||
continue
|
||||
}
|
||||
_, _ = s.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS `+kbTextIndex(c[0])+` ON `+kbPartition(c[1])+
|
||||
` USING gin (tsv) WHERE collection = `+sqlLiteral(c[0]))
|
||||
}
|
||||
}
|
||||
|
||||
func sqlLiteral(v string) string { return "'" + strings.ReplaceAll(v, "'", "''") + "'" }
|
||||
|
||||
// kbScope sets, for this transaction, which rows of kb_chunks it may touch.
|
||||
@@ -263,14 +321,22 @@ func (s *DB) CreateCollection(ctx context.Context, c KBCollection) (KBCollection
|
||||
return KBCollection{}, errors.New("a collection needs a name")
|
||||
case c.EmbedModel == "":
|
||||
return KBCollection{}, errors.New("a collection needs the embedding model its documents are embedded with")
|
||||
case c.Dims < 1 || c.Dims > MaxEmbeddingDims:
|
||||
return KBCollection{}, fmt.Errorf("%d dimensions: an indexed embedding has 1 to %d", c.Dims, MaxEmbeddingDims)
|
||||
case c.Dims < 1 || c.Dims > MaxHalfvecDims || c.Dims > MaxEmbeddingDims && !s.kbHalfvec:
|
||||
limit := MaxEmbeddingDims
|
||||
if s.kbHalfvec {
|
||||
limit = MaxHalfvecDims
|
||||
}
|
||||
return KBCollection{}, fmt.Errorf("%d dimensions: an indexed embedding has 1 to %d", c.Dims, limit)
|
||||
case c.OwnerScope == ScopeUser && c.OrgID == "":
|
||||
return KBCollection{}, errors.New("a person's collection names the organization it belongs to")
|
||||
case c.OwnerScope == ScopeKey:
|
||||
return KBCollection{}, errors.New("a key cannot own a collection; its project or its person can")
|
||||
}
|
||||
c.ID = NewID("kb_")
|
||||
c.VecType = "vector"
|
||||
if s.kbHalfvec {
|
||||
c.VecType = "halfvec"
|
||||
}
|
||||
if c.CreatedAt.IsZero() {
|
||||
c.CreatedAt = time.Now()
|
||||
}
|
||||
@@ -292,9 +358,9 @@ func (s *DB) CreateCollection(ctx context.Context, c KBCollection) (KBCollection
|
||||
return fmt.Errorf("%w: collection %q", ErrDuplicateName, c.Name)
|
||||
}
|
||||
if _, err := tx.ExecContext(ctx, `INSERT INTO kb_collections
|
||||
(id, org_id, owner_scope, owner_id, name, embed_model, dims, created_at, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
c.ID, c.OrgID, string(c.OwnerScope), c.OwnerID, c.Name, c.EmbedModel, c.Dims,
|
||||
(id, org_id, owner_scope, owner_id, name, embed_model, dims, vec_type, created_at, created_by)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
c.ID, c.OrgID, string(c.OwnerScope), c.OwnerID, c.Name, c.EmbedModel, c.Dims, c.VecType,
|
||||
millis(c.CreatedAt), c.CreatedBy); err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -303,12 +369,18 @@ func (s *DB) CreateCollection(ctx context.Context, c KBCollection) (KBCollection
|
||||
` PARTITION OF kb_chunks FOR VALUES IN (`+sqlLiteral(c.OrgID)+`)`); err != nil {
|
||||
return err
|
||||
}
|
||||
// A partial index per collection, on its organization's partition
|
||||
// Partial indexes per collection, on its organization's partition
|
||||
// only: the dimension is the collection's, and a search, which
|
||||
// names one collection at a time, uses exactly this one.
|
||||
_, err = tx.ExecContext(ctx, `CREATE INDEX `+kbVectorIndex(c.ID)+` ON `+part+
|
||||
` USING hnsw ((embedding::vector(`+strconv.Itoa(c.Dims)+`)) vector_cosine_ops)`+
|
||||
` WHERE collection = `+sqlLiteral(c.ID))
|
||||
// names one collection at a time, uses exactly these — a text
|
||||
// search over one collection does not walk its whole
|
||||
// organization's index.
|
||||
if _, err := tx.ExecContext(ctx, `CREATE INDEX `+kbVectorIndex(c.ID)+` ON `+part+
|
||||
` USING hnsw ((embedding`+c.cast()+`) `+c.VecType+`_cosine_ops)`+
|
||||
` WHERE collection = `+sqlLiteral(c.ID)); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, `CREATE INDEX `+kbTextIndex(c.ID)+` ON `+part+
|
||||
` USING gin (tsv) WHERE collection = `+sqlLiteral(c.ID))
|
||||
return err
|
||||
})
|
||||
if err != nil {
|
||||
@@ -317,13 +389,13 @@ func (s *DB) CreateCollection(ctx context.Context, c KBCollection) (KBCollection
|
||||
return c, nil
|
||||
}
|
||||
|
||||
const kbCollectionCols = `id, org_id, owner_scope, owner_id, name, embed_model, dims, created_at, created_by`
|
||||
const kbCollectionCols = `id, org_id, owner_scope, owner_id, name, embed_model, dims, vec_type, created_at, created_by`
|
||||
|
||||
func scanCollection(sc scanner) (KBCollection, error) {
|
||||
var c KBCollection
|
||||
var scope string
|
||||
var at int64
|
||||
err := sc.Scan(&c.ID, &c.OrgID, &scope, &c.OwnerID, &c.Name, &c.EmbedModel, &c.Dims, &at, &c.CreatedBy)
|
||||
err := sc.Scan(&c.ID, &c.OrgID, &scope, &c.OwnerID, &c.Name, &c.EmbedModel, &c.Dims, &c.VecType, &at, &c.CreatedBy)
|
||||
c.OwnerScope, c.CreatedAt = Scope(scope), fromMillis(at)
|
||||
return c, err
|
||||
}
|
||||
@@ -377,7 +449,10 @@ func (s *DB) DeleteCollection(ctx context.Context, id string) error {
|
||||
if _, err := c.ExecContext(ctx, `DELETE FROM kb_defaults WHERE collections = ''`); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = c.ExecContext(ctx, `DROP INDEX IF EXISTS `+kbVectorIndex(id))
|
||||
if _, err := c.ExecContext(ctx, `DROP INDEX IF EXISTS `+kbVectorIndex(id)); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = c.ExecContext(ctx, `DROP INDEX IF EXISTS `+kbTextIndex(id))
|
||||
return err
|
||||
})
|
||||
}
|
||||
@@ -729,6 +804,14 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
||||
if err := kbScope(ctx, tx, q.OrgID, ids); err != nil {
|
||||
return err
|
||||
}
|
||||
// pgvector's HNSW search returns at most ef_search rows (40 by
|
||||
// default), whatever LIMIT asks: more candidates need it raised.
|
||||
if len(q.Embeddings) > 0 {
|
||||
ef := min(max(candidates(q.K), 40), 1000)
|
||||
if _, err := tx.ExecContext(ctx, `SET LOCAL hnsw.ef_search = `+strconv.Itoa(ef)); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, c := range q.Collections {
|
||||
// The literal collection and cast are what let the planner use
|
||||
// that collection's partial index.
|
||||
@@ -739,7 +822,7 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
||||
if len(v) != c.Dims {
|
||||
return fmt.Errorf("the query has %d dimensions; collection %s takes %d", len(v), c.Name, c.Dims)
|
||||
}
|
||||
cast := `::vector(` + strconv.Itoa(c.Dims) + `)`
|
||||
cast := c.cast()
|
||||
sim = `1 - (k.embedding` + cast + ` <=> ?` + cast + `)`
|
||||
args = append(args, vectorText(v))
|
||||
if err := query(tx, false, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, 0::float8
|
||||
@@ -753,15 +836,20 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
||||
if tsq != "" {
|
||||
// The text search's finds are measured against the question's
|
||||
// meaning too, so the two are compared on one scale.
|
||||
// Candidates by the cheap rank first, from the collection's
|
||||
// own index; the share of words held and the similarity are
|
||||
// worked out for those alone, not for every chunk that holds
|
||||
// one word of the question.
|
||||
targs := append([]any{}, args...)
|
||||
for _, t := range terms {
|
||||
targs = append(targs, t)
|
||||
}
|
||||
targs = append(targs, tsq, q.OrgID, candidates(q.K))
|
||||
targs = append(targs, q.OrgID, tsq, tsq, candidates(q.K))
|
||||
if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, `+coverage+` AS coverage
|
||||
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq
|
||||
WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq
|
||||
ORDER BY coverage DESC, ts_rank(k.tsv, tq) DESC LIMIT ?`, targs...); err != nil {
|
||||
FROM (SELECT * FROM kb_chunks k
|
||||
WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ to_tsquery('simple', ?)
|
||||
ORDER BY ts_rank(k.tsv, to_tsquery('simple', ?)) DESC LIMIT ?) k
|
||||
JOIN kb_documents d ON d.id = k.doc_id`, targs...); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
@@ -887,8 +975,10 @@ func (s *DB) kbForgetScope(ctx context.Context, c conn, scope Scope, id string)
|
||||
return err
|
||||
}
|
||||
if kbIDPattern.MatchString(cid) {
|
||||
if _, err := c.ExecContext(ctx, `DROP INDEX IF EXISTS `+kbVectorIndex(cid)); err != nil {
|
||||
return err
|
||||
for _, idx := range []string{kbVectorIndex(cid), kbTextIndex(cid)} {
|
||||
if _, err := c.ExecContext(ctx, `DROP INDEX IF EXISTS `+idx); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Reference in new issue
Block a user