Knowledge bases at scale: halfvec, a text index per collection, ef_search raised to the candidates wanted

A search asking for more than 10 passages got at most 40 candidates from
HNSW, pgvector's default ef_search. The full-text arm scored every chunk
holding a word of the question before limiting, over the whole
organization's GIN index; it now takes candidates by ts_rank from the
collection's own partial index and scores those. New collections index
halfvec on pgvector 0.7 and later (up to 4000 dimensions).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 22:08:29 +02:00
1 parent 42d3b7ea22
commit 9c675e7293
7 files changed
+223 -26

No files matched your search

+113 -23
View File
@@ -29,9 +29,13 @@ import (
// ErrKnowledgeOff is a store that cannot hold knowledge bases.
var ErrKnowledgeOff = errors.New("knowledge bases are off")
// MaxEmbeddingDims is the most dimensions pgvector can index (HNSW on vector).
// MaxEmbeddingDims is the most dimensions pgvector can index with HNSW: on
// vector; halfvec takes up to MaxHalfvecDims.
const MaxEmbeddingDims = 2000
// MaxHalfvecDims is the most dimensions an HNSW index on halfvec takes.
const MaxHalfvecDims = 4000
// KBPerm is what a grant allows.
type KBPerm string
@@ -50,8 +54,11 @@ type KBCollection struct {
Name string
EmbedModel string
Dims int
CreatedAt time.Time
CreatedBy string
// VecType is how its vectors are indexed: vector, or halfvec where
// pgvector has it (0.7 and later), at half the index's memory.
VecType string
CreatedAt time.Time
CreatedBy string
}
// KBGrant lets a scope, and everything below it, read or write a collection.
@@ -133,11 +140,16 @@ func (s *DB) ensureKnowledge(ctx context.Context) error {
"as postgres, run CREATE EXTENSION vector in it (%v)", ErrKnowledgeOff, err)
}
}
var version string
if err := s.db.QueryRowContext(ctx, `SELECT extversion FROM pg_extension WHERE extname = 'vector'`).Scan(&version); err == nil {
s.kbHalfvec = versionAtLeast(version, 0, 7)
}
var exists bool
if err := s.db.QueryRowContext(ctx, `SELECT to_regclass('kb_chunks') IS NOT NULL`).Scan(&exists); err != nil {
return fmt.Errorf("%w: %v", ErrKnowledgeOff, err)
}
if exists {
s.kbBackfillIndexes(ctx)
return nil
}
err := s.inTx(ctx, func(c conn) error {
@@ -197,6 +209,52 @@ func kbVectorIndex(collection string) string {
return "kb_vec_" + strings.TrimPrefix(collection, "kb_")
}
// kbTextIndex is the name of a collection's full-text index.
func kbTextIndex(collection string) string { return "kb_tsv_" + strings.TrimPrefix(collection, "kb_") }
// cast is the type a collection's vectors are compared as, which its index
// is built on: the query must say the same for the planner to use it.
func (c KBCollection) cast() string {
t := c.VecType
if t != "halfvec" {
t = "vector"
}
return "::" + t + "(" + strconv.Itoa(c.Dims) + ")"
}
// versionAtLeast compares an extension's "major.minor.patch".
func versionAtLeast(v string, major, minor int) bool {
parts := strings.SplitN(v, ".", 3)
if len(parts) < 2 {
return false
}
ma, err1 := strconv.Atoi(parts[0])
mi, err2 := strconv.Atoi(parts[1])
if err1 != nil || err2 != nil {
return false
}
return ma > major || ma == major && mi >= minor
}
// kbBackfillIndexes gives collections made before their full-text index
// existed one. Best effort, at open: a search still works without it.
func (s *DB) kbBackfillIndexes(ctx context.Context) {
cs, err := queryAll(ctx, s.db, `SELECT id, org_id FROM kb_collections`, nil, func(r *sql.Rows) ([2]string, error) {
var x [2]string
return x, r.Scan(&x[0], &x[1])
})
if err != nil {
return
}
for _, c := range cs {
if !kbIDPattern.MatchString(c[0]) {
continue
}
_, _ = s.db.ExecContext(ctx, `CREATE INDEX IF NOT EXISTS `+kbTextIndex(c[0])+` ON `+kbPartition(c[1])+
` USING gin (tsv) WHERE collection = `+sqlLiteral(c[0]))
}
}
func sqlLiteral(v string) string { return "'" + strings.ReplaceAll(v, "'", "''") + "'" }
// kbScope sets, for this transaction, which rows of kb_chunks it may touch.
@@ -263,14 +321,22 @@ func (s *DB) CreateCollection(ctx context.Context, c KBCollection) (KBCollection
return KBCollection{}, errors.New("a collection needs a name")
case c.EmbedModel == "":
return KBCollection{}, errors.New("a collection needs the embedding model its documents are embedded with")
case c.Dims < 1 || c.Dims > MaxEmbeddingDims:
return KBCollection{}, fmt.Errorf("%d dimensions: an indexed embedding has 1 to %d", c.Dims, MaxEmbeddingDims)
case c.Dims < 1 || c.Dims > MaxHalfvecDims || c.Dims > MaxEmbeddingDims && !s.kbHalfvec:
limit := MaxEmbeddingDims
if s.kbHalfvec {
limit = MaxHalfvecDims
}
return KBCollection{}, fmt.Errorf("%d dimensions: an indexed embedding has 1 to %d", c.Dims, limit)
case c.OwnerScope == ScopeUser && c.OrgID == "":
return KBCollection{}, errors.New("a person's collection names the organization it belongs to")
case c.OwnerScope == ScopeKey:
return KBCollection{}, errors.New("a key cannot own a collection; its project or its person can")
}
c.ID = NewID("kb_")
c.VecType = "vector"
if s.kbHalfvec {
c.VecType = "halfvec"
}
if c.CreatedAt.IsZero() {
c.CreatedAt = time.Now()
}
@@ -292,9 +358,9 @@ func (s *DB) CreateCollection(ctx context.Context, c KBCollection) (KBCollection
return fmt.Errorf("%w: collection %q", ErrDuplicateName, c.Name)
}
if _, err := tx.ExecContext(ctx, `INSERT INTO kb_collections
(id, org_id, owner_scope, owner_id, name, embed_model, dims, created_at, created_by)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`,
c.ID, c.OrgID, string(c.OwnerScope), c.OwnerID, c.Name, c.EmbedModel, c.Dims,
(id, org_id, owner_scope, owner_id, name, embed_model, dims, vec_type, created_at, created_by)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
c.ID, c.OrgID, string(c.OwnerScope), c.OwnerID, c.Name, c.EmbedModel, c.Dims, c.VecType,
millis(c.CreatedAt), c.CreatedBy); err != nil {
return err
}
@@ -303,12 +369,18 @@ func (s *DB) CreateCollection(ctx context.Context, c KBCollection) (KBCollection
` PARTITION OF kb_chunks FOR VALUES IN (`+sqlLiteral(c.OrgID)+`)`); err != nil {
return err
}
// A partial index per collection, on its organization's partition
// Partial indexes per collection, on its organization's partition
// only: the dimension is the collection's, and a search, which
// names one collection at a time, uses exactly this one.
_, err = tx.ExecContext(ctx, `CREATE INDEX `+kbVectorIndex(c.ID)+` ON `+part+
` USING hnsw ((embedding::vector(`+strconv.Itoa(c.Dims)+`)) vector_cosine_ops)`+
` WHERE collection = `+sqlLiteral(c.ID))
// names one collection at a time, uses exactly these — a text
// search over one collection does not walk its whole
// organization's index.
if _, err := tx.ExecContext(ctx, `CREATE INDEX `+kbVectorIndex(c.ID)+` ON `+part+
` USING hnsw ((embedding`+c.cast()+`) `+c.VecType+`_cosine_ops)`+
` WHERE collection = `+sqlLiteral(c.ID)); err != nil {
return err
}
_, err = tx.ExecContext(ctx, `CREATE INDEX `+kbTextIndex(c.ID)+` ON `+part+
` USING gin (tsv) WHERE collection = `+sqlLiteral(c.ID))
return err
})
if err != nil {
@@ -317,13 +389,13 @@ func (s *DB) CreateCollection(ctx context.Context, c KBCollection) (KBCollection
return c, nil
}
const kbCollectionCols = `id, org_id, owner_scope, owner_id, name, embed_model, dims, created_at, created_by`
const kbCollectionCols = `id, org_id, owner_scope, owner_id, name, embed_model, dims, vec_type, created_at, created_by`
func scanCollection(sc scanner) (KBCollection, error) {
var c KBCollection
var scope string
var at int64
err := sc.Scan(&c.ID, &c.OrgID, &scope, &c.OwnerID, &c.Name, &c.EmbedModel, &c.Dims, &at, &c.CreatedBy)
err := sc.Scan(&c.ID, &c.OrgID, &scope, &c.OwnerID, &c.Name, &c.EmbedModel, &c.Dims, &c.VecType, &at, &c.CreatedBy)
c.OwnerScope, c.CreatedAt = Scope(scope), fromMillis(at)
return c, err
}
@@ -377,7 +449,10 @@ func (s *DB) DeleteCollection(ctx context.Context, id string) error {
if _, err := c.ExecContext(ctx, `DELETE FROM kb_defaults WHERE collections = ''`); err != nil {
return err
}
_, err = c.ExecContext(ctx, `DROP INDEX IF EXISTS `+kbVectorIndex(id))
if _, err := c.ExecContext(ctx, `DROP INDEX IF EXISTS `+kbVectorIndex(id)); err != nil {
return err
}
_, err = c.ExecContext(ctx, `DROP INDEX IF EXISTS `+kbTextIndex(id))
return err
})
}
@@ -729,6 +804,14 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
if err := kbScope(ctx, tx, q.OrgID, ids); err != nil {
return err
}
// pgvector's HNSW search returns at most ef_search rows (40 by
// default), whatever LIMIT asks: more candidates need it raised.
if len(q.Embeddings) > 0 {
ef := min(max(candidates(q.K), 40), 1000)
if _, err := tx.ExecContext(ctx, `SET LOCAL hnsw.ef_search = `+strconv.Itoa(ef)); err != nil {
return err
}
}
for _, c := range q.Collections {
// The literal collection and cast are what let the planner use
// that collection's partial index.
@@ -739,7 +822,7 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
if len(v) != c.Dims {
return fmt.Errorf("the query has %d dimensions; collection %s takes %d", len(v), c.Name, c.Dims)
}
cast := `::vector(` + strconv.Itoa(c.Dims) + `)`
cast := c.cast()
sim = `1 - (k.embedding` + cast + ` <=> ?` + cast + `)`
args = append(args, vectorText(v))
if err := query(tx, false, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, 0::float8
@@ -753,15 +836,20 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
if tsq != "" {
// The text search's finds are measured against the question's
// meaning too, so the two are compared on one scale.
// Candidates by the cheap rank first, from the collection's
// own index; the share of words held and the similarity are
// worked out for those alone, not for every chunk that holds
// one word of the question.
targs := append([]any{}, args...)
for _, t := range terms {
targs = append(targs, t)
}
targs = append(targs, tsq, q.OrgID, candidates(q.K))
targs = append(targs, q.OrgID, tsq, tsq, candidates(q.K))
if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, `+coverage+` AS coverage
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq
WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq
ORDER BY coverage DESC, ts_rank(k.tsv, tq) DESC LIMIT ?`, targs...); err != nil {
FROM (SELECT * FROM kb_chunks k
WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ to_tsquery('simple', ?)
ORDER BY ts_rank(k.tsv, to_tsquery('simple', ?)) DESC LIMIT ?) k
JOIN kb_documents d ON d.id = k.doc_id`, targs...); err != nil {
return err
}
}
@@ -887,8 +975,10 @@ func (s *DB) kbForgetScope(ctx context.Context, c conn, scope Scope, id string)
return err
}
if kbIDPattern.MatchString(cid) {
if _, err := c.ExecContext(ctx, `DROP INDEX IF EXISTS `+kbVectorIndex(cid)); err != nil {
return err
for _, idx := range []string{kbVectorIndex(cid), kbTextIndex(cid)} {
if _, err := c.ExecContext(ctx, `DROP INDEX IF EXISTS `+idx); err != nil {
return err
}
}
}
}