Knowledge bases at scale: halfvec, a text index per collection, ef_search raised to the candidates wanted

A search asking for more than 10 passages got at most 40 candidates from
HNSW, pgvector's default ef_search. The full-text arm scored every chunk
holding a word of the question before limiting, over the whole
organization's GIN index; it now takes candidates by ts_rank from the
collection's own partial index and scores those. New collections index
halfvec on pgvector 0.7 and later (up to 4000 dimensions).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 22:08:29 +02:00
1 parent 42d3b7ea22
commit 9c675e7293
7 files changed
+223 -26

No files matched your search

+79 -3
View File
@@ -3,6 +3,7 @@ package store
import (
"context"
"errors"
"fmt"
"math"
"strings"
"testing"
@@ -320,10 +321,85 @@ func TestPostgresScopesWithCollectionsAreNotDeleted(t *testing.T) {
if err := s.DeleteProject(ctx, "proj2"); err != nil {
t.Errorf("deleting the project once its collection is gone: %v", err)
}
for _, idx := range []string{kbVectorIndex(c.ID), kbTextIndex(c.ID)} {
if indexExists(t, s, idx) {
t.Errorf("the collection's index %s is left behind", idx)
}
}
}
func indexExists(t *testing.T, s *DB, name string) bool {
t.Helper()
var n int
if err := s.raw.QueryRowContext(ctx, `SELECT COUNT(*) FROM pg_indexes WHERE indexname = $1`,
kbVectorIndex(c.ID)).Scan(&n); err != nil || n != 0 {
t.Errorf("the collection's index is left behind (%d, %v)", n, err)
if err := s.raw.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM pg_indexes WHERE indexname = $1`,
name).Scan(&n); err != nil {
t.Fatal(err)
}
return n == 1
}
func TestPostgresCollectionsHaveTheirOwnIndexes(t *testing.T) {
ctx := context.Background()
s := openKB(t)
kbTwoOrgs(t, s)
c := mustCollection(t, s, ScopeProject, "proj1", "", "docs")
if !indexExists(t, s, kbVectorIndex(c.ID)) || !indexExists(t, s, kbTextIndex(c.ID)) {
t.Fatal("a new collection lacks its vector or text index")
}
// The test image's pgvector has halfvec; a collection there uses it, and
// is searched as it is indexed.
if !s.kbHalfvec || c.VecType != "halfvec" {
t.Errorf("halfvec %v, the collection's type %q", s.kbHalfvec, c.VecType)
}
if got, err := s.Collection(ctx, c.ID); err != nil || got.VecType != c.VecType {
t.Errorf("read back %q, %v", got.VecType, err)
}
var def string
if err := s.raw.QueryRowContext(ctx, `SELECT indexdef FROM pg_indexes WHERE indexname = $1`, kbVectorIndex(c.ID)).Scan(&def); err != nil ||
!strings.Contains(def, "halfvec_cosine_ops") {
t.Errorf("the vector index is %q (%v)", def, err)
}
if _, err := s.CreateCollection(ctx, KBCollection{OwnerScope: ScopeProject, OwnerID: "proj1",
Name: "wide", EmbedModel: "emb", Dims: 3072}); err != nil {
t.Errorf("3072 dimensions, which halfvec indexes: %v", err)
}
if _, err := s.CreateCollection(ctx, KBCollection{OwnerScope: ScopeProject, OwnerID: "proj1",
Name: "too-wide", EmbedModel: "emb", Dims: MaxHalfvecDims + 1}); err == nil {
t.Error("more dimensions than any index takes were accepted")
}
// A collection made before text indexes were gets one when the store
// opens.
if _, err := s.raw.ExecContext(ctx, `DROP INDEX `+kbTextIndex(c.ID)); err != nil {
t.Fatal(err)
}
s.kbBackfillIndexes(ctx)
if !indexExists(t, s, kbTextIndex(c.ID)) {
t.Error("the text index was not put back")
}
// Asking for more than HNSW's default 40 is answered in full. (A table
// this small is read without its index: with one, 40 is all HNSW gives
// unless ef_search is raised, as an EXPLAIN on 20000 rows showed.)
var chunks []KBChunk
for i := range 120 {
chunks = append(chunks, KBChunk{Body: fmt.Sprintf("parça %d izin", i), Embedding: unit(i % 3)})
}
if _, _, err := s.PutDocument(ctx, c, KBDocument{Name: "big.md", Digest: "big"}, chunks); err != nil {
t.Fatal(err)
}
hits, err := s.SearchChunks(ctx, KBSearch{OrgID: "org1", Collections: []KBCollection{c},
Embeddings: map[string][]float32{"emb": unit(0)}, K: 50})
if err != nil || len(hits) != 50 {
t.Errorf("K=50 gave %d hits, %v", len(hits), err)
}
}
func TestVersionAtLeast(t *testing.T) {
for v, want := range map[string]bool{"0.6.0": false, "0.7.0": true, "0.8.7": true, "1.0": true, "0": false, "x.y": false} {
if got := versionAtLeast(v, 0, 7); got != want {
t.Errorf("%s: %v", v, got)
}
}
}