Knowledge bases at scale: halfvec, a text index per collection, ef_search raised to the candidates wanted
A search asking for more than 10 passages got at most 40 candidates from HNSW, pgvector's default ef_search. The full-text arm scored every chunk holding a word of the question before limiting, over the whole organization's GIN index; it now takes candidates by ts_rank from the collection's own partial index and scores those. New collections index halfvec on pgvector 0.7 and later (up to 4000 dimensions). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
42d3b7ea22
commit
9c675e7293
7 files changed
+223
-26
No files matched your search
@@ -3,6 +3,7 @@ package store
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math"
|
||||
"strings"
|
||||
"testing"
|
||||
@@ -320,10 +321,85 @@ func TestPostgresScopesWithCollectionsAreNotDeleted(t *testing.T) {
|
||||
if err := s.DeleteProject(ctx, "proj2"); err != nil {
|
||||
t.Errorf("deleting the project once its collection is gone: %v", err)
|
||||
}
|
||||
for _, idx := range []string{kbVectorIndex(c.ID), kbTextIndex(c.ID)} {
|
||||
if indexExists(t, s, idx) {
|
||||
t.Errorf("the collection's index %s is left behind", idx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func indexExists(t *testing.T, s *DB, name string) bool {
|
||||
t.Helper()
|
||||
var n int
|
||||
if err := s.raw.QueryRowContext(ctx, `SELECT COUNT(*) FROM pg_indexes WHERE indexname = $1`,
|
||||
kbVectorIndex(c.ID)).Scan(&n); err != nil || n != 0 {
|
||||
t.Errorf("the collection's index is left behind (%d, %v)", n, err)
|
||||
if err := s.raw.QueryRowContext(context.Background(), `SELECT COUNT(*) FROM pg_indexes WHERE indexname = $1`,
|
||||
name).Scan(&n); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return n == 1
|
||||
}
|
||||
|
||||
func TestPostgresCollectionsHaveTheirOwnIndexes(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
s := openKB(t)
|
||||
kbTwoOrgs(t, s)
|
||||
c := mustCollection(t, s, ScopeProject, "proj1", "", "docs")
|
||||
if !indexExists(t, s, kbVectorIndex(c.ID)) || !indexExists(t, s, kbTextIndex(c.ID)) {
|
||||
t.Fatal("a new collection lacks its vector or text index")
|
||||
}
|
||||
// The test image's pgvector has halfvec; a collection there uses it, and
|
||||
// is searched as it is indexed.
|
||||
if !s.kbHalfvec || c.VecType != "halfvec" {
|
||||
t.Errorf("halfvec %v, the collection's type %q", s.kbHalfvec, c.VecType)
|
||||
}
|
||||
if got, err := s.Collection(ctx, c.ID); err != nil || got.VecType != c.VecType {
|
||||
t.Errorf("read back %q, %v", got.VecType, err)
|
||||
}
|
||||
var def string
|
||||
if err := s.raw.QueryRowContext(ctx, `SELECT indexdef FROM pg_indexes WHERE indexname = $1`, kbVectorIndex(c.ID)).Scan(&def); err != nil ||
|
||||
!strings.Contains(def, "halfvec_cosine_ops") {
|
||||
t.Errorf("the vector index is %q (%v)", def, err)
|
||||
}
|
||||
if _, err := s.CreateCollection(ctx, KBCollection{OwnerScope: ScopeProject, OwnerID: "proj1",
|
||||
Name: "wide", EmbedModel: "emb", Dims: 3072}); err != nil {
|
||||
t.Errorf("3072 dimensions, which halfvec indexes: %v", err)
|
||||
}
|
||||
if _, err := s.CreateCollection(ctx, KBCollection{OwnerScope: ScopeProject, OwnerID: "proj1",
|
||||
Name: "too-wide", EmbedModel: "emb", Dims: MaxHalfvecDims + 1}); err == nil {
|
||||
t.Error("more dimensions than any index takes were accepted")
|
||||
}
|
||||
|
||||
// A collection made before text indexes were gets one when the store
|
||||
// opens.
|
||||
if _, err := s.raw.ExecContext(ctx, `DROP INDEX `+kbTextIndex(c.ID)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
s.kbBackfillIndexes(ctx)
|
||||
if !indexExists(t, s, kbTextIndex(c.ID)) {
|
||||
t.Error("the text index was not put back")
|
||||
}
|
||||
|
||||
// Asking for more than HNSW's default 40 is answered in full. (A table
|
||||
// this small is read without its index: with one, 40 is all HNSW gives
|
||||
// unless ef_search is raised, as an EXPLAIN on 20000 rows showed.)
|
||||
var chunks []KBChunk
|
||||
for i := range 120 {
|
||||
chunks = append(chunks, KBChunk{Body: fmt.Sprintf("parça %d izin", i), Embedding: unit(i % 3)})
|
||||
}
|
||||
if _, _, err := s.PutDocument(ctx, c, KBDocument{Name: "big.md", Digest: "big"}, chunks); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hits, err := s.SearchChunks(ctx, KBSearch{OrgID: "org1", Collections: []KBCollection{c},
|
||||
Embeddings: map[string][]float32{"emb": unit(0)}, K: 50})
|
||||
if err != nil || len(hits) != 50 {
|
||||
t.Errorf("K=50 gave %d hits, %v", len(hits), err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVersionAtLeast(t *testing.T) {
|
||||
for v, want := range map[string]bool{"0.6.0": false, "0.7.0": true, "0.8.7": true, "1.0": true, "0": false, "x.y": false} {
|
||||
if got := versionAtLeast(v, 0, 7); got != want {
|
||||
t.Errorf("%s: %v", v, got)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user