Knowledge search ranks by how near a chunk's meaning is, not by its place in two lists
Fused by rank, a full-text match on a question's everyday words outvoted a far nearer meaning: on the PoC, "how much leave do fathers get" found the annual-leave page before the paternity one (cosine distance 0.44 against 0.25). Now a chunk scores its similarity to the question plus up to 0.3 for a full-text match, each search brings four times k candidates, the text search drops Turkish and English stop words, and an operator's search says when the embedding model did not answer rather than quietly searching text. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
1ec9b83076
commit
bc5ee9f3aa
3 files changed
+153
-69
No files matched your search
@@ -450,11 +450,15 @@ func (a *API) kbSearch(w http.ResponseWriter, r *http.Request) {
|
|||||||
k = 5
|
k = 5
|
||||||
}
|
}
|
||||||
q := store.KBSearch{OrgID: c.OrgID, Collections: []store.KBCollection{c}, Text: in.Query, K: k}
|
q := store.KBSearch{OrgID: c.OrgID, Collections: []store.KBCollection{c}, Text: in.Query, K: k}
|
||||||
// Without the model, the full-text half still answers.
|
// Said, not swallowed: a search that quietly lost its meaning half
|
||||||
|
// looks like a collection that does not know the answer.
|
||||||
if a.kb.Embed != nil {
|
if a.kb.Embed != nil {
|
||||||
if v, err := a.kb.Embed(r.Context(), c.EmbedModel, in.Query); err == nil {
|
v, err := a.kb.Embed(r.Context(), c.EmbedModel, in.Query)
|
||||||
q.Embeddings = map[string][]float32{c.EmbedModel: v}
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadGateway, c.EmbedModel+" did not embed the question: "+err.Error())
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
q.Embeddings = map[string][]float32{c.EmbedModel: v}
|
||||||
}
|
}
|
||||||
hits, err := a.kb.Store.SearchChunks(r.Context(), q)
|
hits, err := a.kb.Store.SearchChunks(r.Context(), q)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
|||||||
+113
-65
@@ -89,8 +89,9 @@ type KBHit struct {
|
|||||||
Document string
|
Document string
|
||||||
Ord int
|
Ord int
|
||||||
Body string
|
Body string
|
||||||
// Score is the fused rank (reciprocal rank fusion over the vector and
|
// Score is the cosine similarity to the question, plus up to
|
||||||
// text searches): only the order means anything.
|
// textWeight for a full-text match: higher is better, and only the
|
||||||
|
// order is meant to be compared.
|
||||||
Score float64
|
Score float64
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -626,13 +627,22 @@ type KBSearch struct {
|
|||||||
K int
|
K int
|
||||||
}
|
}
|
||||||
|
|
||||||
// rrfK is reciprocal rank fusion's constant: 60, as in the paper and nearly
|
// textWeight is how much a full-text match adds to a chunk's similarity to
|
||||||
// everywhere it is used.
|
// the question, at most. The embedding model reads what a question means;
|
||||||
const rrfK = 60
|
// full text is there for what it cannot — a code, a name, a number. Fused by
|
||||||
|
// rank, as first built, the two were equals and the gap between a close
|
||||||
|
// meaning and a distant one was thrown away: "how much leave do fathers
|
||||||
|
// get" found the annual-leave page over the paternity one on the PoC,
|
||||||
|
// though the paternity page was far nearer (0.25 against 0.44).
|
||||||
|
const textWeight = 0.3
|
||||||
|
|
||||||
|
// candidates is how many each search brings to the fusion for k results:
|
||||||
|
// with only k from each, the meaning search's third-best never met the
|
||||||
|
// text search's first, ties were broken by id, and the order was luck.
|
||||||
|
func candidates(k int) int { return max(4*k, 20) }
|
||||||
|
|
||||||
// SearchChunks finds the best chunks for a query across collections: a
|
// SearchChunks finds the best chunks for a query across collections: a
|
||||||
// nearest-neighbour search in each collection and a full-text search across
|
// nearest-neighbour search and a full-text search in each, put on one scale. Every collection must be in OrgID; the caller decides
|
||||||
// them, fused by rank. Every collection must be in OrgID; the caller decides
|
|
||||||
// which ones the requester may read, and the table refuses rows outside them
|
// which ones the requester may read, and the table refuses rows outside them
|
||||||
// regardless.
|
// regardless.
|
||||||
func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
||||||
@@ -643,86 +653,93 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
|||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
ids := make([]string, 0, len(q.Collections))
|
ids := make([]string, 0, len(q.Collections))
|
||||||
quoted := make([]string, 0, len(q.Collections))
|
|
||||||
for _, c := range q.Collections {
|
for _, c := range q.Collections {
|
||||||
if c.OrgID != q.OrgID || !kbIDPattern.MatchString(c.ID) {
|
if c.OrgID != q.OrgID || !kbIDPattern.MatchString(c.ID) {
|
||||||
return nil, fmt.Errorf("collection %s is not in organization %s", c.ID, q.OrgID)
|
return nil, fmt.Errorf("collection %s is not in organization %s", c.ID, q.OrgID)
|
||||||
}
|
}
|
||||||
ids = append(ids, c.ID)
|
ids = append(ids, c.ID)
|
||||||
quoted = append(quoted, sqlLiteral(c.ID))
|
|
||||||
}
|
}
|
||||||
type key struct {
|
type key struct {
|
||||||
doc string
|
doc string
|
||||||
ord int
|
ord int
|
||||||
}
|
}
|
||||||
hits := map[key]*KBHit{}
|
type found struct {
|
||||||
add := func(h KBHit, rank int) {
|
hit KBHit
|
||||||
k := key{h.DocID, h.Ord}
|
// sim is the cosine similarity to the question, when the question
|
||||||
if have, ok := hits[k]; ok {
|
// was embedded for this chunk's collection; text is its full-text
|
||||||
have.Score += 1.0 / float64(rrfK+rank)
|
// rank, 0 when the text search did not find it.
|
||||||
return
|
sim, text float64
|
||||||
}
|
hasSim bool
|
||||||
h.Score = 1.0 / float64(rrfK+rank)
|
|
||||||
hits[k] = &h
|
|
||||||
}
|
}
|
||||||
scan := func(rows *sql.Rows) ([]KBHit, error) {
|
hits := map[key]*found{}
|
||||||
|
add := func(h KBHit, sim sql.NullFloat64, text float64) {
|
||||||
|
k := key{h.DocID, h.Ord}
|
||||||
|
f, ok := hits[k]
|
||||||
|
if !ok {
|
||||||
|
f = &found{hit: h}
|
||||||
|
hits[k] = f
|
||||||
|
}
|
||||||
|
if sim.Valid {
|
||||||
|
f.sim, f.hasSim = sim.Float64, true
|
||||||
|
}
|
||||||
|
f.text = max(f.text, text)
|
||||||
|
}
|
||||||
|
query := func(tx conn, text bool, q string, args ...any) error {
|
||||||
|
rows, err := tx.QueryContext(ctx, q, args...)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
defer rows.Close()
|
defer rows.Close()
|
||||||
var out []KBHit
|
|
||||||
for rows.Next() {
|
for rows.Next() {
|
||||||
var h KBHit
|
var h KBHit
|
||||||
if err := rows.Scan(&h.Collection, &h.DocID, &h.Document, &h.Ord, &h.Body); err != nil {
|
var sim sql.NullFloat64
|
||||||
return nil, err
|
var rank float64
|
||||||
|
if err := rows.Scan(&h.Collection, &h.DocID, &h.Document, &h.Ord, &h.Body, &sim, &rank); err != nil {
|
||||||
|
return err
|
||||||
}
|
}
|
||||||
out = append(out, h)
|
if !text {
|
||||||
|
rank = 0
|
||||||
|
}
|
||||||
|
add(h, sim, rank)
|
||||||
}
|
}
|
||||||
return out, rows.Err()
|
return rows.Err()
|
||||||
}
|
}
|
||||||
|
tsq := orQuery(q.Text)
|
||||||
err := s.inTx(ctx, func(tx conn) error {
|
err := s.inTx(ctx, func(tx conn) error {
|
||||||
if err := kbScope(ctx, tx, q.OrgID, ids); err != nil {
|
if err := kbScope(ctx, tx, q.OrgID, ids); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
for _, c := range q.Collections {
|
for _, c := range q.Collections {
|
||||||
v, ok := q.Embeddings[c.EmbedModel]
|
|
||||||
if !ok {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
if len(v) != c.Dims {
|
|
||||||
return fmt.Errorf("the query has %d dimensions; collection %s takes %d", len(v), c.Name, c.Dims)
|
|
||||||
}
|
|
||||||
// The literal collection and cast are what let the planner use
|
// The literal collection and cast are what let the planner use
|
||||||
// that collection's partial index.
|
// that collection's partial index.
|
||||||
cast := `::vector(` + strconv.Itoa(c.Dims) + `)`
|
lit := sqlLiteral(c.ID)
|
||||||
rows, err := tx.QueryContext(ctx, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body
|
sim, args := "NULL::float8", []any{}
|
||||||
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id
|
v, embedded := q.Embeddings[c.EmbedModel]
|
||||||
WHERE k.org_id = ? AND k.collection = `+sqlLiteral(c.ID)+`
|
if embedded {
|
||||||
ORDER BY k.embedding`+cast+` <=> ?`+cast+` LIMIT ?`,
|
if len(v) != c.Dims {
|
||||||
q.OrgID, vectorText(v), q.K)
|
return fmt.Errorf("the query has %d dimensions; collection %s takes %d", len(v), c.Name, c.Dims)
|
||||||
if err != nil {
|
}
|
||||||
return err
|
cast := `::vector(` + strconv.Itoa(c.Dims) + `)`
|
||||||
|
sim = `1 - (k.embedding` + cast + ` <=> ?` + cast + `)`
|
||||||
|
args = append(args, vectorText(v))
|
||||||
|
if err := query(tx, false, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, 0::float8
|
||||||
|
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id
|
||||||
|
WHERE k.org_id = ? AND k.collection = `+lit+`
|
||||||
|
ORDER BY k.embedding`+cast+` <=> ?`+cast+` LIMIT ?`,
|
||||||
|
vectorText(v), q.OrgID, vectorText(v), candidates(q.K)); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
}
|
}
|
||||||
got, err := scan(rows)
|
if tsq != "" {
|
||||||
if err != nil {
|
// The text search's finds are measured against the question's
|
||||||
return err
|
// meaning too, so the two are compared on one scale.
|
||||||
}
|
if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, ts_rank(k.tsv, tq)::float8
|
||||||
for i, h := range got {
|
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq
|
||||||
add(h, i+1)
|
WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq
|
||||||
}
|
ORDER BY ts_rank(k.tsv, tq) DESC LIMIT ?`,
|
||||||
}
|
append(args, tsq, q.OrgID, candidates(q.K))...); err != nil {
|
||||||
if tsq := orQuery(q.Text); tsq != "" {
|
return err
|
||||||
rows, err := tx.QueryContext(ctx, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body
|
}
|
||||||
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) q
|
|
||||||
WHERE k.org_id = ? AND k.collection IN (`+strings.Join(quoted, ", ")+`) AND k.tsv @@ q
|
|
||||||
ORDER BY ts_rank(k.tsv, q) DESC LIMIT ?`,
|
|
||||||
tsq, q.OrgID, q.K)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
got, err := scan(rows)
|
|
||||||
if err != nil {
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
for i, h := range got {
|
|
||||||
add(h, i+1)
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
@@ -730,9 +747,21 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
// One scale: similarity, plus the text rank as a share of the best one,
|
||||||
|
// weighted down.
|
||||||
|
best := 0.0
|
||||||
|
for _, f := range hits {
|
||||||
|
best = max(best, f.text)
|
||||||
|
}
|
||||||
|
for _, f := range hits {
|
||||||
|
f.hit.Score = f.sim
|
||||||
|
if best > 0 {
|
||||||
|
f.hit.Score += textWeight * f.text / best
|
||||||
|
}
|
||||||
|
}
|
||||||
out := make([]KBHit, 0, len(hits))
|
out := make([]KBHit, 0, len(hits))
|
||||||
for _, h := range hits {
|
for _, f := range hits {
|
||||||
out = append(out, *h)
|
out = append(out, f.hit)
|
||||||
}
|
}
|
||||||
sort.Slice(out, func(i, j int) bool {
|
sort.Slice(out, func(i, j int) bool {
|
||||||
if out[i].Score != out[j].Score {
|
if out[i].Score != out[j].Score {
|
||||||
@@ -749,6 +778,25 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
|||||||
return out, nil
|
return out, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// stopWords are words nearly every passage has, in Turkish and English:
|
||||||
|
// matching on them only ranks whatever is longest first.
|
||||||
|
var stopWords = func() map[string]bool {
|
||||||
|
m := map[string]bool{}
|
||||||
|
for _, w := range strings.Fields(`
|
||||||
|
acaba ama ancak artık aslında az bana bazı belki ben beni benim beri bile bir birçok biri birkaç
|
||||||
|
biz bize bizi bizim bu buna bunda bundan bunlar bunları bunların bunu bunun burada çok çünkü da
|
||||||
|
daha dahi de defa diye en gibi göre hangi hangisi hem hep hepsi her hiç için ile ise kaç kadar
|
||||||
|
ki kim kime kimi mı mi mu mü nasıl ne neden nerde nerede nereye niçin niye o olan olarak oldu
|
||||||
|
olduğu olsa olur ona ondan onlar onları onların onu onun orada öyle şey şu şuna şunda şundan şunu
|
||||||
|
tüm ve veya ya yani var yok mısın misin musun müsün midir mıdır nedir hangileri lütfen
|
||||||
|
a an and are as at be by can do does for from has have how i in is it its me my of on or our
|
||||||
|
should that the their there this to was what when where which who why will with you your
|
||||||
|
`) {
|
||||||
|
m[w] = true
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}()
|
||||||
|
|
||||||
// orQuery turns a question into a tsquery matching any of its words. All of
|
// orQuery turns a question into a tsquery matching any of its words. All of
|
||||||
// them (plainto_tsquery) would match almost nothing a person types as a
|
// them (plainto_tsquery) would match almost nothing a person types as a
|
||||||
// question; the rank then prefers chunks with more of them.
|
// question; the rank then prefers chunks with more of them.
|
||||||
@@ -761,7 +809,7 @@ func orQuery(text string) string {
|
|||||||
for _, w := range strings.FieldsFunc(text, func(r rune) bool {
|
for _, w := range strings.FieldsFunc(text, func(r rune) bool {
|
||||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r)
|
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r)
|
||||||
}) {
|
}) {
|
||||||
if len([]rune(w)) < 2 || seen[w] {
|
if len([]rune(w)) < 2 || seen[w] || stopWords[strings.ToLowerSpecial(unicode.TurkishCase, w)] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seen[w] = true
|
seen[w] = true
|
||||||
|
|||||||
@@ -352,10 +352,42 @@ func TestPostgresAPersonsCollectionsGoWithTheAccount(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestPostgresMeaningOutweighsSharedWords(t *testing.T) {
|
||||||
|
// The PoC's case: the question shares its everyday words with one
|
||||||
|
// passage and its meaning with another.
|
||||||
|
ctx := context.Background()
|
||||||
|
s := openKB(t)
|
||||||
|
kbTwoOrgs(t, s)
|
||||||
|
c := mustCollection(t, s, ScopeProject, "proj1", "", "docs")
|
||||||
|
put := func(name, body string, v []float32) {
|
||||||
|
t.Helper()
|
||||||
|
if _, _, err := s.PutDocument(ctx, c, KBDocument{Name: name, Digest: name}, []KBChunk{{Body: body, Embedding: v}}); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
put("yillik.md", "yıllık izin kadar izin alır izin", unit(0))
|
||||||
|
put("babalik.md", "babalık hakkı on gündür", unit(1))
|
||||||
|
for _, f := range []string{"a", "b", "c"} {
|
||||||
|
put(f+".md", "başka bir konu", unit(2))
|
||||||
|
}
|
||||||
|
hits, err := s.SearchChunks(ctx, KBSearch{OrgID: "org1", Collections: []KBCollection{c},
|
||||||
|
Embeddings: map[string][]float32{"emb": unit(1)}, Text: "babalar ne kadar izin alır", K: 1})
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(hits) != 1 || hits[0].Document != "babalik.md" {
|
||||||
|
t.Errorf("best hit %+v", hits)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestOrQueryMatchesAnyWord(t *testing.T) {
|
func TestOrQueryMatchesAnyWord(t *testing.T) {
|
||||||
if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | kaç | gün | İzin" {
|
if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün | İzin" {
|
||||||
t.Errorf("orQuery gave %q", got)
|
t.Errorf("orQuery gave %q", got)
|
||||||
}
|
}
|
||||||
|
// Words every passage has are no evidence of anything.
|
||||||
|
if got := orQuery("Babalar NE kadar izin alır, ve İÇİN"); got != "Babalar | izin | alır" {
|
||||||
|
t.Errorf("stop words kept: %q", got)
|
||||||
|
}
|
||||||
if got := orQuery("?!"); got != "" {
|
if got := orQuery("?!"); got != "" {
|
||||||
t.Errorf("no words gave %q", got)
|
t.Errorf("no words gave %q", got)
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in new issue
Block a user