Knowledge search counts a text match by how many of the question's words it holds

The best text match took the whole text weight, however little of the
question it matched: one everyday word ("most") put the remote-work page
above the travel one for "the most I can spend on a hotel". Now a chunk
gains the weight in proportion to the share of the question's words it
holds, counted once whatever their case.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 15:09:24 +02:00
1 parent bc5ee9f3aa
commit 2c7b682587
2 files changed
+40 -20

No files matched your search

+33 -19
View File
@@ -666,8 +666,9 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
type found struct { type found struct {
hit KBHit hit KBHit
// sim is the cosine similarity to the question, when the question // sim is the cosine similarity to the question, when the question
// was embedded for this chunk's collection; text is its full-text // was embedded for this chunk's collection; text is the share of
// rank, 0 when the text search did not find it. // the question's words it holds, 0 when the text search did not
// find it.
sim, text float64 sim, text float64
hasSim bool hasSim bool
} }
@@ -704,7 +705,18 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
} }
return rows.Err() return rows.Err()
} }
tsq := orQuery(q.Text) terms := queryTerms(q.Text)
tsq := strings.Join(terms, " | ")
// coverage is the share of the question's words a chunk holds: one
// everyday word in common is a little evidence, all of them a lot.
var coverage string
if len(terms) > 0 {
parts := make([]string, len(terms))
for i := range terms {
parts[i] = `(k.tsv @@ to_tsquery('simple', ?))::int`
}
coverage = `(` + strings.Join(parts, " + ") + `)::float8 / ` + strconv.Itoa(len(terms))
}
err := s.inTx(ctx, func(tx conn) error { err := s.inTx(ctx, func(tx conn) error {
if err := kbScope(ctx, tx, q.OrgID, ids); err != nil { if err := kbScope(ctx, tx, q.OrgID, ids); err != nil {
return err return err
@@ -733,11 +745,15 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
if tsq != "" { if tsq != "" {
// The text search's finds are measured against the question's // The text search's finds are measured against the question's
// meaning too, so the two are compared on one scale. // meaning too, so the two are compared on one scale.
if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, ts_rank(k.tsv, tq)::float8 targs := append([]any{}, args...)
for _, t := range terms {
targs = append(targs, t)
}
targs = append(targs, tsq, q.OrgID, candidates(q.K))
if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, `+coverage+` AS coverage
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq
WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq
ORDER BY ts_rank(k.tsv, tq) DESC LIMIT ?`, ORDER BY coverage DESC, ts_rank(k.tsv, tq) DESC LIMIT ?`, targs...); err != nil {
append(args, tsq, q.OrgID, candidates(q.K))...); err != nil {
return err return err
} }
} }
@@ -747,17 +763,10 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
if err != nil { if err != nil {
return nil, err return nil, err
} }
// One scale: similarity, plus the text rank as a share of the best one, // One scale: similarity, plus the share of the question's words held,
// weighted down. // weighted down.
best := 0.0
for _, f := range hits { for _, f := range hits {
best = max(best, f.text) f.hit.Score = f.sim + textWeight*f.text
}
for _, f := range hits {
f.hit.Score = f.sim
if best > 0 {
f.hit.Score += textWeight * f.text / best
}
} }
out := make([]KBHit, 0, len(hits)) out := make([]KBHit, 0, len(hits))
for _, f := range hits { for _, f := range hits {
@@ -800,7 +809,11 @@ var stopWords = func() map[string]bool {
// orQuery turns a question into a tsquery matching any of its words. All of // orQuery turns a question into a tsquery matching any of its words. All of
// them (plainto_tsquery) would match almost nothing a person types as a // them (plainto_tsquery) would match almost nothing a person types as a
// question; the rank then prefers chunks with more of them. // question; the rank then prefers chunks with more of them.
func orQuery(text string) string { func orQuery(text string) string { return strings.Join(queryTerms(text), " | ") }
// queryTerms are a question's words that say something: not stop words, not
// single letters, each once whatever its case.
func queryTerms(text string) []string {
seen := map[string]bool{} seen := map[string]bool{}
var words []string var words []string
// Not lowercased here: to_tsquery does that with the database's own // Not lowercased here: to_tsquery does that with the database's own
@@ -809,16 +822,17 @@ func orQuery(text string) string {
for _, w := range strings.FieldsFunc(text, func(r rune) bool { for _, w := range strings.FieldsFunc(text, func(r rune) bool {
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r) return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r)
}) { }) {
if len([]rune(w)) < 2 || seen[w] || stopWords[strings.ToLowerSpecial(unicode.TurkishCase, w)] { low := strings.ToLowerSpecial(unicode.TurkishCase, w)
if len([]rune(w)) < 2 || seen[low] || stopWords[low] {
continue continue
} }
seen[w] = true seen[low] = true
words = append(words, w) words = append(words, w)
if len(words) == 32 { if len(words) == 32 {
break break
} }
} }
return strings.Join(words, " | ") return words
} }
// kbOwnedBy counts the collections a scope owns, for refusing to delete it // kbOwnedBy counts the collections a scope owns, for refusing to delete it
+7 -1
View File
@@ -378,10 +378,16 @@ func TestPostgresMeaningOutweighsSharedWords(t *testing.T) {
if len(hits) != 1 || hits[0].Document != "babalik.md" { if len(hits) != 1 || hits[0].Document != "babalik.md" {
t.Errorf("best hit %+v", hits) t.Errorf("best hit %+v", hits)
} }
// And a question that holds a passage's exact words still finds it,
// with nothing embedded at all (the model down).
hits, err = s.SearchChunks(ctx, KBSearch{OrgID: "org1", Collections: []KBCollection{c}, Text: "babalık hakkı", K: 1})
if err != nil || len(hits) != 1 || hits[0].Document != "babalik.md" || hits[0].Score != textWeight {
t.Errorf("text alone: %+v %v", hits, err)
}
} }
func TestOrQueryMatchesAnyWord(t *testing.T) { func TestOrQueryMatchesAnyWord(t *testing.T) {
if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün | İzin" { if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün" {
t.Errorf("orQuery gave %q", got) t.Errorf("orQuery gave %q", got)
} }
// Words every passage has are no evidence of anything. // Words every passage has are no evidence of anything.