Knowledge search counts a text match by how many of the question's words it holds

The best text match took the whole text weight, however little of the
question it matched: one everyday word ("most") put the remote-work page
above the travel one for "the most I can spend on a hotel". Now a chunk
gains the weight in proportion to the share of the question's words it
holds, counted once whatever their case.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

internal/store/knowledge.go

@@ -666,8 +666,9 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
 	type found struct {
 		hit KBHit
 		// sim is the cosine similarity to the question, when the question
-		// was embedded for this chunk's collection; text is its full-text
-		// rank, 0 when the text search did not find it.
+		// was embedded for this chunk's collection; text is the share of
+		// the question's words it holds, 0 when the text search did not
+		// find it.
 		sim, text float64
 		hasSim    bool
 	}
@@ -704,7 +705,18 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
 		}
 		return rows.Err()
 	}
-	tsq := orQuery(q.Text)
+	terms := queryTerms(q.Text)
+	tsq := strings.Join(terms, " | ")
+	// coverage is the share of the question's words a chunk holds: one
+	// everyday word in common is a little evidence, all of them a lot.
+	var coverage string
+	if len(terms) > 0 {
+		parts := make([]string, len(terms))
+		for i := range terms {
+			parts[i] = `(k.tsv @@ to_tsquery('simple', ?))::int`
+		}
+		coverage = `(` + strings.Join(parts, " + ") + `)::float8 / ` + strconv.Itoa(len(terms))
+	}
 	err := s.inTx(ctx, func(tx conn) error {
 		if err := kbScope(ctx, tx, q.OrgID, ids); err != nil {
 			return err
@@ -733,11 +745,15 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
 			if tsq != "" {
 				// The text search's finds are measured against the question's
 				// meaning too, so the two are compared on one scale.
-				if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, ts_rank(k.tsv, tq)::float8
+				targs := append([]any{}, args...)
+				for _, t := range terms {
+					targs = append(targs, t)
+				}
+				targs = append(targs, tsq, q.OrgID, candidates(q.K))
+				if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, `+coverage+` AS coverage
 					FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq
 					WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq
-					ORDER BY ts_rank(k.tsv, tq) DESC LIMIT ?`,
-					append(args, tsq, q.OrgID, candidates(q.K))...); err != nil {
+					ORDER BY coverage DESC, ts_rank(k.tsv, tq) DESC LIMIT ?`, targs...); err != nil {
 					return err
 				}
 			}
@@ -747,17 +763,10 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
 	if err != nil {
 		return nil, err
 	}
-	// One scale: similarity, plus the text rank as a share of the best one,
+	// One scale: similarity, plus the share of the question's words held,
 	// weighted down.
-	best := 0.0
 	for _, f := range hits {
-		best = max(best, f.text)
-	}
-	for _, f := range hits {
-		f.hit.Score = f.sim
-		if best > 0 {
-			f.hit.Score += textWeight * f.text / best
-		}
+		f.hit.Score = f.sim + textWeight*f.text
 	}
 	out := make([]KBHit, 0, len(hits))
 	for _, f := range hits {
@@ -800,7 +809,11 @@ var stopWords = func() map[string]bool {
 // orQuery turns a question into a tsquery matching any of its words. All of
 // them (plainto_tsquery) would match almost nothing a person types as a
 // question; the rank then prefers chunks with more of them.
-func orQuery(text string) string {
+func orQuery(text string) string { return strings.Join(queryTerms(text), " | ") }
+
+// queryTerms are a question's words that say something: not stop words, not
+// single letters, each once whatever its case.
+func queryTerms(text string) []string {
 	seen := map[string]bool{}
 	var words []string
 	// Not lowercased here: to_tsquery does that with the database's own
@@ -809,16 +822,17 @@ func orQuery(text string) string {
 	for _, w := range strings.FieldsFunc(text, func(r rune) bool {
 		return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r)
 	}) {
-		if len([]rune(w)) < 2 || seen[w] || stopWords[strings.ToLowerSpecial(unicode.TurkishCase, w)] {
+		low := strings.ToLowerSpecial(unicode.TurkishCase, w)
+		if len([]rune(w)) < 2 || seen[low] || stopWords[low] {
 			continue
 		}
-		seen[w] = true
+		seen[low] = true
 		words = append(words, w)
 		if len(words) == 32 {
 			break
 		}
 	}
-	return strings.Join(words, " | ")
+	return words
 }
 
 // kbOwnedBy counts the collections a scope owns, for refusing to delete it

internal/store/knowledge_test.go

@@ -378,10 +378,16 @@ func TestPostgresMeaningOutweighsSharedWords(t *testing.T) {
 	if len(hits) != 1 || hits[0].Document != "babalik.md" {
 		t.Errorf("best hit %+v", hits)
 	}
+	// And a question that holds a passage's exact words still finds it,
+	// with nothing embedded at all (the model down).
+	hits, err = s.SearchChunks(ctx, KBSearch{OrgID: "org1", Collections: []KBCollection{c}, Text: "babalık hakkı", K: 1})
+	if err != nil || len(hits) != 1 || hits[0].Document != "babalik.md" || hits[0].Score != textWeight {
+		t.Errorf("text alone: %+v %v", hits, err)
+	}
 }
 
 func TestOrQueryMatchesAnyWord(t *testing.T) {
-	if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün | İzin" {
+	if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün" {
 		t.Errorf("orQuery gave %q", got)
 	}
 	// Words every passage has are no evidence of anything.