Knowledge search counts a text match by how many of the question's words it holds
The best text match took the whole text weight, however little of the
question it matched: one everyday word ("most") put the remote-work page
above the travel one for "the most I can spend on a hotel". Now a chunk
gains the weight in proportion to the share of the question's words it
holds, counted once whatever their case.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
bc5ee9f3aa
commit
2c7b682587
2 files changed
+40
-20
No files matched your search
+33
-19
@@ -666,8 +666,9 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
|||||||
type found struct {
|
type found struct {
|
||||||
hit KBHit
|
hit KBHit
|
||||||
// sim is the cosine similarity to the question, when the question
|
// sim is the cosine similarity to the question, when the question
|
||||||
// was embedded for this chunk's collection; text is its full-text
|
// was embedded for this chunk's collection; text is the share of
|
||||||
// rank, 0 when the text search did not find it.
|
// the question's words it holds, 0 when the text search did not
|
||||||
|
// find it.
|
||||||
sim, text float64
|
sim, text float64
|
||||||
hasSim bool
|
hasSim bool
|
||||||
}
|
}
|
||||||
@@ -704,7 +705,18 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
|||||||
}
|
}
|
||||||
return rows.Err()
|
return rows.Err()
|
||||||
}
|
}
|
||||||
tsq := orQuery(q.Text)
|
terms := queryTerms(q.Text)
|
||||||
|
tsq := strings.Join(terms, " | ")
|
||||||
|
// coverage is the share of the question's words a chunk holds: one
|
||||||
|
// everyday word in common is a little evidence, all of them a lot.
|
||||||
|
var coverage string
|
||||||
|
if len(terms) > 0 {
|
||||||
|
parts := make([]string, len(terms))
|
||||||
|
for i := range terms {
|
||||||
|
parts[i] = `(k.tsv @@ to_tsquery('simple', ?))::int`
|
||||||
|
}
|
||||||
|
coverage = `(` + strings.Join(parts, " + ") + `)::float8 / ` + strconv.Itoa(len(terms))
|
||||||
|
}
|
||||||
err := s.inTx(ctx, func(tx conn) error {
|
err := s.inTx(ctx, func(tx conn) error {
|
||||||
if err := kbScope(ctx, tx, q.OrgID, ids); err != nil {
|
if err := kbScope(ctx, tx, q.OrgID, ids); err != nil {
|
||||||
return err
|
return err
|
||||||
@@ -733,11 +745,15 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
|||||||
if tsq != "" {
|
if tsq != "" {
|
||||||
// The text search's finds are measured against the question's
|
// The text search's finds are measured against the question's
|
||||||
// meaning too, so the two are compared on one scale.
|
// meaning too, so the two are compared on one scale.
|
||||||
if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, ts_rank(k.tsv, tq)::float8
|
targs := append([]any{}, args...)
|
||||||
|
for _, t := range terms {
|
||||||
|
targs = append(targs, t)
|
||||||
|
}
|
||||||
|
targs = append(targs, tsq, q.OrgID, candidates(q.K))
|
||||||
|
if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, `+coverage+` AS coverage
|
||||||
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq
|
FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq
|
||||||
WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq
|
WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq
|
||||||
ORDER BY ts_rank(k.tsv, tq) DESC LIMIT ?`,
|
ORDER BY coverage DESC, ts_rank(k.tsv, tq) DESC LIMIT ?`, targs...); err != nil {
|
||||||
append(args, tsq, q.OrgID, candidates(q.K))...); err != nil {
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -747,17 +763,10 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
// One scale: similarity, plus the text rank as a share of the best one,
|
// One scale: similarity, plus the share of the question's words held,
|
||||||
// weighted down.
|
// weighted down.
|
||||||
best := 0.0
|
|
||||||
for _, f := range hits {
|
for _, f := range hits {
|
||||||
best = max(best, f.text)
|
f.hit.Score = f.sim + textWeight*f.text
|
||||||
}
|
|
||||||
for _, f := range hits {
|
|
||||||
f.hit.Score = f.sim
|
|
||||||
if best > 0 {
|
|
||||||
f.hit.Score += textWeight * f.text / best
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
out := make([]KBHit, 0, len(hits))
|
out := make([]KBHit, 0, len(hits))
|
||||||
for _, f := range hits {
|
for _, f := range hits {
|
||||||
@@ -800,7 +809,11 @@ var stopWords = func() map[string]bool {
|
|||||||
// orQuery turns a question into a tsquery matching any of its words. All of
|
// orQuery turns a question into a tsquery matching any of its words. All of
|
||||||
// them (plainto_tsquery) would match almost nothing a person types as a
|
// them (plainto_tsquery) would match almost nothing a person types as a
|
||||||
// question; the rank then prefers chunks with more of them.
|
// question; the rank then prefers chunks with more of them.
|
||||||
func orQuery(text string) string {
|
func orQuery(text string) string { return strings.Join(queryTerms(text), " | ") }
|
||||||
|
|
||||||
|
// queryTerms are a question's words that say something: not stop words, not
|
||||||
|
// single letters, each once whatever its case.
|
||||||
|
func queryTerms(text string) []string {
|
||||||
seen := map[string]bool{}
|
seen := map[string]bool{}
|
||||||
var words []string
|
var words []string
|
||||||
// Not lowercased here: to_tsquery does that with the database's own
|
// Not lowercased here: to_tsquery does that with the database's own
|
||||||
@@ -809,16 +822,17 @@ func orQuery(text string) string {
|
|||||||
for _, w := range strings.FieldsFunc(text, func(r rune) bool {
|
for _, w := range strings.FieldsFunc(text, func(r rune) bool {
|
||||||
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r)
|
return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r)
|
||||||
}) {
|
}) {
|
||||||
if len([]rune(w)) < 2 || seen[w] || stopWords[strings.ToLowerSpecial(unicode.TurkishCase, w)] {
|
low := strings.ToLowerSpecial(unicode.TurkishCase, w)
|
||||||
|
if len([]rune(w)) < 2 || seen[low] || stopWords[low] {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
seen[w] = true
|
seen[low] = true
|
||||||
words = append(words, w)
|
words = append(words, w)
|
||||||
if len(words) == 32 {
|
if len(words) == 32 {
|
||||||
break
|
break
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return strings.Join(words, " | ")
|
return words
|
||||||
}
|
}
|
||||||
|
|
||||||
// kbOwnedBy counts the collections a scope owns, for refusing to delete it
|
// kbOwnedBy counts the collections a scope owns, for refusing to delete it
|
||||||
|
|||||||
@@ -378,10 +378,16 @@ func TestPostgresMeaningOutweighsSharedWords(t *testing.T) {
|
|||||||
if len(hits) != 1 || hits[0].Document != "babalik.md" {
|
if len(hits) != 1 || hits[0].Document != "babalik.md" {
|
||||||
t.Errorf("best hit %+v", hits)
|
t.Errorf("best hit %+v", hits)
|
||||||
}
|
}
|
||||||
|
// And a question that holds a passage's exact words still finds it,
|
||||||
|
// with nothing embedded at all (the model down).
|
||||||
|
hits, err = s.SearchChunks(ctx, KBSearch{OrgID: "org1", Collections: []KBCollection{c}, Text: "babalık hakkı", K: 1})
|
||||||
|
if err != nil || len(hits) != 1 || hits[0].Document != "babalik.md" || hits[0].Score != textWeight {
|
||||||
|
t.Errorf("text alone: %+v %v", hits, err)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestOrQueryMatchesAnyWord(t *testing.T) {
|
func TestOrQueryMatchesAnyWord(t *testing.T) {
|
||||||
if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün | İzin" {
|
if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün" {
|
||||||
t.Errorf("orQuery gave %q", got)
|
t.Errorf("orQuery gave %q", got)
|
||||||
}
|
}
|
||||||
// Words every passage has are no evidence of anything.
|
// Words every passage has are no evidence of anything.
|
||||||
|
|||||||
Reference in new issue
Block a user