diff --git a/internal/store/knowledge.go b/internal/store/knowledge.go index f41f595..baca380 100644 --- a/internal/store/knowledge.go +++ b/internal/store/knowledge.go @@ -666,8 +666,9 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) { type found struct { hit KBHit // sim is the cosine similarity to the question, when the question - // was embedded for this chunk's collection; text is its full-text - // rank, 0 when the text search did not find it. + // was embedded for this chunk's collection; text is the share of + // the question's words it holds, 0 when the text search did not + // find it. sim, text float64 hasSim bool } @@ -704,7 +705,18 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) { } return rows.Err() } - tsq := orQuery(q.Text) + terms := queryTerms(q.Text) + tsq := strings.Join(terms, " | ") + // coverage is the share of the question's words a chunk holds: one + // everyday word in common is a little evidence, all of them a lot. + var coverage string + if len(terms) > 0 { + parts := make([]string, len(terms)) + for i := range terms { + parts[i] = `(k.tsv @@ to_tsquery('simple', ?))::int` + } + coverage = `(` + strings.Join(parts, " + ") + `)::float8 / ` + strconv.Itoa(len(terms)) + } err := s.inTx(ctx, func(tx conn) error { if err := kbScope(ctx, tx, q.OrgID, ids); err != nil { return err @@ -733,11 +745,15 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) { if tsq != "" { // The text search's finds are measured against the question's // meaning too, so the two are compared on one scale. - if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, ts_rank(k.tsv, tq)::float8 + targs := append([]any{}, args...) + for _, t := range terms { + targs = append(targs, t) + } + targs = append(targs, tsq, q.OrgID, candidates(q.K)) + if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, `+coverage+` AS coverage FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq - ORDER BY ts_rank(k.tsv, tq) DESC LIMIT ?`, - append(args, tsq, q.OrgID, candidates(q.K))...); err != nil { + ORDER BY coverage DESC, ts_rank(k.tsv, tq) DESC LIMIT ?`, targs...); err != nil { return err } } @@ -747,17 +763,10 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) { if err != nil { return nil, err } - // One scale: similarity, plus the text rank as a share of the best one, + // One scale: similarity, plus the share of the question's words held, // weighted down. - best := 0.0 for _, f := range hits { - best = max(best, f.text) - } - for _, f := range hits { - f.hit.Score = f.sim - if best > 0 { - f.hit.Score += textWeight * f.text / best - } + f.hit.Score = f.sim + textWeight*f.text } out := make([]KBHit, 0, len(hits)) for _, f := range hits { @@ -800,7 +809,11 @@ var stopWords = func() map[string]bool { // orQuery turns a question into a tsquery matching any of its words. All of // them (plainto_tsquery) would match almost nothing a person types as a // question; the rank then prefers chunks with more of them. -func orQuery(text string) string { +func orQuery(text string) string { return strings.Join(queryTerms(text), " | ") } + +// queryTerms are a question's words that say something: not stop words, not +// single letters, each once whatever its case. +func queryTerms(text string) []string { seen := map[string]bool{} var words []string // Not lowercased here: to_tsquery does that with the database's own @@ -809,16 +822,17 @@ func orQuery(text string) string { for _, w := range strings.FieldsFunc(text, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r) }) { - if len([]rune(w)) < 2 || seen[w] || stopWords[strings.ToLowerSpecial(unicode.TurkishCase, w)] { + low := strings.ToLowerSpecial(unicode.TurkishCase, w) + if len([]rune(w)) < 2 || seen[low] || stopWords[low] { continue } - seen[w] = true + seen[low] = true words = append(words, w) if len(words) == 32 { break } } - return strings.Join(words, " | ") + return words } // kbOwnedBy counts the collections a scope owns, for refusing to delete it diff --git a/internal/store/knowledge_test.go b/internal/store/knowledge_test.go index 061e3ea..ef05a79 100644 --- a/internal/store/knowledge_test.go +++ b/internal/store/knowledge_test.go @@ -378,10 +378,16 @@ func TestPostgresMeaningOutweighsSharedWords(t *testing.T) { if len(hits) != 1 || hits[0].Document != "babalik.md" { t.Errorf("best hit %+v", hits) } + // And a question that holds a passage's exact words still finds it, + // with nothing embedded at all (the model down). + hits, err = s.SearchChunks(ctx, KBSearch{OrgID: "org1", Collections: []KBCollection{c}, Text: "babalık hakkı", K: 1}) + if err != nil || len(hits) != 1 || hits[0].Document != "babalik.md" || hits[0].Score != textWeight { + t.Errorf("text alone: %+v %v", hits, err) + } } func TestOrQueryMatchesAnyWord(t *testing.T) { - if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün | İzin" { + if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün" { t.Errorf("orQuery gave %q", got) } // Words every passage has are no evidence of anything.