From bc5ee9f3aaf537be5fd867f780b96758b3c0faeb Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Thu, 8 Oct 2026 15:05:18 +0200 Subject: [PATCH] Knowledge search ranks by how near a chunk's meaning is, not by its place in two lists Fused by rank, a full-text match on a question's everyday words outvoted a far nearer meaning: on the PoC, "how much leave do fathers get" found the annual-leave page before the paternity one (cosine distance 0.44 against 0.25). Now a chunk scores its similarity to the question plus up to 0.3 for a full-text match, each search brings four times k candidates, the text search drops Turkish and English stop words, and an operator's search says when the embedding model did not answer rather than quietly searching text. Co-Authored-By: Claude Opus 5.5 (1M context) --- internal/control/adminapi/kb.go | 10 +- internal/store/knowledge.go | 178 ++++++++++++++++++++----------- internal/store/knowledge_test.go | 34 +++++- 3 files changed, 153 insertions(+), 69 deletions(-) diff --git a/internal/control/adminapi/kb.go b/internal/control/adminapi/kb.go index 4701e40..4471fa5 100644 --- a/internal/control/adminapi/kb.go +++ b/internal/control/adminapi/kb.go @@ -450,11 +450,15 @@ func (a *API) kbSearch(w http.ResponseWriter, r *http.Request) { k = 5 } q := store.KBSearch{OrgID: c.OrgID, Collections: []store.KBCollection{c}, Text: in.Query, K: k} - // Without the model, the full-text half still answers. + // Said, not swallowed: a search that quietly lost its meaning half + // looks like a collection that does not know the answer. if a.kb.Embed != nil { - if v, err := a.kb.Embed(r.Context(), c.EmbedModel, in.Query); err == nil { - q.Embeddings = map[string][]float32{c.EmbedModel: v} + v, err := a.kb.Embed(r.Context(), c.EmbedModel, in.Query) + if err != nil { + writeError(w, http.StatusBadGateway, c.EmbedModel+" did not embed the question: "+err.Error()) + return } + q.Embeddings = map[string][]float32{c.EmbedModel: v} } hits, err := a.kb.Store.SearchChunks(r.Context(), q) if err != nil { diff --git a/internal/store/knowledge.go b/internal/store/knowledge.go index 3354570..f41f595 100644 --- a/internal/store/knowledge.go +++ b/internal/store/knowledge.go @@ -89,8 +89,9 @@ type KBHit struct { Document string Ord int Body string - // Score is the fused rank (reciprocal rank fusion over the vector and - // text searches): only the order means anything. + // Score is the cosine similarity to the question, plus up to + // textWeight for a full-text match: higher is better, and only the + // order is meant to be compared. Score float64 } @@ -626,13 +627,22 @@ type KBSearch struct { K int } -// rrfK is reciprocal rank fusion's constant: 60, as in the paper and nearly -// everywhere it is used. -const rrfK = 60 +// textWeight is how much a full-text match adds to a chunk's similarity to +// the question, at most. The embedding model reads what a question means; +// full text is there for what it cannot — a code, a name, a number. Fused by +// rank, as first built, the two were equals and the gap between a close +// meaning and a distant one was thrown away: "how much leave do fathers +// get" found the annual-leave page over the paternity one on the PoC, +// though the paternity page was far nearer (0.25 against 0.44). +const textWeight = 0.3 + +// candidates is how many each search brings to the fusion for k results: +// with only k from each, the meaning search's third-best never met the +// text search's first, ties were broken by id, and the order was luck. +func candidates(k int) int { return max(4*k, 20) } // SearchChunks finds the best chunks for a query across collections: a -// nearest-neighbour search in each collection and a full-text search across -// them, fused by rank. Every collection must be in OrgID; the caller decides +// nearest-neighbour search and a full-text search in each, put on one scale. Every collection must be in OrgID; the caller decides // which ones the requester may read, and the table refuses rows outside them // regardless. func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) { @@ -643,86 +653,93 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) { return nil, nil } ids := make([]string, 0, len(q.Collections)) - quoted := make([]string, 0, len(q.Collections)) for _, c := range q.Collections { if c.OrgID != q.OrgID || !kbIDPattern.MatchString(c.ID) { return nil, fmt.Errorf("collection %s is not in organization %s", c.ID, q.OrgID) } ids = append(ids, c.ID) - quoted = append(quoted, sqlLiteral(c.ID)) } type key struct { doc string ord int } - hits := map[key]*KBHit{} - add := func(h KBHit, rank int) { - k := key{h.DocID, h.Ord} - if have, ok := hits[k]; ok { - have.Score += 1.0 / float64(rrfK+rank) - return - } - h.Score = 1.0 / float64(rrfK+rank) - hits[k] = &h + type found struct { + hit KBHit + // sim is the cosine similarity to the question, when the question + // was embedded for this chunk's collection; text is its full-text + // rank, 0 when the text search did not find it. + sim, text float64 + hasSim bool } - scan := func(rows *sql.Rows) ([]KBHit, error) { + hits := map[key]*found{} + add := func(h KBHit, sim sql.NullFloat64, text float64) { + k := key{h.DocID, h.Ord} + f, ok := hits[k] + if !ok { + f = &found{hit: h} + hits[k] = f + } + if sim.Valid { + f.sim, f.hasSim = sim.Float64, true + } + f.text = max(f.text, text) + } + query := func(tx conn, text bool, q string, args ...any) error { + rows, err := tx.QueryContext(ctx, q, args...) + if err != nil { + return err + } defer rows.Close() - var out []KBHit for rows.Next() { var h KBHit - if err := rows.Scan(&h.Collection, &h.DocID, &h.Document, &h.Ord, &h.Body); err != nil { - return nil, err + var sim sql.NullFloat64 + var rank float64 + if err := rows.Scan(&h.Collection, &h.DocID, &h.Document, &h.Ord, &h.Body, &sim, &rank); err != nil { + return err } - out = append(out, h) + if !text { + rank = 0 + } + add(h, sim, rank) } - return out, rows.Err() + return rows.Err() } + tsq := orQuery(q.Text) err := s.inTx(ctx, func(tx conn) error { if err := kbScope(ctx, tx, q.OrgID, ids); err != nil { return err } for _, c := range q.Collections { - v, ok := q.Embeddings[c.EmbedModel] - if !ok { - continue - } - if len(v) != c.Dims { - return fmt.Errorf("the query has %d dimensions; collection %s takes %d", len(v), c.Name, c.Dims) - } // The literal collection and cast are what let the planner use // that collection's partial index. - cast := `::vector(` + strconv.Itoa(c.Dims) + `)` - rows, err := tx.QueryContext(ctx, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body - FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id - WHERE k.org_id = ? AND k.collection = `+sqlLiteral(c.ID)+` - ORDER BY k.embedding`+cast+` <=> ?`+cast+` LIMIT ?`, - q.OrgID, vectorText(v), q.K) - if err != nil { - return err + lit := sqlLiteral(c.ID) + sim, args := "NULL::float8", []any{} + v, embedded := q.Embeddings[c.EmbedModel] + if embedded { + if len(v) != c.Dims { + return fmt.Errorf("the query has %d dimensions; collection %s takes %d", len(v), c.Name, c.Dims) + } + cast := `::vector(` + strconv.Itoa(c.Dims) + `)` + sim = `1 - (k.embedding` + cast + ` <=> ?` + cast + `)` + args = append(args, vectorText(v)) + if err := query(tx, false, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, 0::float8 + FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id + WHERE k.org_id = ? AND k.collection = `+lit+` + ORDER BY k.embedding`+cast+` <=> ?`+cast+` LIMIT ?`, + vectorText(v), q.OrgID, vectorText(v), candidates(q.K)); err != nil { + return err + } } - got, err := scan(rows) - if err != nil { - return err - } - for i, h := range got { - add(h, i+1) - } - } - if tsq := orQuery(q.Text); tsq != "" { - rows, err := tx.QueryContext(ctx, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body - FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) q - WHERE k.org_id = ? AND k.collection IN (`+strings.Join(quoted, ", ")+`) AND k.tsv @@ q - ORDER BY ts_rank(k.tsv, q) DESC LIMIT ?`, - tsq, q.OrgID, q.K) - if err != nil { - return err - } - got, err := scan(rows) - if err != nil { - return err - } - for i, h := range got { - add(h, i+1) + if tsq != "" { + // The text search's finds are measured against the question's + // meaning too, so the two are compared on one scale. + if err := query(tx, true, `SELECT k.collection, k.doc_id, d.name, k.ord, k.body, `+sim+`, ts_rank(k.tsv, tq)::float8 + FROM kb_chunks k JOIN kb_documents d ON d.id = k.doc_id, to_tsquery('simple', ?) tq + WHERE k.org_id = ? AND k.collection = `+lit+` AND k.tsv @@ tq + ORDER BY ts_rank(k.tsv, tq) DESC LIMIT ?`, + append(args, tsq, q.OrgID, candidates(q.K))...); err != nil { + return err + } } } return nil @@ -730,9 +747,21 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) { if err != nil { return nil, err } + // One scale: similarity, plus the text rank as a share of the best one, + // weighted down. + best := 0.0 + for _, f := range hits { + best = max(best, f.text) + } + for _, f := range hits { + f.hit.Score = f.sim + if best > 0 { + f.hit.Score += textWeight * f.text / best + } + } out := make([]KBHit, 0, len(hits)) - for _, h := range hits { - out = append(out, *h) + for _, f := range hits { + out = append(out, f.hit) } sort.Slice(out, func(i, j int) bool { if out[i].Score != out[j].Score { @@ -749,6 +778,25 @@ func (s *DB) SearchChunks(ctx context.Context, q KBSearch) ([]KBHit, error) { return out, nil } +// stopWords are words nearly every passage has, in Turkish and English: +// matching on them only ranks whatever is longest first. +var stopWords = func() map[string]bool { + m := map[string]bool{} + for _, w := range strings.Fields(` + acaba ama ancak artık aslında az bana bazı belki ben beni benim beri bile bir birçok biri birkaç + biz bize bizi bizim bu buna bunda bundan bunlar bunları bunların bunu bunun burada çok çünkü da + daha dahi de defa diye en gibi göre hangi hangisi hem hep hepsi her hiç için ile ise kaç kadar + ki kim kime kimi mı mi mu mü nasıl ne neden nerde nerede nereye niçin niye o olan olarak oldu + olduğu olsa olur ona ondan onlar onları onların onu onun orada öyle şey şu şuna şunda şundan şunu + tüm ve veya ya yani var yok mısın misin musun müsün midir mıdır nedir hangileri lütfen + a an and are as at be by can do does for from has have how i in is it its me my of on or our + should that the their there this to was what when where which who why will with you your + `) { + m[w] = true + } + return m +}() + // orQuery turns a question into a tsquery matching any of its words. All of // them (plainto_tsquery) would match almost nothing a person types as a // question; the rank then prefers chunks with more of them. @@ -761,7 +809,7 @@ func orQuery(text string) string { for _, w := range strings.FieldsFunc(text, func(r rune) bool { return !unicode.IsLetter(r) && !unicode.IsDigit(r) && !unicode.IsMark(r) }) { - if len([]rune(w)) < 2 || seen[w] { + if len([]rune(w)) < 2 || seen[w] || stopWords[strings.ToLowerSpecial(unicode.TurkishCase, w)] { continue } seen[w] = true diff --git a/internal/store/knowledge_test.go b/internal/store/knowledge_test.go index 4c67da9..061e3ea 100644 --- a/internal/store/knowledge_test.go +++ b/internal/store/knowledge_test.go @@ -352,10 +352,42 @@ func TestPostgresAPersonsCollectionsGoWithTheAccount(t *testing.T) { } } +func TestPostgresMeaningOutweighsSharedWords(t *testing.T) { + // The PoC's case: the question shares its everyday words with one + // passage and its meaning with another. + ctx := context.Background() + s := openKB(t) + kbTwoOrgs(t, s) + c := mustCollection(t, s, ScopeProject, "proj1", "", "docs") + put := func(name, body string, v []float32) { + t.Helper() + if _, _, err := s.PutDocument(ctx, c, KBDocument{Name: name, Digest: name}, []KBChunk{{Body: body, Embedding: v}}); err != nil { + t.Fatal(err) + } + } + put("yillik.md", "yıllık izin kadar izin alır izin", unit(0)) + put("babalik.md", "babalık hakkı on gündür", unit(1)) + for _, f := range []string{"a", "b", "c"} { + put(f+".md", "başka bir konu", unit(2)) + } + hits, err := s.SearchChunks(ctx, KBSearch{OrgID: "org1", Collections: []KBCollection{c}, + Embeddings: map[string][]float32{"emb": unit(1)}, Text: "babalar ne kadar izin alır", K: 1}) + if err != nil { + t.Fatal(err) + } + if len(hits) != 1 || hits[0].Document != "babalik.md" { + t.Errorf("best hit %+v", hits) + } +} + func TestOrQueryMatchesAnyWord(t *testing.T) { - if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | kaç | gün | İzin" { + if got := orQuery("Yıllık izin kaç gün? İzin & 'x' -- a"); got != "Yıllık | izin | gün | İzin" { t.Errorf("orQuery gave %q", got) } + // Words every passage has are no evidence of anything. + if got := orQuery("Babalar NE kadar izin alır, ve İÇİN"); got != "Babalar | izin | alır" { + t.Errorf("stop words kept: %q", got) + } if got := orQuery("?!"); got != "" { t.Errorf("no words gave %q", got) }