Files
llmgw/internal/router/kb_test.go
T
Ilker ManapandClaude Opus 5.5 496625824b Chat requests answered from knowledge bases
A request naming collections in X-Llmgw-Knowledge has its last question
embedded on the caller's account, the best passages found and added to the
system message, numbered, with an instruction to cite them and to say when
they do not hold the answer; x-llmgw-sources says which were given. Done
before the guardrails judge the request, never served from the answer cache,
and a collection the key may not read is answered as one that does not
exist. Metrics llmgw_kb_retrievals_total and llmgw_kb_retrieval_seconds.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
2026-10-08 15:27:50 +02:00

209 lines
8.0 KiB
Go

package router
import (
"crypto/sha256"
"net/http"
"net/http/httptest"
"net/url"
"os"
"strings"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
)
func TestKnowledgeBasesSayWhyTheyAreOffOnSQLite(t *testing.T) {
e := newFakeEngine(t, "qwen2.5-14b-instruct")
rt, secret := pricedRouter(t, e.srv.URL)
code, _, body := call(t, rt.Handler(), secret, http.MethodGet, "/v1/kb/collections", nil)
if code != http.StatusNotImplemented || !strings.Contains(body, "SQLite") {
t.Errorf("got %d %s", code, body)
}
}
// kbRouter is a router on the PostgreSQL test database, with pgvector.
func kbRouter(t *testing.T) *Router {
t.Helper()
return kbRouterWith(t, newFakeEngine(t, "qwen2.5-14b-instruct").srv.URL)
}
// kbRouterWith is a router on the PostgreSQL test database, with pgvector,
// in front of an engine at engineURL.
func kbRouterWith(t *testing.T, engineURL string) *Router {
t.Helper()
dsn := os.Getenv("LLMGW_TEST_POSTGRES")
if dsn == "" {
t.Skip("set LLMGW_TEST_POSTGRES to a pgvector database to run these")
}
if u, err := url.Parse(dsn); err != nil || !strings.Contains(u.Path, "test") {
t.Fatalf("refusing a database whose name does not say test: %s", dsn)
}
cfg := baseConfig(config.Endpoint{ID: "a", URL: engineURL, Models: []string{"qwen2.5-14b-instruct"}})
cfg.Auth.Enabled = true
cfg.Auth.Store = dsn
rt := newTestRouter(t, cfg)
if err := rt.store.KnowledgeAvailable(); err != nil {
t.Skipf("no pgvector in the test database: %v", err)
}
return rt
}
// kbKey makes a key in a project, or a person's key with no project.
func kbKey(t *testing.T, s *store.DB, project, user string) string {
t.Helper()
secret := store.KeyPrefix + strings.ToLower(store.NewID("")) + strings.Repeat("k", 27)
sum := sha256.Sum256([]byte(secret))
k := store.Key{ID: store.NewID("key-"), Hash: sum[:], Prefix: secret[:8], Name: "t", ProjectID: project, CreatedAt: time.Now()}
var err error
if user != "" {
err = s.CreateUserKey(t.Context(), user, k)
} else {
err = s.CreateKey(t.Context(), k)
}
if err != nil {
t.Fatal(err)
}
return secret
}
func vec(x ...float32) []float32 { return x }
func doc(name, digest string, embeddings ...[]float32) map[string]any {
var chunks []map[string]any
for i, e := range embeddings {
chunks = append(chunks, map[string]any{"text": name + " parça " + string(rune('a'+i)), "embedding": e})
}
return map[string]any{"name": name, "digest": digest, "bytes": 10, "chunks": chunks}
}
func TestKnowledgeBasesKeepTenantsApart(t *testing.T) {
rt := kbRouter(t)
h := rt.Handler()
ctx := t.Context()
s := rt.store
// Unique names: the test database is shared and not emptied here.
sfx := strings.ToLower(store.NewID(""))
orgA, _ := s.CreateOrg(ctx, "a-"+sfx)
teamA, _ := s.CreateTeam(ctx, orgA, "t")
hr, _ := s.CreateProject(ctx, teamA, "hr")
sales, _ := s.CreateProject(ctx, teamA, "sales")
orgB, _ := s.CreateOrg(ctx, "b-"+sfx)
teamB, _ := s.CreateTeam(ctx, orgB, "t")
other, _ := s.CreateProject(ctx, teamB, "x")
c, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeProject, OwnerID: hr,
Name: "politikalar", EmbedModel: "emb", Dims: 3})
if err != nil {
t.Fatal(err)
}
hrKey, salesKey, otherKey := kbKey(t, s, hr, ""), kbKey(t, s, sales, ""), kbKey(t, s, other, "")
base := "/v1/kb/collections/"
// The owner's project writes, by name or by id.
code, out, body := call(t, h, hrKey, http.MethodPut, base+"politikalar/documents", doc("izin.pdf", "d1", vec(1, 0, 0), vec(0, 1, 0)))
if code != http.StatusOK || out["changed"] != true || out["chunks"] != float64(2) {
t.Fatalf("owner's put: %d %s", code, body)
}
code, out, _ = call(t, h, hrKey, http.MethodPut, base+c.ID+"/documents", doc("izin.pdf", "d1", vec(1, 0, 0)))
if code != http.StatusOK || out["changed"] != false {
t.Errorf("the same digest again: %d %v", code, out)
}
code, _, body = call(t, h, hrKey, http.MethodGet, base+"politikalar/manifest", nil)
if code != http.StatusOK || !strings.Contains(body, `"digest":"d1"`) || !strings.Contains(body, `"writable":true`) {
t.Errorf("manifest: %d %s", code, body)
}
// Another organization, and another project without a grant, see no
// such collection — not a refusal that says it exists.
for name, key := range map[string]string{"other org": otherKey, "other project": salesKey} {
for _, m := range []string{http.MethodGet + " manifest", http.MethodPut + " documents"} {
method, path, _ := strings.Cut(m, " ")
code, _, body := call(t, h, key, method, base+c.ID+"/"+path, doc("x", "y", vec(1, 0, 0)))
if code != http.StatusNotFound {
t.Errorf("%s %s: %d %s", name, m, code, body)
}
}
}
// A read grant lets sales read, not write.
if err := s.Grant(ctx, store.KBGrant{Scope: store.ScopeProject, ScopeID: sales, Collection: c.ID, Perm: store.KBRead}); err != nil {
t.Fatal(err)
}
if code, _, body := call(t, h, salesKey, http.MethodGet, base+"politikalar/manifest", nil); code != http.StatusOK ||
!strings.Contains(body, `"writable":false`) {
t.Errorf("read grant, manifest: %d %s", code, body)
}
if code, _, _ := call(t, h, salesKey, http.MethodPut, base+"politikalar/documents", doc("x", "y", vec(1, 0, 0))); code != http.StatusNotFound {
t.Errorf("read grant, put: %d", code)
}
if code, _, body := call(t, h, salesKey, http.MethodGet, "/v1/kb/collections", nil); code != http.StatusOK ||
!strings.Contains(body, "politikalar") {
t.Errorf("read grant, list: %d %s", code, body)
}
if _, _, body := call(t, h, otherKey, http.MethodGet, "/v1/kb/collections", nil); strings.Contains(body, "politikalar") {
t.Errorf("another organization lists it: %s", body)
}
// Vectors of the wrong size are refused before anything is written.
if code, _, body := call(t, h, hrKey, http.MethodPut, base+"politikalar/documents", doc("b.md", "d", vec(1, 0))); code != http.StatusBadRequest ||
!strings.Contains(body, "dimensions") {
t.Errorf("wrong dimensions: %d %s", code, body)
}
if code, _, body := call(t, h, hrKey, http.MethodDelete, base+"politikalar/documents?name=izin.pdf", nil); code != http.StatusOK {
t.Errorf("delete: %d %s", code, body)
}
if code, _, _ := call(t, h, hrKey, http.MethodDelete, base+"politikalar/documents?name=izin.pdf", nil); code != http.StatusNotFound {
t.Errorf("deleting again: %d", code)
}
}
func TestAPersonalKeyReadsOnlyInTheProjectItNames(t *testing.T) {
rt := kbRouter(t)
h := rt.Handler()
ctx := t.Context()
s := rt.store
sfx := strings.ToLower(store.NewID(""))
org, _ := s.CreateOrg(ctx, "p-"+sfx)
team, _ := s.CreateTeam(ctx, org, "t")
hr, _ := s.CreateProject(ctx, team, "hr")
fin, _ := s.CreateProject(ctx, team, "fin")
uid := store.NewID("u-")
if err := s.CreateUser(ctx, store.User{ID: uid, Email: uid + "@example.com", Name: "u", PasswordHash: "x",
State: store.UserActive, CreatedAt: time.Now()}); err != nil {
t.Fatal(err)
}
if err := s.AddMembership(ctx, uid, store.ScopeProject, hr, "member", "t", time.Now()); err != nil {
t.Fatal(err)
}
if _, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeProject, OwnerID: hr,
Name: "ik", EmbedModel: "emb", Dims: 3}); err != nil {
t.Fatal(err)
}
if _, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeProject, OwnerID: fin,
Name: "finans", EmbedModel: "emb", Dims: 3}); err != nil {
t.Fatal(err)
}
key := kbKey(t, s, "", uid)
get := func(project string) (int, string) {
req, _ := http.NewRequest(http.MethodGet, "/v1/kb/collections", nil)
req.Header.Set("Authorization", "Bearer "+key)
if project != "" {
req.Header.Set(projectHeader, project)
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, req)
return rec.Code, rec.Body.String()
}
if code, body := get(""); code != http.StatusOK || strings.Contains(body, "ik") {
t.Errorf("no project named: %d %s", code, body)
}
if code, body := get(hr); code != http.StatusOK || !strings.Contains(body, `"ik"`) || strings.Contains(body, "finans") {
t.Errorf("in hr: %d %s", code, body)
}
if code, _ := get(fin); code != http.StatusForbidden {
t.Errorf("a project the person is not in: %d", code)
}
}