Knowledge bases on the client listener: list, manifest, put and delete a document
What llmgw-kb, run by a person with their own key, reads and writes. A collection the key may not read answers as one that does not exist; writing needs ownership or a write grant on the key's chain; a personal key names its project in X-Llmgw-Project and must be a member of it. Changes are audited by document name. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a0cfafe772
commit
d9e030e0b6
3 files changed
+564
No files matched your search
@@ -121,6 +121,8 @@ func (r *Router) Handler() http.Handler {
|
||||
mux.HandleFunc("GET /v1/batches", r.handleListBatches)
|
||||
mux.HandleFunc("GET /v1/batches/{id}", r.handleGetBatch)
|
||||
mux.HandleFunc("POST /v1/batches/{id}/cancel", r.handleCancelBatch)
|
||||
// Knowledge bases: documents a person's llmgw-kb read and embedded.
|
||||
r.registerKB(mux)
|
||||
// Liveness: this process is up. It says nothing about whether it can
|
||||
// serve, which is why it is not the thing to put a load balancer on.
|
||||
mux.HandleFunc("GET /health", func(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
@@ -0,0 +1,360 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
"unicode"
|
||||
"unicode/utf8"
|
||||
|
||||
"golang.org/x/text/unicode/norm"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/auth"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
storepkg "gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// Knowledge bases on the client listener (BACKLOG F2): what llmgw-kb, run by
|
||||
// a person on their own machine with their own key, reads and writes. The
|
||||
// documents are read, cut and embedded there; what arrives here is a
|
||||
// document's name, digest and chunks with their vectors.
|
||||
//
|
||||
// Which collections a caller may see is decided from the key's place in the
|
||||
// tree and nothing the request says, beyond naming a project for a personal
|
||||
// key. A collection the caller may not read is answered exactly as one that
|
||||
// does not exist.
|
||||
|
||||
// projectHeader names the project a personal key acts in; the person must be
|
||||
// a member of it.
|
||||
const projectHeader = "X-Llmgw-Project"
|
||||
|
||||
// Bounds on one document, so one request cannot hold the control plane's
|
||||
// memory or a transaction for long.
|
||||
const (
|
||||
kbMaxBody = 64 << 20
|
||||
kbMaxChunks = 20000
|
||||
kbMaxChunkRunes = 8000
|
||||
kbMaxNameRunes = 1024
|
||||
)
|
||||
|
||||
// KBCollectionView is a collection as a caller sees it.
|
||||
type KBCollectionView struct {
|
||||
ID string `json:"id"`
|
||||
Name string `json:"name"`
|
||||
Owner string `json:"owner"`
|
||||
EmbedModel string `json:"embed_model"`
|
||||
Dims int `json:"dims"`
|
||||
// Writable is whether this caller may add and remove documents.
|
||||
Writable bool `json:"writable"`
|
||||
}
|
||||
|
||||
// KBDocumentView is a document as the manifest lists it.
|
||||
type KBDocumentView struct {
|
||||
Name string `json:"name"`
|
||||
Digest string `json:"digest"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Chunks int `json:"chunks"`
|
||||
AddedAt time.Time `json:"added_at"`
|
||||
}
|
||||
|
||||
// KBManifest is a collection and its documents: what a sync compares the
|
||||
// files on disk with.
|
||||
type KBManifest struct {
|
||||
Collection KBCollectionView `json:"collection"`
|
||||
Documents []KBDocumentView `json:"documents"`
|
||||
}
|
||||
|
||||
// KBPutDocument is one document, read and embedded by the client.
|
||||
type KBPutDocument struct {
|
||||
Name string `json:"name"`
|
||||
Digest string `json:"digest"`
|
||||
Bytes int64 `json:"bytes"`
|
||||
Chunks []struct {
|
||||
Text string `json:"text"`
|
||||
Embedding []float32 `json:"embedding"`
|
||||
} `json:"chunks"`
|
||||
}
|
||||
|
||||
// KBPutResult is what adding a document did.
|
||||
type KBPutResult struct {
|
||||
Name string `json:"name"`
|
||||
Chunks int `json:"chunks"`
|
||||
Changed bool `json:"changed"`
|
||||
}
|
||||
|
||||
func (r *Router) registerKB(mux *http.ServeMux) {
|
||||
mux.HandleFunc("GET /v1/kb/collections", r.handleKBCollections)
|
||||
mux.HandleFunc("GET /v1/kb/collections/{id}/manifest", r.handleKBManifest)
|
||||
mux.HandleFunc("PUT /v1/kb/collections/{id}/documents", r.handleKBPut)
|
||||
mux.HandleFunc("DELETE /v1/kb/collections/{id}/documents", r.handleKBDelete)
|
||||
}
|
||||
|
||||
// kbCaller is the caller's place in the tree, for deciding what it may read.
|
||||
// A personal key has none unless it names a project it is a member of.
|
||||
func (r *Router) kbCaller(w http.ResponseWriter, req *http.Request) (auth.Principal, storepkg.Effective, bool) {
|
||||
p, ok := auth.FromContext(req.Context())
|
||||
if !ok || r.store == nil {
|
||||
writeError(w, http.StatusNotImplemented, "invalid_request_error",
|
||||
"knowledge bases need authentication and the store, which this gateway runs without")
|
||||
return p, storepkg.Effective{}, false
|
||||
}
|
||||
if err := r.store.KnowledgeAvailable(); err != nil {
|
||||
writeError(w, http.StatusNotImplemented, "invalid_request_error", err.Error())
|
||||
return p, storepkg.Effective{}, false
|
||||
}
|
||||
e := storepkg.Effective{KeyID: p.KeyID, UserID: p.UserID, ProjectID: p.ProjectID, TeamID: p.TeamID, OrgID: p.OrgID}
|
||||
if proj := strings.TrimSpace(req.Header.Get(projectHeader)); proj != "" && p.ProjectID == "" && p.UserID != "" {
|
||||
may, err := r.store.CanUseProject(req.Context(), p.UserID, proj)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return p, e, false
|
||||
}
|
||||
if !may {
|
||||
writeError(w, http.StatusForbidden, "permission_error", "you are not a member of project "+proj)
|
||||
return p, e, false
|
||||
}
|
||||
eff, err := r.store.ResolveFor(req.Context(), p.KeyID, p.UserID, proj)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return p, e, false
|
||||
}
|
||||
e.ProjectID, e.TeamID, e.OrgID = eff.ProjectID, eff.TeamID, eff.OrgID
|
||||
}
|
||||
return p, e, true
|
||||
}
|
||||
|
||||
func kbView(c storepkg.KBCollection, writable bool) KBCollectionView {
|
||||
return KBCollectionView{ID: c.ID, Name: c.Name, Owner: string(c.OwnerScope) + ":" + c.OwnerID,
|
||||
EmbedModel: c.EmbedModel, Dims: c.Dims, Writable: writable}
|
||||
}
|
||||
|
||||
// kbCollection finds a collection by id or name among those the caller may
|
||||
// read (or write), answering 404 for one it may not.
|
||||
func (r *Router) kbCollection(w http.ResponseWriter, req *http.Request, e storepkg.Effective, write bool) (storepkg.KBCollection, bool) {
|
||||
want := req.PathValue("id")
|
||||
cs, err := r.store.CollectionsFor(req.Context(), e, write)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return storepkg.KBCollection{}, false
|
||||
}
|
||||
for _, c := range cs {
|
||||
if c.ID == want {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
for _, c := range cs {
|
||||
if c.Name == want {
|
||||
return c, true
|
||||
}
|
||||
}
|
||||
msg := "no collection " + want
|
||||
if e.OrgID == "" {
|
||||
msg += " (a personal key names its project in " + projectHeader + ")"
|
||||
}
|
||||
writeError(w, http.StatusNotFound, "not_found_error", msg)
|
||||
return storepkg.KBCollection{}, false
|
||||
}
|
||||
|
||||
func (r *Router) handleKBCollections(w http.ResponseWriter, req *http.Request) {
|
||||
_, e, ok := r.kbCaller(w, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
read, err := r.store.CollectionsFor(req.Context(), e, false)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
write, err := r.store.CollectionsFor(req.Context(), e, true)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
writable := map[string]bool{}
|
||||
for _, c := range write {
|
||||
writable[c.ID] = true
|
||||
}
|
||||
out := []KBCollectionView{}
|
||||
for _, c := range read {
|
||||
out = append(out, kbView(c, writable[c.ID]))
|
||||
}
|
||||
writeJSON(w, http.StatusOK, map[string]any{"object": "list", "data": out})
|
||||
}
|
||||
|
||||
func (r *Router) handleKBManifest(w http.ResponseWriter, req *http.Request) {
|
||||
_, e, ok := r.kbCaller(w, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
c, ok := r.kbCollection(w, req, e, false)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
docs, err := r.store.Documents(req.Context(), c.ID)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
writable, _ := r.store.CollectionsFor(req.Context(), e, true)
|
||||
m := KBManifest{Collection: kbView(c, containsCollection(writable, c.ID)), Documents: []KBDocumentView{}}
|
||||
for _, d := range docs {
|
||||
m.Documents = append(m.Documents, KBDocumentView{Name: d.Name, Digest: d.Digest, Bytes: d.Bytes,
|
||||
Chunks: d.Chunks, AddedAt: d.AddedAt})
|
||||
}
|
||||
writeJSON(w, http.StatusOK, m)
|
||||
}
|
||||
|
||||
func containsCollection(cs []storepkg.KBCollection, id string) bool {
|
||||
for _, c := range cs {
|
||||
if c.ID == id {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// kbWriter is a caller about to change a collection: one it may write, and a
|
||||
// caller whose policy does not forbid keeping what it sends.
|
||||
func (r *Router) kbWriter(w http.ResponseWriter, req *http.Request) (auth.Principal, storepkg.KBCollection, bool) {
|
||||
p, e, ok := r.kbCaller(w, req)
|
||||
if !ok {
|
||||
return p, storepkg.KBCollection{}, false
|
||||
}
|
||||
if r.noStore(guard.Subject{KeyID: p.KeyID, UserID: p.UserID, ProjectID: e.ProjectID,
|
||||
TeamID: e.TeamID, OrgID: e.OrgID, Path: req.URL.Path}) {
|
||||
writeError(w, http.StatusForbidden, "permission_error",
|
||||
"the gateway's policy for this caller keeps nothing, and a knowledge base is documents kept")
|
||||
return p, storepkg.KBCollection{}, false
|
||||
}
|
||||
c, ok := r.kbCollection(w, req, e, true)
|
||||
return p, c, ok
|
||||
}
|
||||
|
||||
// validDocName checks a document's name: what the client sends is a path
|
||||
// relative to what it synced, with forward slashes.
|
||||
func validDocName(n string) (string, error) {
|
||||
if !utf8.ValidString(n) {
|
||||
return "", errors.New("the name is not UTF-8")
|
||||
}
|
||||
n = norm.NFC.String(strings.TrimSpace(n))
|
||||
switch {
|
||||
case n == "":
|
||||
return "", errors.New("a document needs a name")
|
||||
case utf8.RuneCountInString(n) > kbMaxNameRunes:
|
||||
return "", fmt.Errorf("a name has at most %d characters", kbMaxNameRunes)
|
||||
case strings.ContainsFunc(n, unicode.IsControl):
|
||||
return "", errors.New("the name holds a control character")
|
||||
}
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (r *Router) handleKBPut(w http.ResponseWriter, req *http.Request) {
|
||||
p, c, ok := r.kbWriter(w, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
var in KBPutDocument
|
||||
dec := json.NewDecoder(http.MaxBytesReader(w, req.Body, kbMaxBody))
|
||||
if err := dec.Decode(&in); err != nil {
|
||||
var tooBig *http.MaxBytesError
|
||||
if errors.As(err, &tooBig) {
|
||||
writeError(w, http.StatusRequestEntityTooLarge, "invalid_request_error",
|
||||
fmt.Sprintf("a document is at most %d MiB of JSON; split the file", kbMaxBody>>20))
|
||||
return
|
||||
}
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "the request is not valid JSON: "+err.Error())
|
||||
return
|
||||
}
|
||||
name, err := validDocName(in.Name)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
return
|
||||
}
|
||||
switch {
|
||||
case in.Digest == "":
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "a document needs the digest of its file")
|
||||
return
|
||||
case len(in.Chunks) == 0:
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", "a document needs at least one chunk")
|
||||
return
|
||||
case len(in.Chunks) > kbMaxChunks:
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error",
|
||||
fmt.Sprintf("%d chunks: a document has at most %d", len(in.Chunks), kbMaxChunks))
|
||||
return
|
||||
}
|
||||
chunks := make([]storepkg.KBChunk, len(in.Chunks))
|
||||
for i, ch := range in.Chunks {
|
||||
if !utf8.ValidString(ch.Text) || strings.TrimSpace(ch.Text) == "" {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", fmt.Sprintf("chunk %d has no UTF-8 text", i))
|
||||
return
|
||||
}
|
||||
if utf8.RuneCountInString(ch.Text) > kbMaxChunkRunes {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error",
|
||||
fmt.Sprintf("chunk %d is over %d characters", i, kbMaxChunkRunes))
|
||||
return
|
||||
}
|
||||
if len(ch.Embedding) != c.Dims {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error",
|
||||
fmt.Sprintf("chunk %d has %d dimensions; collection %s is embedded by %s in %d", i,
|
||||
len(ch.Embedding), c.Name, c.EmbedModel, c.Dims))
|
||||
return
|
||||
}
|
||||
chunks[i] = storepkg.KBChunk{Body: strings.ToValidUTF8(ch.Text, ""), Embedding: ch.Embedding}
|
||||
}
|
||||
d, changed, err := r.store.PutDocument(req.Context(), c, storepkg.KBDocument{Name: name, Digest: in.Digest,
|
||||
Bytes: in.Bytes, AddedBy: kbActor(p)}, chunks)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
if changed {
|
||||
r.kbAudit(req.Context(), p, "PUT /v1/kb/collections/{id}/documents", c, name, http.StatusOK)
|
||||
}
|
||||
writeJSON(w, http.StatusOK, KBPutResult{Name: name, Chunks: d.Chunks, Changed: changed})
|
||||
}
|
||||
|
||||
func (r *Router) handleKBDelete(w http.ResponseWriter, req *http.Request) {
|
||||
p, c, ok := r.kbWriter(w, req)
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
name, err := validDocName(req.URL.Query().Get("name"))
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "invalid_request_error", err.Error())
|
||||
return
|
||||
}
|
||||
gone, err := r.store.DeleteDocument(req.Context(), c.ID, name)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, "server_error", err.Error())
|
||||
return
|
||||
}
|
||||
if !gone {
|
||||
writeError(w, http.StatusNotFound, "not_found_error", "no document "+name+" in "+c.Name)
|
||||
return
|
||||
}
|
||||
r.kbAudit(req.Context(), p, "DELETE /v1/kb/collections/{id}/documents", c, name, http.StatusOK)
|
||||
writeJSON(w, http.StatusOK, map[string]any{"name": name, "deleted": true})
|
||||
}
|
||||
|
||||
// kbActor is who changed a collection, as the audit log names people: the
|
||||
// person, or the key when it belongs to no one.
|
||||
func kbActor(p auth.Principal) string {
|
||||
if p.UserID != "" {
|
||||
return "user:" + p.UserID
|
||||
}
|
||||
return "key:" + p.KeyID
|
||||
}
|
||||
|
||||
// kbAudit records a change to a collection. The document's name is kept; its
|
||||
// text never is.
|
||||
func (r *Router) kbAudit(ctx context.Context, p auth.Principal, action string, c storepkg.KBCollection, doc string, status int) {
|
||||
err := r.store.WriteAudit(context.WithoutCancel(ctx), storepkg.AuditEvent{Actor: kbActor(p), Source: "gateway",
|
||||
Action: action, Target: "collection:" + c.ID, Detail: c.Name + ": " + doc, Status: status})
|
||||
if err != nil {
|
||||
r.log.Warn("could not record a knowledge base change in the audit log", "collection", c.ID, "err", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
package router
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
func TestKnowledgeBasesSayWhyTheyAreOffOnSQLite(t *testing.T) {
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
rt, secret := pricedRouter(t, e.srv.URL)
|
||||
code, _, body := call(t, rt.Handler(), secret, http.MethodGet, "/v1/kb/collections", nil)
|
||||
if code != http.StatusNotImplemented || !strings.Contains(body, "SQLite") {
|
||||
t.Errorf("got %d %s", code, body)
|
||||
}
|
||||
}
|
||||
|
||||
// kbRouter is a router on the PostgreSQL test database, with pgvector.
|
||||
func kbRouter(t *testing.T) *Router {
|
||||
t.Helper()
|
||||
dsn := os.Getenv("LLMGW_TEST_POSTGRES")
|
||||
if dsn == "" {
|
||||
t.Skip("set LLMGW_TEST_POSTGRES to a pgvector database to run these")
|
||||
}
|
||||
if u, err := url.Parse(dsn); err != nil || !strings.Contains(u.Path, "test") {
|
||||
t.Fatalf("refusing a database whose name does not say test: %s", dsn)
|
||||
}
|
||||
e := newFakeEngine(t, "qwen2.5-14b-instruct")
|
||||
cfg := baseConfig(config.Endpoint{ID: "a", URL: e.srv.URL, Models: []string{"qwen2.5-14b-instruct"}})
|
||||
cfg.Auth.Enabled = true
|
||||
cfg.Auth.Store = dsn
|
||||
rt := newTestRouter(t, cfg)
|
||||
if err := rt.store.KnowledgeAvailable(); err != nil {
|
||||
t.Skipf("no pgvector in the test database: %v", err)
|
||||
}
|
||||
return rt
|
||||
}
|
||||
|
||||
// kbKey makes a key in a project, or a person's key with no project.
|
||||
func kbKey(t *testing.T, s *store.DB, project, user string) string {
|
||||
t.Helper()
|
||||
secret := store.KeyPrefix + strings.ToLower(store.NewID("")) + strings.Repeat("k", 27)
|
||||
sum := sha256.Sum256([]byte(secret))
|
||||
k := store.Key{ID: store.NewID("key-"), Hash: sum[:], Prefix: secret[:8], Name: "t", ProjectID: project, CreatedAt: time.Now()}
|
||||
var err error
|
||||
if user != "" {
|
||||
err = s.CreateUserKey(t.Context(), user, k)
|
||||
} else {
|
||||
err = s.CreateKey(t.Context(), k)
|
||||
}
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return secret
|
||||
}
|
||||
|
||||
func vec(x ...float32) []float32 { return x }
|
||||
|
||||
func doc(name, digest string, embeddings ...[]float32) map[string]any {
|
||||
var chunks []map[string]any
|
||||
for i, e := range embeddings {
|
||||
chunks = append(chunks, map[string]any{"text": name + " parça " + string(rune('a'+i)), "embedding": e})
|
||||
}
|
||||
return map[string]any{"name": name, "digest": digest, "bytes": 10, "chunks": chunks}
|
||||
}
|
||||
|
||||
func TestKnowledgeBasesKeepTenantsApart(t *testing.T) {
|
||||
rt := kbRouter(t)
|
||||
h := rt.Handler()
|
||||
ctx := t.Context()
|
||||
s := rt.store
|
||||
// Unique names: the test database is shared and not emptied here.
|
||||
sfx := strings.ToLower(store.NewID(""))
|
||||
orgA, _ := s.CreateOrg(ctx, "a-"+sfx)
|
||||
teamA, _ := s.CreateTeam(ctx, orgA, "t")
|
||||
hr, _ := s.CreateProject(ctx, teamA, "hr")
|
||||
sales, _ := s.CreateProject(ctx, teamA, "sales")
|
||||
orgB, _ := s.CreateOrg(ctx, "b-"+sfx)
|
||||
teamB, _ := s.CreateTeam(ctx, orgB, "t")
|
||||
other, _ := s.CreateProject(ctx, teamB, "x")
|
||||
c, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeProject, OwnerID: hr,
|
||||
Name: "politikalar", EmbedModel: "emb", Dims: 3})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
hrKey, salesKey, otherKey := kbKey(t, s, hr, ""), kbKey(t, s, sales, ""), kbKey(t, s, other, "")
|
||||
base := "/v1/kb/collections/"
|
||||
|
||||
// The owner's project writes, by name or by id.
|
||||
code, out, body := call(t, h, hrKey, http.MethodPut, base+"politikalar/documents", doc("izin.pdf", "d1", vec(1, 0, 0), vec(0, 1, 0)))
|
||||
if code != http.StatusOK || out["changed"] != true || out["chunks"] != float64(2) {
|
||||
t.Fatalf("owner's put: %d %s", code, body)
|
||||
}
|
||||
code, out, _ = call(t, h, hrKey, http.MethodPut, base+c.ID+"/documents", doc("izin.pdf", "d1", vec(1, 0, 0)))
|
||||
if code != http.StatusOK || out["changed"] != false {
|
||||
t.Errorf("the same digest again: %d %v", code, out)
|
||||
}
|
||||
code, _, body = call(t, h, hrKey, http.MethodGet, base+"politikalar/manifest", nil)
|
||||
if code != http.StatusOK || !strings.Contains(body, `"digest":"d1"`) || !strings.Contains(body, `"writable":true`) {
|
||||
t.Errorf("manifest: %d %s", code, body)
|
||||
}
|
||||
|
||||
// Another organization, and another project without a grant, see no
|
||||
// such collection — not a refusal that says it exists.
|
||||
for name, key := range map[string]string{"other org": otherKey, "other project": salesKey} {
|
||||
for _, m := range []string{http.MethodGet + " manifest", http.MethodPut + " documents"} {
|
||||
method, path, _ := strings.Cut(m, " ")
|
||||
code, _, body := call(t, h, key, method, base+c.ID+"/"+path, doc("x", "y", vec(1, 0, 0)))
|
||||
if code != http.StatusNotFound {
|
||||
t.Errorf("%s %s: %d %s", name, m, code, body)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// A read grant lets sales read, not write.
|
||||
if err := s.Grant(ctx, store.KBGrant{Scope: store.ScopeProject, ScopeID: sales, Collection: c.ID, Perm: store.KBRead}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if code, _, body := call(t, h, salesKey, http.MethodGet, base+"politikalar/manifest", nil); code != http.StatusOK ||
|
||||
!strings.Contains(body, `"writable":false`) {
|
||||
t.Errorf("read grant, manifest: %d %s", code, body)
|
||||
}
|
||||
if code, _, _ := call(t, h, salesKey, http.MethodPut, base+"politikalar/documents", doc("x", "y", vec(1, 0, 0))); code != http.StatusNotFound {
|
||||
t.Errorf("read grant, put: %d", code)
|
||||
}
|
||||
if code, _, body := call(t, h, salesKey, http.MethodGet, "/v1/kb/collections", nil); code != http.StatusOK ||
|
||||
!strings.Contains(body, "politikalar") {
|
||||
t.Errorf("read grant, list: %d %s", code, body)
|
||||
}
|
||||
if _, _, body := call(t, h, otherKey, http.MethodGet, "/v1/kb/collections", nil); strings.Contains(body, "politikalar") {
|
||||
t.Errorf("another organization lists it: %s", body)
|
||||
}
|
||||
|
||||
// Vectors of the wrong size are refused before anything is written.
|
||||
if code, _, body := call(t, h, hrKey, http.MethodPut, base+"politikalar/documents", doc("b.md", "d", vec(1, 0))); code != http.StatusBadRequest ||
|
||||
!strings.Contains(body, "dimensions") {
|
||||
t.Errorf("wrong dimensions: %d %s", code, body)
|
||||
}
|
||||
|
||||
if code, _, body := call(t, h, hrKey, http.MethodDelete, base+"politikalar/documents?name=izin.pdf", nil); code != http.StatusOK {
|
||||
t.Errorf("delete: %d %s", code, body)
|
||||
}
|
||||
if code, _, _ := call(t, h, hrKey, http.MethodDelete, base+"politikalar/documents?name=izin.pdf", nil); code != http.StatusNotFound {
|
||||
t.Errorf("deleting again: %d", code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPersonalKeyReadsOnlyInTheProjectItNames(t *testing.T) {
|
||||
rt := kbRouter(t)
|
||||
h := rt.Handler()
|
||||
ctx := t.Context()
|
||||
s := rt.store
|
||||
sfx := strings.ToLower(store.NewID(""))
|
||||
org, _ := s.CreateOrg(ctx, "p-"+sfx)
|
||||
team, _ := s.CreateTeam(ctx, org, "t")
|
||||
hr, _ := s.CreateProject(ctx, team, "hr")
|
||||
fin, _ := s.CreateProject(ctx, team, "fin")
|
||||
uid := store.NewID("u-")
|
||||
if err := s.CreateUser(ctx, store.User{ID: uid, Email: uid + "@example.com", Name: "u", PasswordHash: "x",
|
||||
State: store.UserActive, CreatedAt: time.Now()}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := s.AddMembership(ctx, uid, store.ScopeProject, hr, "member", "t", time.Now()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeProject, OwnerID: hr,
|
||||
Name: "ik", EmbedModel: "emb", Dims: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := s.CreateCollection(ctx, store.KBCollection{OwnerScope: store.ScopeProject, OwnerID: fin,
|
||||
Name: "finans", EmbedModel: "emb", Dims: 3}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
key := kbKey(t, s, "", uid)
|
||||
get := func(project string) (int, string) {
|
||||
req, _ := http.NewRequest(http.MethodGet, "/v1/kb/collections", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+key)
|
||||
if project != "" {
|
||||
req.Header.Set(projectHeader, project)
|
||||
}
|
||||
rec := httptest.NewRecorder()
|
||||
h.ServeHTTP(rec, req)
|
||||
return rec.Code, rec.Body.String()
|
||||
}
|
||||
if code, body := get(""); code != http.StatusOK || strings.Contains(body, "ik") {
|
||||
t.Errorf("no project named: %d %s", code, body)
|
||||
}
|
||||
if code, body := get(hr); code != http.StatusOK || !strings.Contains(body, `"ik"`) || strings.Contains(body, "finans") {
|
||||
t.Errorf("in hr: %d %s", code, body)
|
||||
}
|
||||
if code, _ := get(fin); code != http.StatusForbidden {
|
||||
t.Errorf("a project the person is not in: %d", code)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user