Knowledge bases in the store: collections, grants and chunks kept apart per tenant

Collections owned by an organization, team, project or person; grants that
add up down the tree and never cross an organization; chunks in PostgreSQL
with pgvector, partitioned by organization, an HNSW index per collection, and
row-level security as a second line behind each query's filter. Search fuses
nearest-neighbour and full-text ranks. Off on SQLite and without pgvector,
saying why. server init and the HA database setup install and enable it.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-08 12:47:57 +02:00
1 parent 0a7fb821d8
commit d35630defe
11 files changed
+1408 -25

No files matched your search

+60 -2
View File
@@ -370,6 +370,8 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)")
}
in.setupPgvector(ctx, discovery.DiscoverHost())
// Proof the address works, before anything is moved into it.
address := "@" + in.dsnPath()
s, err := store.Open(ctx, address)
@@ -482,8 +484,17 @@ func lastLines(b []byte, n int) string {
// psql runs one statement as the postgres superuser, over the local socket.
func (in *serverInit) psql(ctx context.Context, query string) (string, error) {
cmd := exec.CommandContext(ctx, "runuser", "-u", "postgres", "--",
in.pg.tool("psql"), "-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query)
return in.psqlIn(ctx, "", query)
}
// psqlIn runs a statement as postgres in one database, or in postgres's
// default one when db is "".
func (in *serverInit) psqlIn(ctx context.Context, db, query string) (string, error) {
args := []string{"-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query}
if db != "" {
args = append(args, "-d", db)
}
cmd := exec.CommandContext(ctx, "runuser", append([]string{"-u", "postgres", "--", in.pg.tool("psql")}, args...)...)
// postgres cannot enter root's home, and psql says so on every call.
cmd.Dir = "/"
out, err := cmd.CombinedOutput()
@@ -538,3 +549,50 @@ func (in *serverInit) stopService(ctx context.Context) (bool, error) {
in.done("llmgw-server stopped while its store moves")
return true, nil
}
// pgvectorPackages are the package names of pgvector for a PostgreSQL major
// version: Debian's and Ubuntu's (and PGDG's apt repository) name it after the
// server, PGDG's rpm repository after the version alone.
func pgvectorPackages(major int) map[string][]string {
return map[string][]string{
"deb": {fmt.Sprintf("postgresql-%d-pgvector", major)},
"rpm": {fmt.Sprintf("pgvector_%d", major)},
}
}
// setupPgvector installs pgvector and enables it in llmgw's database, for
// knowledge bases. Not a reason to stop: without it everything else works,
// and the store says why knowledge bases are off when somebody asks for one.
func (in *serverInit) setupPgvector(ctx context.Context, h discovery.Host) {
warn := func(format string, a ...any) {
fmt.Fprintf(in.out, " !! pgvector: "+format+"; knowledge bases stay off\n", a...)
}
if on, err := in.psqlIn(ctx, pgDatabase, "SELECT 1 FROM pg_extension WHERE extname = 'vector'"); err == nil && on == "1" {
in.kept("pgvector in database %s", pgDatabase)
return
}
have, err := in.psqlBool(ctx, "SELECT 1 FROM pg_available_extensions WHERE name = 'vector'")
if err != nil {
warn("%v", err)
return
}
if !have {
num, err := in.psql(ctx, "SHOW server_version_num")
n, convErr := strconv.Atoi(num)
if err != nil || convErr != nil || h.Family() == "" {
warn("cannot tell which package to install (version %q, %s)", num, h.OSID)
return
}
pkgs := pgvectorPackages(n / 10000)
pi := &provision.Installer{Host: h, Log: slog.Default()}
if err := pi.InstallPackages(ctx, pkgs); err != nil {
warn("installing %s: %v", strings.Join(pkgs[h.Family()], " "), err)
return
}
}
if _, err := in.psqlIn(ctx, pgDatabase, "CREATE EXTENSION IF NOT EXISTS vector"); err != nil {
warn("%v", err)
return
}
in.done("pgvector enabled in database %s", pgDatabase)
}