Knowledge bases in the store: collections, grants and chunks kept apart per tenant
Collections owned by an organization, team, project or person; grants that add up down the tree and never cross an organization; chunks in PostgreSQL with pgvector, partitioned by organization, an HNSW index per collection, and row-level security as a second line behind each query's filter. Search fuses nearest-neighbour and full-text ranks. Off on SQLite and without pgvector, saying why. server init and the HA database setup install and enable it. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
0a7fb821d8
commit
d35630defe
11 files changed
+1408
-25
No files matched your search
@@ -370,6 +370,8 @@ func (in *serverInit) setupPostgres(ctx context.Context) (string, error) {
|
||||
in.done("PostgreSQL sized for a machine it shares (shared_buffers 96MB, 40 connections)")
|
||||
}
|
||||
|
||||
in.setupPgvector(ctx, discovery.DiscoverHost())
|
||||
|
||||
// Proof the address works, before anything is moved into it.
|
||||
address := "@" + in.dsnPath()
|
||||
s, err := store.Open(ctx, address)
|
||||
@@ -482,8 +484,17 @@ func lastLines(b []byte, n int) string {
|
||||
|
||||
// psql runs one statement as the postgres superuser, over the local socket.
|
||||
func (in *serverInit) psql(ctx context.Context, query string) (string, error) {
|
||||
cmd := exec.CommandContext(ctx, "runuser", "-u", "postgres", "--",
|
||||
in.pg.tool("psql"), "-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query)
|
||||
return in.psqlIn(ctx, "", query)
|
||||
}
|
||||
|
||||
// psqlIn runs a statement as postgres in one database, or in postgres's
|
||||
// default one when db is "".
|
||||
func (in *serverInit) psqlIn(ctx context.Context, db, query string) (string, error) {
|
||||
args := []string{"-v", "ON_ERROR_STOP=1", "-qAtX", "-c", query}
|
||||
if db != "" {
|
||||
args = append(args, "-d", db)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, "runuser", append([]string{"-u", "postgres", "--", in.pg.tool("psql")}, args...)...)
|
||||
// postgres cannot enter root's home, and psql says so on every call.
|
||||
cmd.Dir = "/"
|
||||
out, err := cmd.CombinedOutput()
|
||||
@@ -538,3 +549,50 @@ func (in *serverInit) stopService(ctx context.Context) (bool, error) {
|
||||
in.done("llmgw-server stopped while its store moves")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
// pgvectorPackages are the package names of pgvector for a PostgreSQL major
|
||||
// version: Debian's and Ubuntu's (and PGDG's apt repository) name it after the
|
||||
// server, PGDG's rpm repository after the version alone.
|
||||
func pgvectorPackages(major int) map[string][]string {
|
||||
return map[string][]string{
|
||||
"deb": {fmt.Sprintf("postgresql-%d-pgvector", major)},
|
||||
"rpm": {fmt.Sprintf("pgvector_%d", major)},
|
||||
}
|
||||
}
|
||||
|
||||
// setupPgvector installs pgvector and enables it in llmgw's database, for
|
||||
// knowledge bases. Not a reason to stop: without it everything else works,
|
||||
// and the store says why knowledge bases are off when somebody asks for one.
|
||||
func (in *serverInit) setupPgvector(ctx context.Context, h discovery.Host) {
|
||||
warn := func(format string, a ...any) {
|
||||
fmt.Fprintf(in.out, " !! pgvector: "+format+"; knowledge bases stay off\n", a...)
|
||||
}
|
||||
if on, err := in.psqlIn(ctx, pgDatabase, "SELECT 1 FROM pg_extension WHERE extname = 'vector'"); err == nil && on == "1" {
|
||||
in.kept("pgvector in database %s", pgDatabase)
|
||||
return
|
||||
}
|
||||
have, err := in.psqlBool(ctx, "SELECT 1 FROM pg_available_extensions WHERE name = 'vector'")
|
||||
if err != nil {
|
||||
warn("%v", err)
|
||||
return
|
||||
}
|
||||
if !have {
|
||||
num, err := in.psql(ctx, "SHOW server_version_num")
|
||||
n, convErr := strconv.Atoi(num)
|
||||
if err != nil || convErr != nil || h.Family() == "" {
|
||||
warn("cannot tell which package to install (version %q, %s)", num, h.OSID)
|
||||
return
|
||||
}
|
||||
pkgs := pgvectorPackages(n / 10000)
|
||||
pi := &provision.Installer{Host: h, Log: slog.Default()}
|
||||
if err := pi.InstallPackages(ctx, pkgs); err != nil {
|
||||
warn("installing %s: %v", strings.Join(pkgs[h.Family()], " "), err)
|
||||
return
|
||||
}
|
||||
}
|
||||
if _, err := in.psqlIn(ctx, pgDatabase, "CREATE EXTENSION IF NOT EXISTS vector"); err != nil {
|
||||
warn("%v", err)
|
||||
return
|
||||
}
|
||||
in.done("pgvector enabled in database %s", pgDatabase)
|
||||
}
|
||||
Reference in new issue
Block a user