Set up a replicated store with server init --patroni, and watch it
init --patroni user@host ... installs PostgreSQL, Patroni and etcd on the given machines over ssh, with this one as the third etcd vote, makes the role and database, and moves the store in use into the cluster. Each step checks first, so it changes nothing on a cluster that is up. For a store of several hosts the control plane publishes each member's role, state and lag from Patroni, a stopped member as missing, prints it in ctl status, and alerts on it. Making an administrator on PostgreSQL sent a bool to an integer column; it sends an integer now. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
7a562ac1e7
commit
975c93994f
22 files changed
+1495
-43
No files matched your search
+6
-35
@@ -8,7 +8,6 @@ import (
|
||||
"fmt"
|
||||
"io"
|
||||
"io/fs"
|
||||
"net/url"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
@@ -190,42 +189,14 @@ func pgDump(ctx context.Context, dsn, out string) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
// pgEnv turns a store address into libpq's environment.
|
||||
//
|
||||
// Several hosts are allowed, as pgx allows them — "h1:5432,h2:5432", the
|
||||
// members of a replicated cluster — and become libpq's own lists in PGHOST and
|
||||
// PGPORT. url.Parse does not read such a host: it took everything before the
|
||||
// last colon as one name, which libpq could not resolve, and a backup worked
|
||||
// only because it fell through to the second host.
|
||||
// pgEnv turns a store address into libpq's environment. Several hosts — the
|
||||
// members of a replicated cluster — become libpq's own lists in PGHOST and
|
||||
// PGPORT; read as one URL host, the list was one name libpq could not
|
||||
// resolve, and a backup worked only by falling through to the second.
|
||||
func pgEnv(dsn string) ([]string, error) {
|
||||
scheme, rest, ok := strings.Cut(dsn, "://")
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("the store address is not a URL")
|
||||
}
|
||||
authority, tail := rest, ""
|
||||
if i := strings.IndexAny(rest, "/?"); i >= 0 {
|
||||
authority, tail = rest[:i], rest[i:]
|
||||
}
|
||||
userinfo, hostlist := "", authority
|
||||
if i := strings.LastIndexByte(authority, '@'); i >= 0 {
|
||||
userinfo, hostlist = authority[:i+1], authority[i+1:]
|
||||
}
|
||||
var hosts, ports []string
|
||||
for _, hp := range strings.Split(hostlist, ",") {
|
||||
u, err := url.Parse("x://" + hp)
|
||||
if err != nil || u.Hostname() == "" {
|
||||
return nil, fmt.Errorf("the store address has a host that is not one: %q", hp)
|
||||
}
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
port = "5432"
|
||||
}
|
||||
hosts, ports = append(hosts, u.Hostname()), append(ports, port)
|
||||
}
|
||||
// The rest of the address with one placeholder host, which url.Parse reads.
|
||||
u, err := url.Parse(scheme + "://" + userinfo + "host" + tail)
|
||||
hosts, ports, u, err := store.SplitPostgres(dsn)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("the store address is not a URL")
|
||||
return nil, err
|
||||
}
|
||||
var env []string
|
||||
set := func(k, v string) {
|
||||
|
||||
@@ -1,9 +1,14 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/adminapi"
|
||||
"github.com/spf13/cobra"
|
||||
)
|
||||
|
||||
func TestAQueuedChangeIsNotReportedAsAnError(t *testing.T) {
|
||||
@@ -22,3 +27,27 @@ func TestAQueuedChangeIsNotReportedAsAnError(t *testing.T) {
|
||||
t.Errorf("the body of a 202 was not read: %+v", out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStatusSaysWhoLeadsTheStoreAndWhetherTheRestKeepUp(t *testing.T) {
|
||||
render := func(s adminapi.Summary) string {
|
||||
cmd := &cobra.Command{}
|
||||
var b bytes.Buffer
|
||||
cmd.SetOut(&b)
|
||||
printSummary(cmd, s)
|
||||
return b.String()
|
||||
}
|
||||
out := render(adminapi.Summary{Store: []adminapi.StoreMember{
|
||||
{Name: "llmgw-ctl02", Role: "replica", State: "streaming", Lag: 2048},
|
||||
{Name: "llmgw-pg01", Role: "leader", State: "running"},
|
||||
}})
|
||||
if !strings.Contains(out, "STORE llmgw-ctl02 streaming (2 KB behind), llmgw-pg01 leads") {
|
||||
t.Errorf("status:\n%s", out)
|
||||
}
|
||||
out = render(adminapi.Summary{Store: []adminapi.StoreMember{{Name: "llmgw-pg01", Role: "replica", State: "stopped"}}})
|
||||
if !strings.Contains(out, "NO LEADER") {
|
||||
t.Errorf("a store with no leader was not called out:\n%s", out)
|
||||
}
|
||||
if out := render(adminapi.Summary{}); strings.Contains(out, "STORE") {
|
||||
t.Error("a single database got a store line")
|
||||
}
|
||||
}
|
||||
@@ -19,6 +19,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/gguf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/sourcecheck"
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
|
||||
@@ -127,6 +128,9 @@ func newServerCmd() *cobra.Command {
|
||||
var agentHandler http.Handler
|
||||
var agentPath string
|
||||
var admin *control.Controller
|
||||
// The store's cluster, when it is one; read by the metrics and
|
||||
// by `ctl status`.
|
||||
var storeHA *pgha.Collector
|
||||
if controlMode {
|
||||
cat, err := cfg.Catalog()
|
||||
if err != nil {
|
||||
@@ -179,6 +183,17 @@ func newServerCmd() *cobra.Command {
|
||||
if err := rt.RegisterMetrics(fleetmetrics.New(ctl)); err != nil {
|
||||
return err
|
||||
}
|
||||
// A store of several hosts is a replicated cluster: what state
|
||||
// it is in is published beside everything else, read from its
|
||||
// Patroni. One host is a single PostgreSQL, which has none.
|
||||
if addr, err := storepkg.ResolveAddress(cfg.Auth.Store); err == nil && storepkg.IsPostgres(addr) {
|
||||
if hosts, _, _, err := storepkg.SplitPostgres(strings.TrimSpace(addr)); err == nil && len(hosts) > 1 {
|
||||
storeHA = pgha.NewCollector(hosts)
|
||||
if err := rt.RegisterMetrics(storeHA); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
// So a request for a model that is configured but not running
|
||||
// starts it, rather than being told nothing serves it.
|
||||
rt.SetActivator(ctl.Reconciler())
|
||||
@@ -201,6 +216,16 @@ func newServerCmd() *cobra.Command {
|
||||
)
|
||||
if admin != nil {
|
||||
api := adminapi.New(admin)
|
||||
if storeHA != nil {
|
||||
api.SetStoreStatus(func() ([]adminapi.StoreMember, error) {
|
||||
c, err := storeHA.Read()
|
||||
var out []adminapi.StoreMember
|
||||
for _, m := range c.Members {
|
||||
out = append(out, adminapi.StoreMember{Name: m.Name, Host: m.Host, Role: m.Role, State: m.State, Lag: m.Lag})
|
||||
}
|
||||
return out, err
|
||||
})
|
||||
}
|
||||
adminAPI = api
|
||||
api.SetReleases(cfg.Releases)
|
||||
// Catalog changes need somewhere to persist, so this is wired
|
||||
|
||||
@@ -69,6 +69,9 @@ type serverInit struct {
|
||||
// postgres keeps the store in a PostgreSQL on this machine, installed if
|
||||
// need be, moving whatever SQLite already holds.
|
||||
postgres bool
|
||||
// patroni are user@host of the machines to keep the store on under
|
||||
// Patroni, replicated; this machine is the third etcd vote for two.
|
||||
patroni []string
|
||||
// noMirror, noNode and noModel turn off the parts of a first install that
|
||||
// make it serve something: a mirror, this machine as a node, and a small
|
||||
// model on it.
|
||||
@@ -111,6 +114,12 @@ func newServerInitCmd() *cobra.Command {
|
||||
"installed if there is none, with its own role and a generated password\n" +
|
||||
"kept in /etc/llmgw/secrets. An existing SQLite store is copied into it,\n" +
|
||||
"so accounts and keys survive, and server.yaml is pointed at it.\n\n" +
|
||||
"With --patroni, given two or more machines, the store is PostgreSQL\n" +
|
||||
"replicated across them under Patroni, which promotes a standby when the\n" +
|
||||
"primary is lost. etcd holds its leader lock on those machines — and on\n" +
|
||||
"this one, when they are two, as the third vote. Installed over ssh with\n" +
|
||||
"passwordless sudo, as 'llmgw node add' does; the store already in use,\n" +
|
||||
"SQLite or PostgreSQL, is moved in. Run again, it changes nothing.\n\n" +
|
||||
"Each step is skipped when its result already exists, so it is safe to run\n" +
|
||||
"again: to finish an install that stopped, or with another --host to\n" +
|
||||
"reissue the certificate. It never overwrites server.yaml or the authority.\n\n" +
|
||||
@@ -144,6 +153,9 @@ func newServerInitCmd() *cobra.Command {
|
||||
f.BoolVar(&in.noModel, "no-model", false, "do not fetch and serve the starter model ("+starterName+")")
|
||||
f.BoolVar(&in.postgres, "postgres", false,
|
||||
"keep the store in PostgreSQL on this machine, installing it and moving an existing SQLite store into it")
|
||||
f.StringArrayVar(&in.patroni, "patroni", nil,
|
||||
"user@host of a machine to keep the store on, replicated under Patroni; give two or more. "+
|
||||
"Installs PostgreSQL, Patroni and etcd there over ssh, and moves the existing store in")
|
||||
return cmd
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
package cli
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/config"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/onboard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/pgha"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/store"
|
||||
)
|
||||
|
||||
// ensurePatroni puts the store in PostgreSQL under Patroni on the machines
|
||||
// given with --patroni, and moves whatever store server.yaml names into it:
|
||||
// SQLite, or a PostgreSQL on one machine. This machine is the third etcd vote
|
||||
// when they are two.
|
||||
func (in *serverInit) ensurePatroni(ctx context.Context) (err error) {
|
||||
setup := pgha.Setup{Emit: func(step, detail string, changed bool) {
|
||||
if changed {
|
||||
in.done("%s: %s", step, detail)
|
||||
} else {
|
||||
in.kept("%s: %s", step, detail)
|
||||
}
|
||||
}}
|
||||
for _, t := range in.patroni {
|
||||
user, host, ok := strings.Cut(t, "@")
|
||||
if !ok {
|
||||
user, host = "", t
|
||||
}
|
||||
remote := &onboard.SSH{Addr: host, User: user}
|
||||
out, err := remote.Run(ctx, "hostname && sudo -n true", nil)
|
||||
if err != nil {
|
||||
return fmt.Errorf("%s: cannot log in and use sudo without a password: %v: %s",
|
||||
t, err, strings.TrimSpace(string(out)))
|
||||
}
|
||||
ip, err := ipv4Of(host)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
setup.DB = append(setup.DB, pgha.Node{
|
||||
Member: pgha.Member{Name: strings.TrimSpace(strings.SplitN(string(out), "\n", 2)[0]), IP: ip},
|
||||
Remote: remote,
|
||||
})
|
||||
}
|
||||
self, err := ipv4Of(in.reachableAddr())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
name, _ := os.Hostname()
|
||||
setup.Witness = &pgha.Node{Member: pgha.Member{Name: name, IP: self}, Remote: pgha.Local{}}
|
||||
setup.Clients = []string{self}
|
||||
|
||||
dsn, err := setup.Run(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
cfg, err := config.Load(in.configPath())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
current := cfg.Auth.Store
|
||||
if current == "" {
|
||||
current = in.storeAddress()
|
||||
}
|
||||
resolved, _ := store.ResolveAddress(current)
|
||||
if strings.TrimSpace(resolved) == dsn {
|
||||
in.kept("store in the Patroni cluster (%s)", in.dsnPath())
|
||||
return nil
|
||||
}
|
||||
|
||||
// Still while it moves, and started again on the old store if the move
|
||||
// fails: a gateway down over a database that did not work out is worse
|
||||
// than one left where it was.
|
||||
stopped, err := in.stopService(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil && stopped {
|
||||
_ = exec.Command("systemctl", "start", "llmgw-server").Run()
|
||||
fmt.Fprintf(in.out, " .. llmgw-server started again on its old store\n")
|
||||
}
|
||||
}()
|
||||
if err := in.moveInto(ctx, current, dsn); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// The old address is kept beside the new one: the way back, if the
|
||||
// cluster is not what was wanted.
|
||||
if b, err := os.ReadFile(in.dsnPath()); err == nil && strings.TrimSpace(string(b)) != dsn {
|
||||
if err := os.WriteFile(in.dsnPath()+".pre-patroni", b, 0o600); err != nil {
|
||||
return err
|
||||
}
|
||||
_ = in.chown(in.dsnPath()+".pre-patroni", true)
|
||||
}
|
||||
if err := in.writeDSN(dsn); err != nil {
|
||||
return err
|
||||
}
|
||||
if current != "@"+in.dsnPath() {
|
||||
if err := setAuthStore(in.configPath(), "@"+in.dsnPath()); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Started again by the service step that follows, on the new address.
|
||||
in.done("server.yaml now keeps the store in the Patroni cluster")
|
||||
return nil
|
||||
}
|
||||
|
||||
// moveInto copies a store into the cluster, unless the cluster already holds
|
||||
// one — from a run that copied and stopped before changing the address.
|
||||
func (in *serverInit) moveInto(ctx context.Context, from, to string) error {
|
||||
src, err := store.Open(ctx, from)
|
||||
if err != nil {
|
||||
// Nothing there yet: a first install, with nothing to move.
|
||||
in.kept("no store at %s to move", from)
|
||||
return nil
|
||||
}
|
||||
defer src.Close()
|
||||
dst, err := store.Open(ctx, to)
|
||||
if err != nil {
|
||||
return fmt.Errorf("opening the cluster: %w", err)
|
||||
}
|
||||
defer dst.Close()
|
||||
empty, err := dst.Empty(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !empty {
|
||||
in.kept("the cluster already holds a store; %s is left as it is", from)
|
||||
return nil
|
||||
}
|
||||
res, err := src.CopyTo(ctx, dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("copying the store into the cluster: %w", err)
|
||||
}
|
||||
var rows int64
|
||||
for _, r := range res {
|
||||
rows += r.Rows
|
||||
}
|
||||
in.done("copied %d rows into the cluster (the old store is kept as it was)", rows)
|
||||
return nil
|
||||
}
|
||||
|
||||
// ipv4Of is the address a host name or address stands for, as the other
|
||||
// machines will be told to reach it.
|
||||
func ipv4Of(host string) (string, error) {
|
||||
if ip := net.ParseIP(host); ip != nil {
|
||||
return ip.String(), nil
|
||||
}
|
||||
ips, err := net.LookupIP(host)
|
||||
if err != nil {
|
||||
return "", fmt.Errorf("%s does not resolve: %w", host, err)
|
||||
}
|
||||
for _, ip := range ips {
|
||||
if v4 := ip.To4(); v4 != nil {
|
||||
return v4.String(), nil
|
||||
}
|
||||
}
|
||||
return "", fmt.Errorf("%s has no IPv4 address", host)
|
||||
}
|
||||
@@ -44,6 +44,12 @@ func (in *serverInit) storeAddress() string {
|
||||
// means reissuing every key and being unable to say which was which; and the
|
||||
// administrator init created a minute ago would be gone too.
|
||||
func (in *serverInit) ensureDatabase(ctx context.Context) (err error) {
|
||||
if len(in.patroni) > 0 {
|
||||
if in.postgres {
|
||||
return fmt.Errorf("--postgres keeps the store on this machine and --patroni on others: give one")
|
||||
}
|
||||
return in.ensurePatroni(ctx)
|
||||
}
|
||||
if !in.postgres {
|
||||
return nil
|
||||
}
|
||||
|
||||
@@ -2,6 +2,7 @@ package cli
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/spf13/cobra"
|
||||
@@ -94,6 +95,31 @@ func printSummary(cmd *cobra.Command, s adminapi.Summary) {
|
||||
fmt.Fprintf(out, "\nMODELS %d of %d have the replicas they asked for\n",
|
||||
s.Models.Satisfied, s.Models.Total)
|
||||
|
||||
// The store, when it is a replicated cluster: one line, saying who leads
|
||||
// and whether every other member keeps up.
|
||||
switch {
|
||||
case s.StoreError != "":
|
||||
fmt.Fprintf(out, "STORE cannot read its cluster: %s\n", s.StoreError)
|
||||
case len(s.Store) > 0:
|
||||
var parts []string
|
||||
leader := false
|
||||
for _, m := range s.Store {
|
||||
switch {
|
||||
case m.Role == "leader":
|
||||
leader = true
|
||||
parts = append(parts, m.Name+" leads")
|
||||
case m.State == "streaming" && m.Lag >= 0:
|
||||
parts = append(parts, fmt.Sprintf("%s streaming (%s behind)", m.Name, bytesShort(m.Lag)))
|
||||
default:
|
||||
parts = append(parts, fmt.Sprintf("%s %s", m.Name, m.State))
|
||||
}
|
||||
}
|
||||
if !leader {
|
||||
parts = append([]string{"NO LEADER — writes fail"}, parts...)
|
||||
}
|
||||
fmt.Fprintf(out, "STORE %s\n", strings.Join(parts, ", "))
|
||||
}
|
||||
|
||||
if len(s.Parked) > 0 {
|
||||
fmt.Fprintln(out, "\nPARKED — failed every time they started; not retried until fixed")
|
||||
for _, p := range s.Parked {
|
||||
@@ -169,3 +195,16 @@ func unhealthyCell(n int) string {
|
||||
}
|
||||
return fmt.Sprintf("%d", n)
|
||||
}
|
||||
|
||||
// bytesShort is a byte count as a person reads it: 0 B, 12 KB, 3.4 MB.
|
||||
func bytesShort(n int64) string {
|
||||
switch {
|
||||
case n < 1024:
|
||||
return fmt.Sprintf("%d B", n)
|
||||
case n < 1<<20:
|
||||
return fmt.Sprintf("%d KB", n>>10)
|
||||
case n < 1<<30:
|
||||
return fmt.Sprintf("%.1f MB", float64(n)/(1<<20))
|
||||
}
|
||||
return fmt.Sprintf("%.1f GB", float64(n)/(1<<30))
|
||||
}
|
||||
Reference in new issue
Block a user