Ilker ManapandClaude Opus 5.5
975c93994f
init --patroni user@host ... installs PostgreSQL, Patroni and etcd on the given machines over ssh, with this one as the third etcd vote, makes the role and database, and moves the store in use into the cluster. Each step checks first, so it changes nothing on a cluster that is up. For a store of several hosts the control plane publishes each member's role, state and lag from Patroni, a stopped member as missing, prints it in ctl status, and alerts on it. Making an administrator on PostgreSQL sent a bool to an integer column; it sends an integer now. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
101 lines
3.0 KiB
Go
101 lines
3.0 KiB
Go
package pgha
|
|
|
|
import (
|
|
"context"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/prometheus/client_golang/prometheus"
|
|
)
|
|
|
|
// Collector publishes the store cluster's state as the control plane sees it
|
|
// through Patroni's API: who leads, who streams, how far behind. Read at most
|
|
// every fifteen seconds, however often it is scraped.
|
|
type Collector struct {
|
|
hosts []string
|
|
status func(ctx context.Context, hosts []string) (Cluster, error)
|
|
|
|
mu sync.Mutex
|
|
at time.Time
|
|
last Cluster
|
|
err error
|
|
|
|
up, member, lag *prometheus.Desc
|
|
}
|
|
|
|
// NewCollector reads the cluster from the Patroni of each host in turn.
|
|
func NewCollector(hosts []string) *Collector {
|
|
return &Collector{
|
|
hosts: hosts,
|
|
status: HTTPStatus,
|
|
up: prometheus.NewDesc("llmgw_store_patroni_up",
|
|
"1 when a member's Patroni answered for the store's cluster.", nil, nil),
|
|
member: prometheus.NewDesc("llmgw_store_member",
|
|
"Each member of the store's cluster, by role (leader, replica) and state (running, streaming, …); always 1.",
|
|
[]string{"member", "host", "role", "state"}, nil),
|
|
lag: prometheus.NewDesc("llmgw_store_replica_lag_bytes",
|
|
"How far each replica is behind the leader; -1 when Patroni does not know.",
|
|
[]string{"member"}, nil),
|
|
}
|
|
}
|
|
|
|
// Describe implements prometheus.Collector.
|
|
func (c *Collector) Describe(ch chan<- *prometheus.Desc) {
|
|
ch <- c.up
|
|
ch <- c.member
|
|
ch <- c.lag
|
|
}
|
|
|
|
// Collect implements prometheus.Collector.
|
|
func (c *Collector) Collect(ch chan<- prometheus.Metric) {
|
|
cl, err := c.read()
|
|
if err != nil {
|
|
ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, 0)
|
|
return
|
|
}
|
|
ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, 1)
|
|
for _, m := range cl.Members {
|
|
ch <- prometheus.MustNewConstMetric(c.member, prometheus.GaugeValue, 1, m.Name, m.Host, m.Role, m.State)
|
|
if m.Role != "leader" {
|
|
ch <- prometheus.MustNewConstMetric(c.lag, prometheus.GaugeValue, float64(m.Lag), m.Name)
|
|
}
|
|
}
|
|
}
|
|
|
|
// Read is the cluster's state, at most fifteen seconds old.
|
|
func (c *Collector) Read() (Cluster, error) { return c.read() }
|
|
|
|
func (c *Collector) read() (Cluster, error) {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
if time.Since(c.at) < 15*time.Second {
|
|
return c.last, c.err
|
|
}
|
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
defer cancel()
|
|
c.last, c.err = c.status(ctx, c.hosts)
|
|
if c.err == nil {
|
|
c.last = withMissing(c.last, c.hosts)
|
|
}
|
|
c.at = time.Now()
|
|
return c.last, c.err
|
|
}
|
|
|
|
// withMissing adds every host of the store address that the cluster does not
|
|
// list, as a member in state "missing". A member that stops is dropped from
|
|
// Patroni's list once its key expires — so without this a cluster down to
|
|
// one machine reads as a healthy cluster of one, and nothing says a standby
|
|
// is gone.
|
|
func withMissing(c Cluster, hosts []string) Cluster {
|
|
seen := map[string]bool{}
|
|
for _, m := range c.Members {
|
|
seen[m.Host] = true
|
|
}
|
|
for _, h := range hosts {
|
|
if !seen[h] {
|
|
c.Members = append(c.Members, MemberState{Name: h, Host: h, Role: "replica", State: "missing", Lag: -1})
|
|
}
|
|
}
|
|
return c
|
|
}
|