package pgha import ( "context" "sync" "time" "github.com/prometheus/client_golang/prometheus" ) // Collector publishes the store cluster's state as the control plane sees it // through Patroni's API: who leads, who streams, how far behind. Read at most // every fifteen seconds, however often it is scraped. type Collector struct { hosts []string status func(ctx context.Context, hosts []string) (Cluster, error) mu sync.Mutex at time.Time last Cluster err error up, member, lag *prometheus.Desc } // NewCollector reads the cluster from the Patroni of each host in turn. func NewCollector(hosts []string) *Collector { return &Collector{ hosts: hosts, status: HTTPStatus, up: prometheus.NewDesc("llmgw_store_patroni_up", "1 when a member's Patroni answered for the store's cluster.", nil, nil), member: prometheus.NewDesc("llmgw_store_member", "Each member of the store's cluster, by role (leader, replica) and state (running, streaming, …); always 1.", []string{"member", "host", "role", "state"}, nil), lag: prometheus.NewDesc("llmgw_store_replica_lag_bytes", "How far each replica is behind the leader; -1 when Patroni does not know.", []string{"member"}, nil), } } // Describe implements prometheus.Collector. func (c *Collector) Describe(ch chan<- *prometheus.Desc) { ch <- c.up ch <- c.member ch <- c.lag } // Collect implements prometheus.Collector. func (c *Collector) Collect(ch chan<- prometheus.Metric) { cl, err := c.read() if err != nil { ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, 0) return } ch <- prometheus.MustNewConstMetric(c.up, prometheus.GaugeValue, 1) for _, m := range cl.Members { ch <- prometheus.MustNewConstMetric(c.member, prometheus.GaugeValue, 1, m.Name, m.Host, m.Role, m.State) if m.Role != "leader" { ch <- prometheus.MustNewConstMetric(c.lag, prometheus.GaugeValue, float64(m.Lag), m.Name) } } } // Read is the cluster's state, at most fifteen seconds old. func (c *Collector) Read() (Cluster, error) { return c.read() } func (c *Collector) read() (Cluster, error) { c.mu.Lock() defer c.mu.Unlock() if time.Since(c.at) < 15*time.Second { return c.last, c.err } ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) defer cancel() c.last, c.err = c.status(ctx, c.hosts) if c.err == nil { c.last = withMissing(c.last, c.hosts) } c.at = time.Now() return c.last, c.err } // withMissing adds every host of the store address that the cluster does not // list, as a member in state "missing". A member that stops is dropped from // Patroni's list once its key expires — so without this a cluster down to // one machine reads as a healthy cluster of one, and nothing says a standby // is gone. func withMissing(c Cluster, hosts []string) Cluster { seen := map[string]bool{} for _, m := range c.Members { seen[m.Host] = true } for _, h := range hosts { if !seen[h] { c.Members = append(c.Members, MemberState{Name: h, Host: h, Role: "replica", State: "missing", Lag: -1}) } } return c }