Deployed against real llama.cpp, every scrape failed. The mapping named llamacpp:kv_cache_usage_ratio; build b10991 serves fifteen metric families and none is a cache ratio. Because KV usage was mandatory, that one absent name threw away the two metrics the engine does publish, and the router scheduled to all three replicas blind while the cluster looked healthy. A mapping that leaves a metric unnamed now means the engine does not publish it, and the field stays zero. Naming one the engine does not serve is still a violation, so this does not become a way to tolerate a typo. Running and waiting can never be omitted: an engine that reports neither what it is doing nor what is queued cannot be scheduled against at all. For llama.cpp this costs the KV half of the saturation signal, which is a real loss softened by its fixed slot count — queue depth already says when it is full. The second half is why it took a deployment to notice. A contract violation was logged per scrape at debug level, so at the default level nothing said anything. It is now a warning, once, with a matching line when it clears. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
234 lines
5.7 KiB
Go
234 lines
5.7 KiB
Go
package scrape
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// Config holds poller settings.
|
|
type Config struct {
|
|
// Interval is the scrape period. Plan §7.3 suggests 1-2s.
|
|
Interval time.Duration
|
|
// Timeout bounds a single scrape; it must be shorter than Interval.
|
|
Timeout time.Duration
|
|
// Path is the metrics endpoint.
|
|
Path string
|
|
// Mapping selects the metric names for endpoints that name no engine. An
|
|
// endpoint that names one is read with that engine's mapping instead.
|
|
Mapping Mapping
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
Interval: 2 * time.Second,
|
|
Timeout: time.Second,
|
|
Path: "/metrics",
|
|
Mapping: VLLMMapping(),
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
if c.Interval <= 0 {
|
|
c.Interval = 2 * time.Second
|
|
}
|
|
if c.Timeout <= 0 || c.Timeout >= c.Interval {
|
|
c.Timeout = c.Interval / 2
|
|
}
|
|
if c.Path == "" {
|
|
c.Path = "/metrics"
|
|
}
|
|
c.Mapping.normalize()
|
|
}
|
|
|
|
// Poller scrapes each registered endpoint periodically, one goroutine each.
|
|
//
|
|
// Per-endpoint goroutines plus jitter: scraping every endpoint at the same
|
|
// instant loads both the engines and the router in bursts and synchronizes the
|
|
// measurements, which makes the scores oscillate in lockstep.
|
|
type Poller struct {
|
|
cfg Config
|
|
reg *endpoint.Registry
|
|
http *http.Client
|
|
log *slog.Logger
|
|
|
|
mu sync.Mutex
|
|
running map[string]context.CancelFunc
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
// NewPoller creates a poller. Call Sync to start it.
|
|
func NewPoller(reg *endpoint.Registry, cfg Config, log *slog.Logger) *Poller {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Poller{
|
|
cfg: cfg,
|
|
reg: reg,
|
|
log: log,
|
|
http: &http.Client{
|
|
Timeout: cfg.Timeout,
|
|
Transport: &http.Transport{
|
|
MaxIdleConnsPerHost: 2,
|
|
DisableCompression: true,
|
|
},
|
|
},
|
|
running: make(map[string]context.CancelFunc),
|
|
}
|
|
}
|
|
|
|
// Sync starts and stops scrape goroutines to match the registry's endpoint
|
|
// list. Call it again whenever that list changes.
|
|
func (p *Poller) Sync(ctx context.Context) {
|
|
infos := p.reg.Infos()
|
|
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
|
|
want := make(map[string]endpoint.Info, len(infos))
|
|
for _, in := range infos {
|
|
want[in.ID] = in
|
|
}
|
|
|
|
for id, cancel := range p.running {
|
|
if _, ok := want[id]; !ok {
|
|
cancel()
|
|
delete(p.running, id)
|
|
}
|
|
}
|
|
for id, in := range want {
|
|
if _, ok := p.running[id]; ok {
|
|
continue
|
|
}
|
|
cctx, cancel := context.WithCancel(ctx)
|
|
p.running[id] = cancel
|
|
p.wg.Add(1)
|
|
go func() {
|
|
defer p.wg.Done()
|
|
p.loop(cctx, in)
|
|
}()
|
|
}
|
|
}
|
|
|
|
// Stop halts every scrape goroutine and waits for them to finish.
|
|
func (p *Poller) Stop() {
|
|
p.mu.Lock()
|
|
for id, cancel := range p.running {
|
|
cancel()
|
|
delete(p.running, id)
|
|
}
|
|
p.mu.Unlock()
|
|
p.wg.Wait()
|
|
}
|
|
|
|
func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
|
|
// Jitter keeps goroutines that start together from staying in lockstep.
|
|
jitter := time.Duration(rand.Int64N(int64(p.cfg.Interval))) //nolint:gosec // jitter
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(jitter):
|
|
}
|
|
|
|
t := time.NewTicker(p.cfg.Interval)
|
|
defer t.Stop()
|
|
|
|
// One goroutine per endpoint, so this needs no lock.
|
|
var reported string
|
|
|
|
p.once(ctx, in, &reported)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
p.once(ctx, in, &reported)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) {
|
|
stats, err := p.fetch(ctx, in.URL, p.mappingFor(in))
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
p.reg.ScrapeFailed(in.ID)
|
|
p.report(in, err, reported)
|
|
return
|
|
}
|
|
if *reported != "" {
|
|
p.log.Info("endpoint metrics are readable again", "endpoint", in.ID)
|
|
*reported = ""
|
|
}
|
|
p.reg.Observe(in.ID, stats)
|
|
}
|
|
|
|
// report logs a scrape failure the first time it appears and then stays quiet
|
|
// until the reason changes.
|
|
//
|
|
// Per-scrape debug logging was not enough. A contract violation is a
|
|
// compatibility problem, not a blip: every scrape fails, the router has no
|
|
// numbers for that endpoint and schedules to it blind, and at the default log
|
|
// level nothing says so. A cluster can look entirely healthy while none of its
|
|
// routing decisions are informed by anything.
|
|
func (p *Poller) report(in endpoint.Info, err error, reported *string) {
|
|
var missing *MissingError
|
|
if errors.As(err, &missing) {
|
|
msg := err.Error()
|
|
if *reported == msg {
|
|
return
|
|
}
|
|
*reported = msg
|
|
p.log.Warn("engine does not serve the metrics this endpoint is read with; "+
|
|
"routing to it is uninformed",
|
|
"endpoint", in.ID, "engine", in.Engine, "missing", missing.Missing)
|
|
return
|
|
}
|
|
|
|
// Ordinary failures — a restarting engine, a timeout — are expected and
|
|
// transient, so they stay quiet unless someone is looking.
|
|
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
|
|
*reported = "unreachable"
|
|
}
|
|
|
|
func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+p.cfg.Path, nil)
|
|
if err != nil {
|
|
return endpoint.Stats{}, err
|
|
}
|
|
resp, err := p.http.Do(req)
|
|
if err != nil {
|
|
return endpoint.Stats{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return endpoint.Stats{}, fmt.Errorf("unexpected status: %s", resp.Status)
|
|
}
|
|
return Parse(resp.Body, mapping)
|
|
}
|
|
|
|
// mappingFor picks the vocabulary this endpoint's metrics are written in.
|
|
//
|
|
// The configured mapping wins for endpoints that name no engine, so an operator
|
|
// can still override the names when a version renames them (plan §14).
|
|
func (p *Poller) mappingFor(in endpoint.Info) Mapping {
|
|
if in.Engine == "" {
|
|
return p.cfg.Mapping
|
|
}
|
|
return MappingFor(in.Engine)
|
|
}
|