Files
llmgw/internal/router/scrape/poller.go
T
Ilker ManapandClaude Opus 5 f19db242ae Stop naming a metric llama.cpp does not have, and say so when it happens
Deployed against real llama.cpp, every scrape failed. The mapping named
llamacpp:kv_cache_usage_ratio; build b10991 serves fifteen metric families
and none is a cache ratio. Because KV usage was mandatory, that one absent
name threw away the two metrics the engine does publish, and the router
scheduled to all three replicas blind while the cluster looked healthy.

A mapping that leaves a metric unnamed now means the engine does not publish
it, and the field stays zero. Naming one the engine does not serve is still a
violation, so this does not become a way to tolerate a typo. Running and
waiting can never be omitted: an engine that reports neither what it is doing
nor what is queued cannot be scheduled against at all. For llama.cpp this
costs the KV half of the saturation signal, which is a real loss softened by
its fixed slot count — queue depth already says when it is full.

The second half is why it took a deployment to notice. A contract violation
was logged per scrape at debug level, so at the default level nothing said
anything. It is now a warning, once, with a matching line when it clears.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 08:18:27 +02:00

234 lines
5.7 KiB
Go

package scrape
import (
"context"
"errors"
"fmt"
"log/slog"
"math/rand/v2"
"net/http"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// Config holds poller settings.
type Config struct {
// Interval is the scrape period. Plan §7.3 suggests 1-2s.
Interval time.Duration
// Timeout bounds a single scrape; it must be shorter than Interval.
Timeout time.Duration
// Path is the metrics endpoint.
Path string
// Mapping selects the metric names for endpoints that name no engine. An
// endpoint that names one is read with that engine's mapping instead.
Mapping Mapping
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
Interval: 2 * time.Second,
Timeout: time.Second,
Path: "/metrics",
Mapping: VLLMMapping(),
}
}
func (c *Config) normalize() {
if c.Interval <= 0 {
c.Interval = 2 * time.Second
}
if c.Timeout <= 0 || c.Timeout >= c.Interval {
c.Timeout = c.Interval / 2
}
if c.Path == "" {
c.Path = "/metrics"
}
c.Mapping.normalize()
}
// Poller scrapes each registered endpoint periodically, one goroutine each.
//
// Per-endpoint goroutines plus jitter: scraping every endpoint at the same
// instant loads both the engines and the router in bursts and synchronizes the
// measurements, which makes the scores oscillate in lockstep.
type Poller struct {
cfg Config
reg *endpoint.Registry
http *http.Client
log *slog.Logger
mu sync.Mutex
running map[string]context.CancelFunc
wg sync.WaitGroup
}
// NewPoller creates a poller. Call Sync to start it.
func NewPoller(reg *endpoint.Registry, cfg Config, log *slog.Logger) *Poller {
cfg.normalize()
if log == nil {
log = slog.Default()
}
return &Poller{
cfg: cfg,
reg: reg,
log: log,
http: &http.Client{
Timeout: cfg.Timeout,
Transport: &http.Transport{
MaxIdleConnsPerHost: 2,
DisableCompression: true,
},
},
running: make(map[string]context.CancelFunc),
}
}
// Sync starts and stops scrape goroutines to match the registry's endpoint
// list. Call it again whenever that list changes.
func (p *Poller) Sync(ctx context.Context) {
infos := p.reg.Infos()
p.mu.Lock()
defer p.mu.Unlock()
want := make(map[string]endpoint.Info, len(infos))
for _, in := range infos {
want[in.ID] = in
}
for id, cancel := range p.running {
if _, ok := want[id]; !ok {
cancel()
delete(p.running, id)
}
}
for id, in := range want {
if _, ok := p.running[id]; ok {
continue
}
cctx, cancel := context.WithCancel(ctx)
p.running[id] = cancel
p.wg.Add(1)
go func() {
defer p.wg.Done()
p.loop(cctx, in)
}()
}
}
// Stop halts every scrape goroutine and waits for them to finish.
func (p *Poller) Stop() {
p.mu.Lock()
for id, cancel := range p.running {
cancel()
delete(p.running, id)
}
p.mu.Unlock()
p.wg.Wait()
}
func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
// Jitter keeps goroutines that start together from staying in lockstep.
jitter := time.Duration(rand.Int64N(int64(p.cfg.Interval))) //nolint:gosec // jitter
select {
case <-ctx.Done():
return
case <-time.After(jitter):
}
t := time.NewTicker(p.cfg.Interval)
defer t.Stop()
// One goroutine per endpoint, so this needs no lock.
var reported string
p.once(ctx, in, &reported)
for {
select {
case <-ctx.Done():
return
case <-t.C:
p.once(ctx, in, &reported)
}
}
}
func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) {
stats, err := p.fetch(ctx, in.URL, p.mappingFor(in))
if err != nil {
if ctx.Err() != nil {
return
}
p.reg.ScrapeFailed(in.ID)
p.report(in, err, reported)
return
}
if *reported != "" {
p.log.Info("endpoint metrics are readable again", "endpoint", in.ID)
*reported = ""
}
p.reg.Observe(in.ID, stats)
}
// report logs a scrape failure the first time it appears and then stays quiet
// until the reason changes.
//
// Per-scrape debug logging was not enough. A contract violation is a
// compatibility problem, not a blip: every scrape fails, the router has no
// numbers for that endpoint and schedules to it blind, and at the default log
// level nothing says so. A cluster can look entirely healthy while none of its
// routing decisions are informed by anything.
func (p *Poller) report(in endpoint.Info, err error, reported *string) {
var missing *MissingError
if errors.As(err, &missing) {
msg := err.Error()
if *reported == msg {
return
}
*reported = msg
p.log.Warn("engine does not serve the metrics this endpoint is read with; "+
"routing to it is uninformed",
"endpoint", in.ID, "engine", in.Engine, "missing", missing.Missing)
return
}
// Ordinary failures — a restarting engine, a timeout — are expected and
// transient, so they stay quiet unless someone is looking.
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
*reported = "unreachable"
}
func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) {
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+p.cfg.Path, nil)
if err != nil {
return endpoint.Stats{}, err
}
resp, err := p.http.Do(req)
if err != nil {
return endpoint.Stats{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return endpoint.Stats{}, fmt.Errorf("unexpected status: %s", resp.Status)
}
return Parse(resp.Body, mapping)
}
// mappingFor picks the vocabulary this endpoint's metrics are written in.
//
// The configured mapping wins for endpoints that name no engine, so an operator
// can still override the names when a version renames them (plan §14).
func (p *Poller) mappingFor(in endpoint.Info) Mapping {
if in.Engine == "" {
return p.cfg.Mapping
}
return MappingFor(in.Engine)
}