Files
llmgw/internal/router/scrape/poller.go
T
ilkerandClaude Opus 5 b7417e1a69 Phase 1 (1/2): scheduler pipeline, endpoint registry, metric scraping
The code behind plan §7.3-7.6:

- breaker: folds cooldown and circuit breaking into one component
  (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff
  and half-open probes.
- endpoint: live state registry. Inflight counting, staleness detection, drain
  rate derived from counters. A scrape failure does not trip the breaker — the
  engine may be serving fine while only its metrics endpoint is down.
- scrape: parses the §7.6 metric contract. Errors when any of the mandatory
  three is missing; the metric name mapping is overridable, since names drift
  between versions (plan §14).
- scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a
  set of scorer weights; chat, batch and long-context are the same code with
  different weights. Scorers: queue, kv (hard penalty above the ceiling),
  inflight, load, weight, session (bounded-load consistent hashing). Pickers:
  max-score, p2c, softmax, round-robin.

Decision carries the rationale — which scorer produced what, which filter
dropped how many. Tuning the weights without that is guesswork.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-09-16 01:15:21 +02:00

186 lines
4.0 KiB
Go

package scrape
import (
"context"
"fmt"
"log/slog"
"math/rand/v2"
"net/http"
"sync"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// Config holds poller settings.
type Config struct {
// Interval is the scrape period. Plan §7.3 suggests 1-2s.
Interval time.Duration
// Timeout bounds a single scrape; it must be shorter than Interval.
Timeout time.Duration
// Path is the metrics endpoint.
Path string
// Mapping selects the metric names.
Mapping Mapping
}
// DefaultConfig returns sensible defaults.
func DefaultConfig() Config {
return Config{
Interval: 2 * time.Second,
Timeout: time.Second,
Path: "/metrics",
Mapping: VLLMMapping(),
}
}
func (c *Config) normalize() {
if c.Interval <= 0 {
c.Interval = 2 * time.Second
}
if c.Timeout <= 0 || c.Timeout >= c.Interval {
c.Timeout = c.Interval / 2
}
if c.Path == "" {
c.Path = "/metrics"
}
c.Mapping.normalize()
}
// Poller scrapes each registered endpoint periodically, one goroutine each.
//
// Per-endpoint goroutines plus jitter: scraping every endpoint at the same
// instant loads both the engines and the router in bursts and synchronizes the
// measurements, which makes the scores oscillate in lockstep.
type Poller struct {
cfg Config
reg *endpoint.Registry
http *http.Client
log *slog.Logger
mu sync.Mutex
running map[string]context.CancelFunc
wg sync.WaitGroup
}
// NewPoller creates a poller. Call Sync to start it.
func NewPoller(reg *endpoint.Registry, cfg Config, log *slog.Logger) *Poller {
cfg.normalize()
if log == nil {
log = slog.Default()
}
return &Poller{
cfg: cfg,
reg: reg,
log: log,
http: &http.Client{
Timeout: cfg.Timeout,
Transport: &http.Transport{
MaxIdleConnsPerHost: 2,
DisableCompression: true,
},
},
running: make(map[string]context.CancelFunc),
}
}
// Sync starts and stops scrape goroutines to match the registry's endpoint
// list. Call it again whenever that list changes.
func (p *Poller) Sync(ctx context.Context) {
infos := p.reg.Infos()
p.mu.Lock()
defer p.mu.Unlock()
want := make(map[string]endpoint.Info, len(infos))
for _, in := range infos {
want[in.ID] = in
}
for id, cancel := range p.running {
if _, ok := want[id]; !ok {
cancel()
delete(p.running, id)
}
}
for id, in := range want {
if _, ok := p.running[id]; ok {
continue
}
cctx, cancel := context.WithCancel(ctx)
p.running[id] = cancel
p.wg.Add(1)
go func() {
defer p.wg.Done()
p.loop(cctx, in)
}()
}
}
// Stop halts every scrape goroutine and waits for them to finish.
func (p *Poller) Stop() {
p.mu.Lock()
for id, cancel := range p.running {
cancel()
delete(p.running, id)
}
p.mu.Unlock()
p.wg.Wait()
}
func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
// Jitter keeps goroutines that start together from staying in lockstep.
jitter := time.Duration(rand.Int64N(int64(p.cfg.Interval))) //nolint:gosec // jitter
select {
case <-ctx.Done():
return
case <-time.After(jitter):
}
t := time.NewTicker(p.cfg.Interval)
defer t.Stop()
p.once(ctx, in)
for {
select {
case <-ctx.Done():
return
case <-t.C:
p.once(ctx, in)
}
}
}
func (p *Poller) once(ctx context.Context, in endpoint.Info) {
stats, err := p.fetch(ctx, in.URL)
if err != nil {
if ctx.Err() != nil {
return
}
p.reg.ScrapeFailed(in.ID)
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
return
}
p.reg.Observe(in.ID, stats)
}
func (p *Poller) fetch(ctx context.Context, base string) (endpoint.Stats, error) {
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
defer cancel()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+p.cfg.Path, nil)
if err != nil {
return endpoint.Stats{}, err
}
resp, err := p.http.Do(req)
if err != nil {
return endpoint.Stats{}, err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return endpoint.Stats{}, fmt.Errorf("unexpected status: %s", resp.Status)
}
return Parse(resp.Body, p.cfg.Mapping)
}