The code behind plan §7.3-7.6: - breaker: folds cooldown and circuit breaking into one component (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff and half-open probes. - endpoint: live state registry. Inflight counting, staleness detection, drain rate derived from counters. A scrape failure does not trip the breaker — the engine may be serving fine while only its metrics endpoint is down. - scrape: parses the §7.6 metric contract. Errors when any of the mandatory three is missing; the metric name mapping is overridable, since names drift between versions (plan §14). - scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a set of scorer weights; chat, batch and long-context are the same code with different weights. Scorers: queue, kv (hard penalty above the ceiling), inflight, load, weight, session (bounded-load consistent hashing). Pickers: max-score, p2c, softmax, round-robin. Decision carries the rationale — which scorer produced what, which filter dropped how many. Tuning the weights without that is guesswork. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
186 lines
4.0 KiB
Go
186 lines
4.0 KiB
Go
package scrape
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// Config holds poller settings.
|
|
type Config struct {
|
|
// Interval is the scrape period. Plan §7.3 suggests 1-2s.
|
|
Interval time.Duration
|
|
// Timeout bounds a single scrape; it must be shorter than Interval.
|
|
Timeout time.Duration
|
|
// Path is the metrics endpoint.
|
|
Path string
|
|
// Mapping selects the metric names.
|
|
Mapping Mapping
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
Interval: 2 * time.Second,
|
|
Timeout: time.Second,
|
|
Path: "/metrics",
|
|
Mapping: VLLMMapping(),
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
if c.Interval <= 0 {
|
|
c.Interval = 2 * time.Second
|
|
}
|
|
if c.Timeout <= 0 || c.Timeout >= c.Interval {
|
|
c.Timeout = c.Interval / 2
|
|
}
|
|
if c.Path == "" {
|
|
c.Path = "/metrics"
|
|
}
|
|
c.Mapping.normalize()
|
|
}
|
|
|
|
// Poller scrapes each registered endpoint periodically, one goroutine each.
|
|
//
|
|
// Per-endpoint goroutines plus jitter: scraping every endpoint at the same
|
|
// instant loads both the engines and the router in bursts and synchronizes the
|
|
// measurements, which makes the scores oscillate in lockstep.
|
|
type Poller struct {
|
|
cfg Config
|
|
reg *endpoint.Registry
|
|
http *http.Client
|
|
log *slog.Logger
|
|
|
|
mu sync.Mutex
|
|
running map[string]context.CancelFunc
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
// NewPoller creates a poller. Call Sync to start it.
|
|
func NewPoller(reg *endpoint.Registry, cfg Config, log *slog.Logger) *Poller {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Poller{
|
|
cfg: cfg,
|
|
reg: reg,
|
|
log: log,
|
|
http: &http.Client{
|
|
Timeout: cfg.Timeout,
|
|
Transport: &http.Transport{
|
|
MaxIdleConnsPerHost: 2,
|
|
DisableCompression: true,
|
|
},
|
|
},
|
|
running: make(map[string]context.CancelFunc),
|
|
}
|
|
}
|
|
|
|
// Sync starts and stops scrape goroutines to match the registry's endpoint
|
|
// list. Call it again whenever that list changes.
|
|
func (p *Poller) Sync(ctx context.Context) {
|
|
infos := p.reg.Infos()
|
|
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
|
|
want := make(map[string]endpoint.Info, len(infos))
|
|
for _, in := range infos {
|
|
want[in.ID] = in
|
|
}
|
|
|
|
for id, cancel := range p.running {
|
|
if _, ok := want[id]; !ok {
|
|
cancel()
|
|
delete(p.running, id)
|
|
}
|
|
}
|
|
for id, in := range want {
|
|
if _, ok := p.running[id]; ok {
|
|
continue
|
|
}
|
|
cctx, cancel := context.WithCancel(ctx)
|
|
p.running[id] = cancel
|
|
p.wg.Add(1)
|
|
go func() {
|
|
defer p.wg.Done()
|
|
p.loop(cctx, in)
|
|
}()
|
|
}
|
|
}
|
|
|
|
// Stop halts every scrape goroutine and waits for them to finish.
|
|
func (p *Poller) Stop() {
|
|
p.mu.Lock()
|
|
for id, cancel := range p.running {
|
|
cancel()
|
|
delete(p.running, id)
|
|
}
|
|
p.mu.Unlock()
|
|
p.wg.Wait()
|
|
}
|
|
|
|
func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
|
|
// Jitter keeps goroutines that start together from staying in lockstep.
|
|
jitter := time.Duration(rand.Int64N(int64(p.cfg.Interval))) //nolint:gosec // jitter
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(jitter):
|
|
}
|
|
|
|
t := time.NewTicker(p.cfg.Interval)
|
|
defer t.Stop()
|
|
|
|
p.once(ctx, in)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
p.once(ctx, in)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *Poller) once(ctx context.Context, in endpoint.Info) {
|
|
stats, err := p.fetch(ctx, in.URL)
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
p.reg.ScrapeFailed(in.ID)
|
|
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
|
|
return
|
|
}
|
|
p.reg.Observe(in.ID, stats)
|
|
}
|
|
|
|
func (p *Poller) fetch(ctx context.Context, base string) (endpoint.Stats, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+p.cfg.Path, nil)
|
|
if err != nil {
|
|
return endpoint.Stats{}, err
|
|
}
|
|
resp, err := p.http.Do(req)
|
|
if err != nil {
|
|
return endpoint.Stats{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return endpoint.Stats{}, fmt.Errorf("unexpected status: %s", resp.Status)
|
|
}
|
|
return Parse(resp.Body, p.cfg.Mapping)
|
|
}
|