llama.cpp is what makes a cluster with no cards useful: it serves the same OpenAI API on CPU, so the whole path — placement, supervision, scraping, scheduling, admission — can be exercised end to end on ordinary machines. Three things had to give way for it. A profile may now omit its GPU class, and that is how it says it runs on the CPU. Only engines that can actually serve without a card may omit it; for vLLM an omitted class stays an error, because otherwise a typo becomes a replica placed on a machine with no hardware and diagnosed minutes later during model loading. Placement spreads CPU replicas instead of bin packing them, since unlike GPUs there is no allocation to stop two of them sharing a machine's cores. Metric mappings are now per engine rather than per cluster. llama.cpp calls the same three numbers requests_processing, requests_deferred and kv_cache_usage_ratio, and a single global mapping would leave half a mixed fleet unmeasurable. An empty mapping still defaults to vLLM's names; one that names its own Running metric is taken as a complete statement, so the fields it leaves blank stay blank rather than being filled with names that engine cannot have. The adapter refuses multi-node and tensor-parallel profiles. llama.cpp splits across local GPUs by a different mechanism than this adapter drives, and accepting either would produce a replica that starts and then serves from one machine while the catalog believes it has several. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
198 lines
4.5 KiB
Go
198 lines
4.5 KiB
Go
package scrape
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"log/slog"
|
|
"math/rand/v2"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// Config holds poller settings.
|
|
type Config struct {
|
|
// Interval is the scrape period. Plan §7.3 suggests 1-2s.
|
|
Interval time.Duration
|
|
// Timeout bounds a single scrape; it must be shorter than Interval.
|
|
Timeout time.Duration
|
|
// Path is the metrics endpoint.
|
|
Path string
|
|
// Mapping selects the metric names for endpoints that name no engine. An
|
|
// endpoint that names one is read with that engine's mapping instead.
|
|
Mapping Mapping
|
|
}
|
|
|
|
// DefaultConfig returns sensible defaults.
|
|
func DefaultConfig() Config {
|
|
return Config{
|
|
Interval: 2 * time.Second,
|
|
Timeout: time.Second,
|
|
Path: "/metrics",
|
|
Mapping: VLLMMapping(),
|
|
}
|
|
}
|
|
|
|
func (c *Config) normalize() {
|
|
if c.Interval <= 0 {
|
|
c.Interval = 2 * time.Second
|
|
}
|
|
if c.Timeout <= 0 || c.Timeout >= c.Interval {
|
|
c.Timeout = c.Interval / 2
|
|
}
|
|
if c.Path == "" {
|
|
c.Path = "/metrics"
|
|
}
|
|
c.Mapping.normalize()
|
|
}
|
|
|
|
// Poller scrapes each registered endpoint periodically, one goroutine each.
|
|
//
|
|
// Per-endpoint goroutines plus jitter: scraping every endpoint at the same
|
|
// instant loads both the engines and the router in bursts and synchronizes the
|
|
// measurements, which makes the scores oscillate in lockstep.
|
|
type Poller struct {
|
|
cfg Config
|
|
reg *endpoint.Registry
|
|
http *http.Client
|
|
log *slog.Logger
|
|
|
|
mu sync.Mutex
|
|
running map[string]context.CancelFunc
|
|
wg sync.WaitGroup
|
|
}
|
|
|
|
// NewPoller creates a poller. Call Sync to start it.
|
|
func NewPoller(reg *endpoint.Registry, cfg Config, log *slog.Logger) *Poller {
|
|
cfg.normalize()
|
|
if log == nil {
|
|
log = slog.Default()
|
|
}
|
|
return &Poller{
|
|
cfg: cfg,
|
|
reg: reg,
|
|
log: log,
|
|
http: &http.Client{
|
|
Timeout: cfg.Timeout,
|
|
Transport: &http.Transport{
|
|
MaxIdleConnsPerHost: 2,
|
|
DisableCompression: true,
|
|
},
|
|
},
|
|
running: make(map[string]context.CancelFunc),
|
|
}
|
|
}
|
|
|
|
// Sync starts and stops scrape goroutines to match the registry's endpoint
|
|
// list. Call it again whenever that list changes.
|
|
func (p *Poller) Sync(ctx context.Context) {
|
|
infos := p.reg.Infos()
|
|
|
|
p.mu.Lock()
|
|
defer p.mu.Unlock()
|
|
|
|
want := make(map[string]endpoint.Info, len(infos))
|
|
for _, in := range infos {
|
|
want[in.ID] = in
|
|
}
|
|
|
|
for id, cancel := range p.running {
|
|
if _, ok := want[id]; !ok {
|
|
cancel()
|
|
delete(p.running, id)
|
|
}
|
|
}
|
|
for id, in := range want {
|
|
if _, ok := p.running[id]; ok {
|
|
continue
|
|
}
|
|
cctx, cancel := context.WithCancel(ctx)
|
|
p.running[id] = cancel
|
|
p.wg.Add(1)
|
|
go func() {
|
|
defer p.wg.Done()
|
|
p.loop(cctx, in)
|
|
}()
|
|
}
|
|
}
|
|
|
|
// Stop halts every scrape goroutine and waits for them to finish.
|
|
func (p *Poller) Stop() {
|
|
p.mu.Lock()
|
|
for id, cancel := range p.running {
|
|
cancel()
|
|
delete(p.running, id)
|
|
}
|
|
p.mu.Unlock()
|
|
p.wg.Wait()
|
|
}
|
|
|
|
func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
|
|
// Jitter keeps goroutines that start together from staying in lockstep.
|
|
jitter := time.Duration(rand.Int64N(int64(p.cfg.Interval))) //nolint:gosec // jitter
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-time.After(jitter):
|
|
}
|
|
|
|
t := time.NewTicker(p.cfg.Interval)
|
|
defer t.Stop()
|
|
|
|
p.once(ctx, in)
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-t.C:
|
|
p.once(ctx, in)
|
|
}
|
|
}
|
|
}
|
|
|
|
func (p *Poller) once(ctx context.Context, in endpoint.Info) {
|
|
stats, err := p.fetch(ctx, in.URL, p.mappingFor(in))
|
|
if err != nil {
|
|
if ctx.Err() != nil {
|
|
return
|
|
}
|
|
p.reg.ScrapeFailed(in.ID)
|
|
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
|
|
return
|
|
}
|
|
p.reg.Observe(in.ID, stats)
|
|
}
|
|
|
|
func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) {
|
|
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
|
|
defer cancel()
|
|
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+p.cfg.Path, nil)
|
|
if err != nil {
|
|
return endpoint.Stats{}, err
|
|
}
|
|
resp, err := p.http.Do(req)
|
|
if err != nil {
|
|
return endpoint.Stats{}, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
if resp.StatusCode != http.StatusOK {
|
|
return endpoint.Stats{}, fmt.Errorf("unexpected status: %s", resp.Status)
|
|
}
|
|
return Parse(resp.Body, mapping)
|
|
}
|
|
|
|
// mappingFor picks the vocabulary this endpoint's metrics are written in.
|
|
//
|
|
// The configured mapping wins for endpoints that name no engine, so an operator
|
|
// can still override the names when a version renames them (plan §14).
|
|
func (p *Poller) mappingFor(in endpoint.Info) Mapping {
|
|
if in.Engine == "" {
|
|
return p.cfg.Mapping
|
|
}
|
|
return MappingFor(in.Engine)
|
|
}
|