llama.cpp is what makes a cluster with no cards useful: it serves the same OpenAI API on CPU, so the whole path — placement, supervision, scraping, scheduling, admission — can be exercised end to end on ordinary machines. Three things had to give way for it. A profile may now omit its GPU class, and that is how it says it runs on the CPU. Only engines that can actually serve without a card may omit it; for vLLM an omitted class stays an error, because otherwise a typo becomes a replica placed on a machine with no hardware and diagnosed minutes later during model loading. Placement spreads CPU replicas instead of bin packing them, since unlike GPUs there is no allocation to stop two of them sharing a machine's cores. Metric mappings are now per engine rather than per cluster. llama.cpp calls the same three numbers requests_processing, requests_deferred and kv_cache_usage_ratio, and a single global mapping would leave half a mixed fleet unmeasurable. An empty mapping still defaults to vLLM's names; one that names its own Running metric is taken as a complete statement, so the fields it leaves blank stay blank rather than being filled with names that engine cannot have. The adapter refuses multi-node and tensor-parallel profiles. llama.cpp splits across local GPUs by a different mechanism than this adapter drives, and accepting either would produce a replica that starts and then serves from one machine while the catalog believes it has several. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
217 lines
6.1 KiB
Go
217 lines
6.1 KiB
Go
// Package scrape reads engines' /metrics output and turns it into something
|
|
// the router can make decisions from (plan §7.6).
|
|
package scrape
|
|
|
|
import (
|
|
"fmt"
|
|
"io"
|
|
"strconv"
|
|
|
|
dto "github.com/prometheus/client_model/go"
|
|
"github.com/prometheus/common/expfmt"
|
|
"github.com/prometheus/common/model"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// Mapping says which metric names the contract fields are read from.
|
|
//
|
|
// Being configurable is essential: engine versions rename their metrics
|
|
// (plan §14). Once the agent reports an engine version the control plane can
|
|
// pick the right mapping; in phase 1 it comes from configuration.
|
|
type Mapping struct {
|
|
Running string
|
|
Waiting string
|
|
KVCacheUsage string
|
|
|
|
CacheConfig string
|
|
PromptTokens string
|
|
GenerationTokens string
|
|
SuccessTotal string
|
|
}
|
|
|
|
// MappingFor returns the metric names for an engine.
|
|
//
|
|
// Per engine rather than per cluster: a cluster running llama.cpp on the CPU
|
|
// nodes and vLLM on the GPU nodes has two vocabularies for the same three
|
|
// numbers, and one global mapping would leave half the fleet unmeasurable.
|
|
// An unknown engine falls back to vLLM's names, which is also what an endpoint
|
|
// that names no engine gets.
|
|
func MappingFor(engine string) Mapping {
|
|
switch engine {
|
|
case "llamacpp":
|
|
return LlamaCppMapping()
|
|
default:
|
|
return VLLMMapping()
|
|
}
|
|
}
|
|
|
|
// LlamaCppMapping returns llama.cpp's metric names, which llama-server only
|
|
// exposes when started with --metrics.
|
|
//
|
|
// "Deferred" is llama.cpp's word for a request that arrived with every slot
|
|
// busy, which is the same thing vLLM calls waiting. The fields left empty are
|
|
// metrics llama.cpp does not have: it counts decodes rather than requests, so
|
|
// there is no success counter and no cache configuration to read block sizes
|
|
// from.
|
|
func LlamaCppMapping() Mapping {
|
|
return Mapping{
|
|
Running: "llamacpp:requests_processing",
|
|
Waiting: "llamacpp:requests_deferred",
|
|
KVCacheUsage: "llamacpp:kv_cache_usage_ratio",
|
|
PromptTokens: "llamacpp:prompt_tokens_total",
|
|
GenerationTokens: "llamacpp:tokens_predicted_total",
|
|
}
|
|
}
|
|
|
|
// VLLMMapping returns vLLM's current metric names.
|
|
func VLLMMapping() Mapping {
|
|
return Mapping{
|
|
Running: "vllm:num_requests_running",
|
|
Waiting: "vllm:num_requests_waiting",
|
|
KVCacheUsage: "vllm:kv_cache_usage_perc",
|
|
CacheConfig: "vllm:cache_config_info",
|
|
PromptTokens: "vllm:prompt_tokens_total",
|
|
GenerationTokens: "vllm:generation_tokens_total",
|
|
SuccessTotal: "vllm:request_success_total",
|
|
}
|
|
}
|
|
|
|
// normalize fills in vLLM's names for an empty mapping.
|
|
//
|
|
// Only for an empty one. A mapping that names its own Running metric is a
|
|
// complete statement about some other engine, and the fields it leaves blank
|
|
// are metrics that engine does not expose — filling those with vLLM's names
|
|
// would have the router hunting for metrics that cannot be there.
|
|
func (m *Mapping) normalize() {
|
|
if m.Running != "" {
|
|
return
|
|
}
|
|
d := VLLMMapping()
|
|
if m.Running == "" {
|
|
m.Running = d.Running
|
|
}
|
|
if m.Waiting == "" {
|
|
m.Waiting = d.Waiting
|
|
}
|
|
if m.KVCacheUsage == "" {
|
|
m.KVCacheUsage = d.KVCacheUsage
|
|
}
|
|
if m.CacheConfig == "" {
|
|
m.CacheConfig = d.CacheConfig
|
|
}
|
|
if m.PromptTokens == "" {
|
|
m.PromptTokens = d.PromptTokens
|
|
}
|
|
if m.GenerationTokens == "" {
|
|
m.GenerationTokens = d.GenerationTokens
|
|
}
|
|
if m.SuccessTotal == "" {
|
|
m.SuccessTotal = d.SuccessTotal
|
|
}
|
|
}
|
|
|
|
// MissingError reports that mandatory contract metrics are absent.
|
|
type MissingError struct {
|
|
Missing []string
|
|
}
|
|
|
|
func (e *MissingError) Error() string {
|
|
return fmt.Sprintf("engine metric contract incomplete: %v", e.Missing)
|
|
}
|
|
|
|
// Parse converts Prometheus text-format metrics into Stats.
|
|
//
|
|
// If any of the mandatory three (running, waiting, kv usage) is absent it
|
|
// returns *MissingError: routing to that endpoint would be blind.
|
|
func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) {
|
|
m.normalize()
|
|
|
|
// TextParser's zero value is invalid; the scheme must be explicit.
|
|
// The legacy scheme accepts vLLM's names containing ':'.
|
|
p := expfmt.NewTextParser(model.LegacyValidation)
|
|
fams, err := p.TextToMetricFamilies(r)
|
|
if err != nil {
|
|
return endpoint.Stats{}, fmt.Errorf("failed to parse metrics: %w", err)
|
|
}
|
|
|
|
var s endpoint.Stats
|
|
var missing []string
|
|
|
|
if v, ok := firstValue(fams, m.Running); ok {
|
|
s.Running = int(v)
|
|
} else {
|
|
missing = append(missing, m.Running)
|
|
}
|
|
if v, ok := firstValue(fams, m.Waiting); ok {
|
|
s.Waiting = int(v)
|
|
} else {
|
|
missing = append(missing, m.Waiting)
|
|
}
|
|
if v, ok := firstValue(fams, m.KVCacheUsage); ok {
|
|
s.KVCacheUsage = v
|
|
} else {
|
|
missing = append(missing, m.KVCacheUsage)
|
|
}
|
|
if len(missing) > 0 {
|
|
return endpoint.Stats{}, &MissingError{Missing: missing}
|
|
}
|
|
|
|
// Optional fields: silently stay zero when absent.
|
|
s.PromptTokens, _ = sumValues(fams, m.PromptTokens)
|
|
s.GenerationTokens, _ = sumValues(fams, m.GenerationTokens)
|
|
// request_success_total is split by finished_reason; sum across labels.
|
|
s.SuccessTotal, _ = sumValues(fams, m.SuccessTotal)
|
|
|
|
if f, ok := fams[m.CacheConfig]; ok && len(f.Metric) > 0 {
|
|
for _, lp := range f.Metric[0].Label {
|
|
switch lp.GetName() {
|
|
case "block_size":
|
|
s.BlockSize, _ = strconv.Atoi(lp.GetValue())
|
|
case "num_gpu_blocks":
|
|
s.NumGPUBlocks, _ = strconv.Atoi(lp.GetValue())
|
|
}
|
|
}
|
|
}
|
|
|
|
return s, nil
|
|
}
|
|
|
|
// metricValue extracts a numeric value regardless of the metric type.
|
|
func metricValue(mt *dto.Metric) (float64, bool) {
|
|
switch {
|
|
case mt.Gauge != nil:
|
|
return mt.Gauge.GetValue(), true
|
|
case mt.Counter != nil:
|
|
return mt.Counter.GetValue(), true
|
|
case mt.Untyped != nil:
|
|
return mt.Untyped.GetValue(), true
|
|
default:
|
|
return 0, false
|
|
}
|
|
}
|
|
|
|
func firstValue(fams map[string]*dto.MetricFamily, name string) (float64, bool) {
|
|
f, ok := fams[name]
|
|
if !ok || len(f.Metric) == 0 {
|
|
return 0, false
|
|
}
|
|
return metricValue(f.Metric[0])
|
|
}
|
|
|
|
func sumValues(fams map[string]*dto.MetricFamily, name string) (float64, bool) {
|
|
f, ok := fams[name]
|
|
if !ok {
|
|
return 0, false
|
|
}
|
|
var sum float64
|
|
var any bool
|
|
for _, mt := range f.Metric {
|
|
if v, ok := metricValue(mt); ok {
|
|
sum += v
|
|
any = true
|
|
}
|
|
}
|
|
return sum, any
|
|
}
|