Stop naming a metric llama.cpp does not have, and say so when it happens

Deployed against real llama.cpp, every scrape failed. The mapping named
llamacpp:kv_cache_usage_ratio; build b10991 serves fifteen metric families
and none is a cache ratio. Because KV usage was mandatory, that one absent
name threw away the two metrics the engine does publish, and the router
scheduled to all three replicas blind while the cluster looked healthy.

A mapping that leaves a metric unnamed now means the engine does not publish
it, and the field stays zero. Naming one the engine does not serve is still a
violation, so this does not become a way to tolerate a typo. Running and
waiting can never be omitted: an engine that reports neither what it is doing
nor what is queued cannot be scheduled against at all. For llama.cpp this
costs the KV half of the saturation signal, which is a real loss softened by
its fixed slot count — queue depth already says when it is full.

The second half is why it took a deployment to notice. A contract violation
was logged per scrape at debug level, so at the default level nothing said
anything. It is now a warning, once, with a matching line when it clears.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 08:18:27 +02:00
1 parent 501db3d3e8
commit f19db242ae
4 files changed
+227 -29

No files matched your search

+129 -6
View File
@@ -1,9 +1,18 @@
package scrape
import (
"context"
"errors"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// llamaMetrics is what llama-server --metrics actually serves, trimmed to the
@@ -14,9 +23,6 @@ llamacpp:prompt_tokens_total 1200
# HELP llamacpp:tokens_predicted_total Number of generation tokens processed.
# TYPE llamacpp:tokens_predicted_total counter
llamacpp:tokens_predicted_total 3400
# HELP llamacpp:kv_cache_usage_ratio KV-cache usage.
# TYPE llamacpp:kv_cache_usage_ratio gauge
llamacpp:kv_cache_usage_ratio 0.42
# HELP llamacpp:requests_processing Number of requests processing.
# TYPE llamacpp:requests_processing gauge
llamacpp:requests_processing 3
@@ -38,7 +44,9 @@ func TestLlamaCppMetricsSatisfyTheContract(t *testing.T) {
if s.Waiting != 7 {
t.Errorf("waiting=%d", s.Waiting)
}
if s.KVCacheUsage != 0.42 {
// Current llama.cpp publishes no cache ratio at all, so this stays zero
// rather than failing the whole scrape.
if s.KVCacheUsage != 0 {
t.Errorf("kv=%v", s.KVCacheUsage)
}
if s.PromptTokens != 1200 || s.GenerationTokens != 3400 {
@@ -52,7 +60,7 @@ func TestLlamaCppMappingDoesNotInheritVLLMNames(t *testing.T) {
// would hide the fact that this engine simply does not report it.
m := LlamaCppMapping()
m.normalize()
if m.SuccessTotal != "" || m.CacheConfig != "" {
if m.SuccessTotal != "" || m.CacheConfig != "" || m.KVCacheUsage != "" {
t.Errorf("vLLM names leaked into the llama.cpp mapping: %+v", m)
}
if m.Running != "llamacpp:requests_processing" {
@@ -78,7 +86,7 @@ func TestVLLMNamesAreNotReadAsLlamaCpp(t *testing.T) {
t.Fatalf("expected the contract to be reported incomplete, got %v", err)
}
if len(missing.Missing) != 3 {
t.Errorf("all three mandatory metrics should be named: %v", missing.Missing)
t.Errorf("every metric vLLM's mapping expects should be named: %v", missing.Missing)
}
}
@@ -94,3 +102,118 @@ func TestMappingForPicksByEngine(t *testing.T) {
}
}
}
func TestAMetricTheMappingNamesIsStillMandatory(t *testing.T) {
// Leaving a name out says "this engine does not publish it". Naming one the
// engine does not serve is a contract violation, and must stay one: the
// alternative is an endpoint quietly scheduled against numbers that are
// zero because nothing ever set them.
m := LlamaCppMapping()
m.KVCacheUsage = "llamacpp:kv_cache_usage_ratio"
var missing *MissingError
if _, err := Parse(strings.NewReader(llamaMetrics), m); !errors.As(err, &missing) {
t.Fatalf("a named but absent metric was tolerated: %v", err)
}
}
func TestRunningAndWaitingCannotBeOmitted(t *testing.T) {
// There is no engine worth routing to that reports neither how much it is
// doing nor how much is queued.
_, err := Parse(strings.NewReader(llamaMetrics), Mapping{
Running: "llamacpp:requests_processing", Waiting: "nope:waiting",
})
if err == nil {
t.Error("an endpoint with no queue signal was accepted")
}
}
// logCapture collects records so a test can assert what an operator would see.
type logCapture struct {
mu sync.Mutex
records []slog.Record
}
func (c *logCapture) Enabled(context.Context, slog.Level) bool { return true }
func (c *logCapture) WithAttrs([]slog.Attr) slog.Handler { return c }
func (c *logCapture) WithGroup(string) slog.Handler { return c }
func (c *logCapture) Handle(_ context.Context, r slog.Record) error {
c.mu.Lock()
defer c.mu.Unlock()
c.records = append(c.records, r)
return nil
}
func (c *logCapture) count(level slog.Level, substr string) int {
c.mu.Lock()
defer c.mu.Unlock()
n := 0
for _, r := range c.records {
if r.Level == level && strings.Contains(r.Message, substr) {
n++
}
}
return n
}
func TestABrokenContractIsReportedOnceAndLoudly(t *testing.T) {
// The failure this guards against is a cluster that looks healthy while
// none of its routing decisions are informed by anything. Debug-level
// logging was not enough: a contract violation fails every scrape forever,
// and at the default level nothing said so.
served := llamaMetrics
var mu sync.Mutex
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
mu.Lock()
defer mu.Unlock()
_, _ = io.WriteString(w, served)
}))
defer srv.Close()
cap := &logCapture{}
reg := endpoint.NewRegistry(endpoint.Config{})
in := endpoint.Info{ID: "e1", URL: srv.URL, Engine: "vllm", Models: []string{"m"}}
reg.Replace([]endpoint.Info{in})
p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 10 * time.Millisecond},
slog.New(cap))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Sync(ctx)
defer p.Stop()
// vLLM's names against llama.cpp's output: nothing matches.
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) && cap.count(slog.LevelWarn, "does not serve the metrics") == 0 {
time.Sleep(10 * time.Millisecond)
}
if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 {
t.Fatalf("the violation was reported %d times; expected exactly one", n)
}
// It must not turn into a per-scrape stream either.
time.Sleep(200 * time.Millisecond)
if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 {
t.Errorf("the warning repeated: %d times", n)
}
// And recovery has to be visible, or an operator cannot tell a fixed
// cluster from a broken one.
mu.Lock()
served = `# TYPE vllm:num_requests_running gauge
vllm:num_requests_running 1
# TYPE vllm:num_requests_waiting gauge
vllm:num_requests_waiting 2
# TYPE vllm:kv_cache_usage_perc gauge
vllm:kv_cache_usage_perc 0.5
`
mu.Unlock()
deadline = time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) && cap.count(slog.LevelInfo, "readable again") == 0 {
time.Sleep(10 * time.Millisecond)
}
if cap.count(slog.LevelInfo, "readable again") == 0 {
t.Error("recovery was never reported")
}
}
+27 -11
View File
@@ -50,15 +50,23 @@ func MappingFor(engine string) Mapping {
// exposes when started with --metrics.
//
// "Deferred" is llama.cpp's word for a request that arrived with every slot
// busy, which is the same thing vLLM calls waiting. The fields left empty are
// metrics llama.cpp does not have: it counts decodes rather than requests, so
// there is no success counter and no cache configuration to read block sizes
// from.
// busy, which is the same thing vLLM calls waiting.
//
// KV cache usage is deliberately absent: current llama.cpp does not publish it
// (b10991 serves fifteen families and none of them is a cache ratio), and
// naming a metric that is not there would fail the contract on every scrape and
// leave the router with no numbers at all rather than two out of three. The
// loss is smaller here than it would be for vLLM — llama.cpp serves a fixed
// number of slots, so its queue depth already says when it is full — but it is
// a loss: saturation falls back to queue depth alone.
//
// The rest are metrics llama.cpp does not have either. It counts decodes rather
// than requests, so there is no success counter and no cache configuration to
// read block sizes from.
func LlamaCppMapping() Mapping {
return Mapping{
Running: "llamacpp:requests_processing",
Waiting: "llamacpp:requests_deferred",
KVCacheUsage: "llamacpp:kv_cache_usage_ratio",
PromptTokens: "llamacpp:prompt_tokens_total",
GenerationTokens: "llamacpp:tokens_predicted_total",
}
@@ -122,8 +130,11 @@ func (e *MissingError) Error() string {
// Parse converts Prometheus text-format metrics into Stats.
//
// If any of the mandatory three (running, waiting, kv usage) is absent it
// returns *MissingError: routing to that endpoint would be blind.
// A metric the mapping names and the engine does not serve is a contract
// violation and returns *MissingError: routing to that endpoint would be blind.
// A metric the mapping leaves unnamed is one the engine is known not to
// publish, and stays zero — the router loses that signal for those endpoints
// rather than losing the endpoints.
func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) {
m.normalize()
@@ -138,6 +149,9 @@ func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) {
var s endpoint.Stats
var missing []string
// Running and waiting have no such escape: an engine that reports neither
// how much it is doing nor how much is queued cannot be scheduled against
// at all.
if v, ok := firstValue(fams, m.Running); ok {
s.Running = int(v)
} else {
@@ -148,10 +162,12 @@ func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) {
} else {
missing = append(missing, m.Waiting)
}
if v, ok := firstValue(fams, m.KVCacheUsage); ok {
s.KVCacheUsage = v
} else {
missing = append(missing, m.KVCacheUsage)
if m.KVCacheUsage != "" {
if v, ok := firstValue(fams, m.KVCacheUsage); ok {
s.KVCacheUsage = v
} else {
missing = append(missing, m.KVCacheUsage)
}
}
if len(missing) > 0 {
return endpoint.Stats{}, &MissingError{Missing: missing}
+40 -4
View File
@@ -2,6 +2,7 @@ package scrape
import (
"context"
"errors"
"fmt"
"log/slog"
"math/rand/v2"
@@ -141,30 +142,65 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
t := time.NewTicker(p.cfg.Interval)
defer t.Stop()
p.once(ctx, in)
// One goroutine per endpoint, so this needs no lock.
var reported string
p.once(ctx, in, &reported)
for {
select {
case <-ctx.Done():
return
case <-t.C:
p.once(ctx, in)
p.once(ctx, in, &reported)
}
}
}
func (p *Poller) once(ctx context.Context, in endpoint.Info) {
func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) {
stats, err := p.fetch(ctx, in.URL, p.mappingFor(in))
if err != nil {
if ctx.Err() != nil {
return
}
p.reg.ScrapeFailed(in.ID)
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
p.report(in, err, reported)
return
}
if *reported != "" {
p.log.Info("endpoint metrics are readable again", "endpoint", in.ID)
*reported = ""
}
p.reg.Observe(in.ID, stats)
}
// report logs a scrape failure the first time it appears and then stays quiet
// until the reason changes.
//
// Per-scrape debug logging was not enough. A contract violation is a
// compatibility problem, not a blip: every scrape fails, the router has no
// numbers for that endpoint and schedules to it blind, and at the default log
// level nothing says so. A cluster can look entirely healthy while none of its
// routing decisions are informed by anything.
func (p *Poller) report(in endpoint.Info, err error, reported *string) {
var missing *MissingError
if errors.As(err, &missing) {
msg := err.Error()
if *reported == msg {
return
}
*reported = msg
p.log.Warn("engine does not serve the metrics this endpoint is read with; "+
"routing to it is uninformed",
"endpoint", in.ID, "engine", in.Engine, "missing", missing.Missing)
return
}
// Ordinary failures — a restarting engine, a timeout — are expected and
// transient, so they stay quiet unless someone is looking.
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
*reported = "unreachable"
}
func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) {
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
defer cancel()
+31 -8
View File
@@ -791,16 +791,35 @@ restarts, its index is reset.
### 7.6 The engine metric contract
This is the formal definition of "adding a new engine". If an engine exposes these three metrics
This is the formal definition of "adding a new engine". If an engine exposes these metrics
in Prometheus format, llmgw can route to it (GIE's model server protocol):
| Contract field | vLLM metric | Mandatory |
|---|---|---|
| `Waiting` | `vllm:num_requests_waiting` | yes |
| `Running` | `vllm:num_requests_running` | yes |
| `KVCacheUsage` | `vllm:kv_cache_usage_perc` | yes |
| `BlockSize`, `NumGPUBlocks` | `vllm:cache_config_info` | no |
| LoRA state | `vllm:lora_requests_info` (`max_lora`, `running_lora_adapters`, `waiting_lora_adapters`) | no |
| Contract field | vLLM metric | llama.cpp metric | Mandatory |
|---|---|---|---|
| `Waiting` | `vllm:num_requests_waiting` | `llamacpp:requests_deferred` | yes |
| `Running` | `vllm:num_requests_running` | `llamacpp:requests_processing` | yes |
| `KVCacheUsage` | `vllm:kv_cache_usage_perc` | — | where the engine has it |
| `PromptTokens` | `vllm:prompt_tokens_total` | `llamacpp:prompt_tokens_total` | no |
| `GenerationTokens` | `vllm:generation_tokens_total` | `llamacpp:tokens_predicted_total` | no |
| `BlockSize`, `NumGPUBlocks` | `vllm:cache_config_info` | — | no |
| LoRA state | `vllm:lora_requests_info` (`max_lora`, `running_lora_adapters`, `waiting_lora_adapters`) | — | no |
Mappings are **per engine, not per cluster**. A cluster running llama.cpp on its CPU nodes and
vLLM on its GPU nodes has two vocabularies for the same numbers, and one global mapping would
leave half the fleet unmeasurable. An endpoint that names no engine is read with vLLM's names.
`Running` and `Waiting` are the two that cannot be omitted: an engine that reports neither how
much it is doing nor how much is queued cannot be scheduled against at all. `KVCacheUsage` is
required *of engines that publish it* — a mapping that names it and does not get it is a contract
violation, while a mapping that leaves it unnamed is a statement that this engine has no such
metric. Current llama.cpp is the second case, and saturation there falls back to queue depth
alone. That is a real loss of signal, mitigated by llama.cpp serving a fixed number of slots, so
its queue depth already says when it is full.
> Correction: an earlier version of this plan gave llama.cpp's KV metric as
> `llamacpp:kv_cache_usage_ratio`. Build b10991 does not publish it — it serves fifteen metric
> families and none is a cache ratio. Naming it failed the contract on every scrape and left the
> router with no numbers at all rather than two out of three.
> Correction: v0.1 said `vllm:gpu_cache_usage_perc`; the correct name is
> `vllm:kv_cache_usage_perc`. Metric names drift between versions, so the `ParseMetrics` mapping
@@ -811,6 +830,10 @@ Scrape rules:
- Every Ready endpoint every 1–2s, one goroutine per endpoint, with jitter.
- A failed scrape marks the endpoint stale; its score drops but it is not removed.
- Three consecutive failures trip the breaker (§7.7).
- A **contract violation is reported at warn level, once**, and again when it clears. It is a
compatibility problem rather than a blip: every scrape fails, the router schedules to that
endpoint blind, and per-scrape debug logging meant a cluster could look entirely healthy while
none of its routing decisions were informed by anything.
- **So that decisions are never made purely on stale data**, the `inflight` scorer (§7.4) stays
enabled at all times.