Stop naming a metric llama.cpp does not have, and say so when it happens

Deployed against real llama.cpp, every scrape failed. The mapping named
llamacpp:kv_cache_usage_ratio; build b10991 serves fifteen metric families
and none is a cache ratio. Because KV usage was mandatory, that one absent
name threw away the two metrics the engine does publish, and the router
scheduled to all three replicas blind while the cluster looked healthy.

A mapping that leaves a metric unnamed now means the engine does not publish
it, and the field stays zero. Naming one the engine does not serve is still a
violation, so this does not become a way to tolerate a typo. Running and
waiting can never be omitted: an engine that reports neither what it is doing
nor what is queued cannot be scheduled against at all. For llama.cpp this
costs the KV half of the saturation signal, which is a real loss softened by
its fixed slot count — queue depth already says when it is full.

The second half is why it took a deployment to notice. A contract violation
was logged per scrape at debug level, so at the default level nothing said
anything. It is now a warning, once, with a matching line when it clears.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5 committed 2026-09-16 08:18:27 +02:00
1 parent 501db3d3e8
commit f19db242ae
4 files changed
+227 -29

No files matched your search

+129 -6
View File
@@ -1,9 +1,18 @@
package scrape
import (
"context"
"errors"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
)
// llamaMetrics is what llama-server --metrics actually serves, trimmed to the
@@ -14,9 +23,6 @@ llamacpp:prompt_tokens_total 1200
# HELP llamacpp:tokens_predicted_total Number of generation tokens processed.
# TYPE llamacpp:tokens_predicted_total counter
llamacpp:tokens_predicted_total 3400
# HELP llamacpp:kv_cache_usage_ratio KV-cache usage.
# TYPE llamacpp:kv_cache_usage_ratio gauge
llamacpp:kv_cache_usage_ratio 0.42
# HELP llamacpp:requests_processing Number of requests processing.
# TYPE llamacpp:requests_processing gauge
llamacpp:requests_processing 3
@@ -38,7 +44,9 @@ func TestLlamaCppMetricsSatisfyTheContract(t *testing.T) {
if s.Waiting != 7 {
t.Errorf("waiting=%d", s.Waiting)
}
if s.KVCacheUsage != 0.42 {
// Current llama.cpp publishes no cache ratio at all, so this stays zero
// rather than failing the whole scrape.
if s.KVCacheUsage != 0 {
t.Errorf("kv=%v", s.KVCacheUsage)
}
if s.PromptTokens != 1200 || s.GenerationTokens != 3400 {
@@ -52,7 +60,7 @@ func TestLlamaCppMappingDoesNotInheritVLLMNames(t *testing.T) {
// would hide the fact that this engine simply does not report it.
m := LlamaCppMapping()
m.normalize()
if m.SuccessTotal != "" || m.CacheConfig != "" {
if m.SuccessTotal != "" || m.CacheConfig != "" || m.KVCacheUsage != "" {
t.Errorf("vLLM names leaked into the llama.cpp mapping: %+v", m)
}
if m.Running != "llamacpp:requests_processing" {
@@ -78,7 +86,7 @@ func TestVLLMNamesAreNotReadAsLlamaCpp(t *testing.T) {
t.Fatalf("expected the contract to be reported incomplete, got %v", err)
}
if len(missing.Missing) != 3 {
t.Errorf("all three mandatory metrics should be named: %v", missing.Missing)
t.Errorf("every metric vLLM's mapping expects should be named: %v", missing.Missing)
}
}
@@ -94,3 +102,118 @@ func TestMappingForPicksByEngine(t *testing.T) {
}
}
}
func TestAMetricTheMappingNamesIsStillMandatory(t *testing.T) {
// Leaving a name out says "this engine does not publish it". Naming one the
// engine does not serve is a contract violation, and must stay one: the
// alternative is an endpoint quietly scheduled against numbers that are
// zero because nothing ever set them.
m := LlamaCppMapping()
m.KVCacheUsage = "llamacpp:kv_cache_usage_ratio"
var missing *MissingError
if _, err := Parse(strings.NewReader(llamaMetrics), m); !errors.As(err, &missing) {
t.Fatalf("a named but absent metric was tolerated: %v", err)
}
}
func TestRunningAndWaitingCannotBeOmitted(t *testing.T) {
// There is no engine worth routing to that reports neither how much it is
// doing nor how much is queued.
_, err := Parse(strings.NewReader(llamaMetrics), Mapping{
Running: "llamacpp:requests_processing", Waiting: "nope:waiting",
})
if err == nil {
t.Error("an endpoint with no queue signal was accepted")
}
}
// logCapture collects records so a test can assert what an operator would see.
type logCapture struct {
mu sync.Mutex
records []slog.Record
}
func (c *logCapture) Enabled(context.Context, slog.Level) bool { return true }
func (c *logCapture) WithAttrs([]slog.Attr) slog.Handler { return c }
func (c *logCapture) WithGroup(string) slog.Handler { return c }
func (c *logCapture) Handle(_ context.Context, r slog.Record) error {
c.mu.Lock()
defer c.mu.Unlock()
c.records = append(c.records, r)
return nil
}
func (c *logCapture) count(level slog.Level, substr string) int {
c.mu.Lock()
defer c.mu.Unlock()
n := 0
for _, r := range c.records {
if r.Level == level && strings.Contains(r.Message, substr) {
n++
}
}
return n
}
func TestABrokenContractIsReportedOnceAndLoudly(t *testing.T) {
// The failure this guards against is a cluster that looks healthy while
// none of its routing decisions are informed by anything. Debug-level
// logging was not enough: a contract violation fails every scrape forever,
// and at the default level nothing said so.
served := llamaMetrics
var mu sync.Mutex
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
mu.Lock()
defer mu.Unlock()
_, _ = io.WriteString(w, served)
}))
defer srv.Close()
cap := &logCapture{}
reg := endpoint.NewRegistry(endpoint.Config{})
in := endpoint.Info{ID: "e1", URL: srv.URL, Engine: "vllm", Models: []string{"m"}}
reg.Replace([]endpoint.Info{in})
p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 10 * time.Millisecond},
slog.New(cap))
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
p.Sync(ctx)
defer p.Stop()
// vLLM's names against llama.cpp's output: nothing matches.
deadline := time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) && cap.count(slog.LevelWarn, "does not serve the metrics") == 0 {
time.Sleep(10 * time.Millisecond)
}
if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 {
t.Fatalf("the violation was reported %d times; expected exactly one", n)
}
// It must not turn into a per-scrape stream either.
time.Sleep(200 * time.Millisecond)
if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 {
t.Errorf("the warning repeated: %d times", n)
}
// And recovery has to be visible, or an operator cannot tell a fixed
// cluster from a broken one.
mu.Lock()
served = `# TYPE vllm:num_requests_running gauge
vllm:num_requests_running 1
# TYPE vllm:num_requests_waiting gauge
vllm:num_requests_waiting 2
# TYPE vllm:kv_cache_usage_perc gauge
vllm:kv_cache_usage_perc 0.5
`
mu.Unlock()
deadline = time.Now().Add(2 * time.Second)
for time.Now().Before(deadline) && cap.count(slog.LevelInfo, "readable again") == 0 {
time.Sleep(10 * time.Millisecond)
}
if cap.count(slog.LevelInfo, "readable again") == 0 {
t.Error("recovery was never reported")
}
}
+27 -11
View File
@@ -50,15 +50,23 @@ func MappingFor(engine string) Mapping {
// exposes when started with --metrics.
//
// "Deferred" is llama.cpp's word for a request that arrived with every slot
// busy, which is the same thing vLLM calls waiting. The fields left empty are
// metrics llama.cpp does not have: it counts decodes rather than requests, so
// there is no success counter and no cache configuration to read block sizes
// from.
// busy, which is the same thing vLLM calls waiting.
//
// KV cache usage is deliberately absent: current llama.cpp does not publish it
// (b10991 serves fifteen families and none of them is a cache ratio), and
// naming a metric that is not there would fail the contract on every scrape and
// leave the router with no numbers at all rather than two out of three. The
// loss is smaller here than it would be for vLLM — llama.cpp serves a fixed
// number of slots, so its queue depth already says when it is full — but it is
// a loss: saturation falls back to queue depth alone.
//
// The rest are metrics llama.cpp does not have either. It counts decodes rather
// than requests, so there is no success counter and no cache configuration to
// read block sizes from.
func LlamaCppMapping() Mapping {
return Mapping{
Running: "llamacpp:requests_processing",
Waiting: "llamacpp:requests_deferred",
KVCacheUsage: "llamacpp:kv_cache_usage_ratio",
PromptTokens: "llamacpp:prompt_tokens_total",
GenerationTokens: "llamacpp:tokens_predicted_total",
}
@@ -122,8 +130,11 @@ func (e *MissingError) Error() string {
// Parse converts Prometheus text-format metrics into Stats.
//
// If any of the mandatory three (running, waiting, kv usage) is absent it
// returns *MissingError: routing to that endpoint would be blind.
// A metric the mapping names and the engine does not serve is a contract
// violation and returns *MissingError: routing to that endpoint would be blind.
// A metric the mapping leaves unnamed is one the engine is known not to
// publish, and stays zero — the router loses that signal for those endpoints
// rather than losing the endpoints.
func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) {
m.normalize()
@@ -138,6 +149,9 @@ func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) {
var s endpoint.Stats
var missing []string
// Running and waiting have no such escape: an engine that reports neither
// how much it is doing nor how much is queued cannot be scheduled against
// at all.
if v, ok := firstValue(fams, m.Running); ok {
s.Running = int(v)
} else {
@@ -148,10 +162,12 @@ func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) {
} else {
missing = append(missing, m.Waiting)
}
if v, ok := firstValue(fams, m.KVCacheUsage); ok {
s.KVCacheUsage = v
} else {
missing = append(missing, m.KVCacheUsage)
if m.KVCacheUsage != "" {
if v, ok := firstValue(fams, m.KVCacheUsage); ok {
s.KVCacheUsage = v
} else {
missing = append(missing, m.KVCacheUsage)
}
}
if len(missing) > 0 {
return endpoint.Stats{}, &MissingError{Missing: missing}
+40 -4
View File
@@ -2,6 +2,7 @@ package scrape
import (
"context"
"errors"
"fmt"
"log/slog"
"math/rand/v2"
@@ -141,30 +142,65 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) {
t := time.NewTicker(p.cfg.Interval)
defer t.Stop()
p.once(ctx, in)
// One goroutine per endpoint, so this needs no lock.
var reported string
p.once(ctx, in, &reported)
for {
select {
case <-ctx.Done():
return
case <-t.C:
p.once(ctx, in)
p.once(ctx, in, &reported)
}
}
}
func (p *Poller) once(ctx context.Context, in endpoint.Info) {
func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) {
stats, err := p.fetch(ctx, in.URL, p.mappingFor(in))
if err != nil {
if ctx.Err() != nil {
return
}
p.reg.ScrapeFailed(in.ID)
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
p.report(in, err, reported)
return
}
if *reported != "" {
p.log.Info("endpoint metrics are readable again", "endpoint", in.ID)
*reported = ""
}
p.reg.Observe(in.ID, stats)
}
// report logs a scrape failure the first time it appears and then stays quiet
// until the reason changes.
//
// Per-scrape debug logging was not enough. A contract violation is a
// compatibility problem, not a blip: every scrape fails, the router has no
// numbers for that endpoint and schedules to it blind, and at the default log
// level nothing says so. A cluster can look entirely healthy while none of its
// routing decisions are informed by anything.
func (p *Poller) report(in endpoint.Info, err error, reported *string) {
var missing *MissingError
if errors.As(err, &missing) {
msg := err.Error()
if *reported == msg {
return
}
*reported = msg
p.log.Warn("engine does not serve the metrics this endpoint is read with; "+
"routing to it is uninformed",
"endpoint", in.ID, "engine", in.Engine, "missing", missing.Missing)
return
}
// Ordinary failures — a restarting engine, a timeout — are expected and
// transient, so they stay quiet unless someone is looking.
p.log.Debug("scrape failed", "endpoint", in.ID, "err", err)
*reported = "unreachable"
}
func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) {
ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout)
defer cancel()