From f19db242ae9813902fa9da0041cb56d24cf641e6 Mon Sep 17 00:00:00 2001 From: Ilker Manap Date: Wed, 16 Sep 2026 08:18:27 +0200 Subject: [PATCH] Stop naming a metric llama.cpp does not have, and say so when it happens MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deployed against real llama.cpp, every scrape failed. The mapping named llamacpp:kv_cache_usage_ratio; build b10991 serves fifteen metric families and none is a cache ratio. Because KV usage was mandatory, that one absent name threw away the two metrics the engine does publish, and the router scheduled to all three replicas blind while the cluster looked healthy. A mapping that leaves a metric unnamed now means the engine does not publish it, and the field stays zero. Naming one the engine does not serve is still a violation, so this does not become a way to tolerate a typo. Running and waiting can never be omitted: an engine that reports neither what it is doing nor what is queued cannot be scheduled against at all. For llama.cpp this costs the KV half of the saturation signal, which is a real loss softened by its fixed slot count — queue depth already says when it is full. The second half is why it took a deployment to notice. A contract violation was logged per scrape at debug level, so at the default level nothing said anything. It is now a warning, once, with a matching line when it clears. Co-Authored-By: Claude Opus 5 (1M context) --- internal/router/scrape/llamacpp_test.go | 135 ++++++++++++++++++++++-- internal/router/scrape/parse.go | 38 +++++-- internal/router/scrape/poller.go | 44 +++++++- llmgw-plan.md | 39 +++++-- 4 files changed, 227 insertions(+), 29 deletions(-) diff --git a/internal/router/scrape/llamacpp_test.go b/internal/router/scrape/llamacpp_test.go index cf0d4ce..a03e23b 100644 --- a/internal/router/scrape/llamacpp_test.go +++ b/internal/router/scrape/llamacpp_test.go @@ -1,9 +1,18 @@ package scrape import ( + "context" "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" "strings" + "sync" "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // llamaMetrics is what llama-server --metrics actually serves, trimmed to the @@ -14,9 +23,6 @@ llamacpp:prompt_tokens_total 1200 # HELP llamacpp:tokens_predicted_total Number of generation tokens processed. # TYPE llamacpp:tokens_predicted_total counter llamacpp:tokens_predicted_total 3400 -# HELP llamacpp:kv_cache_usage_ratio KV-cache usage. -# TYPE llamacpp:kv_cache_usage_ratio gauge -llamacpp:kv_cache_usage_ratio 0.42 # HELP llamacpp:requests_processing Number of requests processing. # TYPE llamacpp:requests_processing gauge llamacpp:requests_processing 3 @@ -38,7 +44,9 @@ func TestLlamaCppMetricsSatisfyTheContract(t *testing.T) { if s.Waiting != 7 { t.Errorf("waiting=%d", s.Waiting) } - if s.KVCacheUsage != 0.42 { + // Current llama.cpp publishes no cache ratio at all, so this stays zero + // rather than failing the whole scrape. + if s.KVCacheUsage != 0 { t.Errorf("kv=%v", s.KVCacheUsage) } if s.PromptTokens != 1200 || s.GenerationTokens != 3400 { @@ -52,7 +60,7 @@ func TestLlamaCppMappingDoesNotInheritVLLMNames(t *testing.T) { // would hide the fact that this engine simply does not report it. m := LlamaCppMapping() m.normalize() - if m.SuccessTotal != "" || m.CacheConfig != "" { + if m.SuccessTotal != "" || m.CacheConfig != "" || m.KVCacheUsage != "" { t.Errorf("vLLM names leaked into the llama.cpp mapping: %+v", m) } if m.Running != "llamacpp:requests_processing" { @@ -78,7 +86,7 @@ func TestVLLMNamesAreNotReadAsLlamaCpp(t *testing.T) { t.Fatalf("expected the contract to be reported incomplete, got %v", err) } if len(missing.Missing) != 3 { - t.Errorf("all three mandatory metrics should be named: %v", missing.Missing) + t.Errorf("every metric vLLM's mapping expects should be named: %v", missing.Missing) } } @@ -94,3 +102,118 @@ func TestMappingForPicksByEngine(t *testing.T) { } } } + +func TestAMetricTheMappingNamesIsStillMandatory(t *testing.T) { + // Leaving a name out says "this engine does not publish it". Naming one the + // engine does not serve is a contract violation, and must stay one: the + // alternative is an endpoint quietly scheduled against numbers that are + // zero because nothing ever set them. + m := LlamaCppMapping() + m.KVCacheUsage = "llamacpp:kv_cache_usage_ratio" + + var missing *MissingError + if _, err := Parse(strings.NewReader(llamaMetrics), m); !errors.As(err, &missing) { + t.Fatalf("a named but absent metric was tolerated: %v", err) + } +} + +func TestRunningAndWaitingCannotBeOmitted(t *testing.T) { + // There is no engine worth routing to that reports neither how much it is + // doing nor how much is queued. + _, err := Parse(strings.NewReader(llamaMetrics), Mapping{ + Running: "llamacpp:requests_processing", Waiting: "nope:waiting", + }) + if err == nil { + t.Error("an endpoint with no queue signal was accepted") + } +} + +// logCapture collects records so a test can assert what an operator would see. +type logCapture struct { + mu sync.Mutex + records []slog.Record +} + +func (c *logCapture) Enabled(context.Context, slog.Level) bool { return true } +func (c *logCapture) WithAttrs([]slog.Attr) slog.Handler { return c } +func (c *logCapture) WithGroup(string) slog.Handler { return c } +func (c *logCapture) Handle(_ context.Context, r slog.Record) error { + c.mu.Lock() + defer c.mu.Unlock() + c.records = append(c.records, r) + return nil +} + +func (c *logCapture) count(level slog.Level, substr string) int { + c.mu.Lock() + defer c.mu.Unlock() + n := 0 + for _, r := range c.records { + if r.Level == level && strings.Contains(r.Message, substr) { + n++ + } + } + return n +} + +func TestABrokenContractIsReportedOnceAndLoudly(t *testing.T) { + // The failure this guards against is a cluster that looks healthy while + // none of its routing decisions are informed by anything. Debug-level + // logging was not enough: a contract violation fails every scrape forever, + // and at the default level nothing said so. + served := llamaMetrics + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + defer mu.Unlock() + _, _ = io.WriteString(w, served) + })) + defer srv.Close() + + cap := &logCapture{} + reg := endpoint.NewRegistry(endpoint.Config{}) + in := endpoint.Info{ID: "e1", URL: srv.URL, Engine: "vllm", Models: []string{"m"}} + reg.Replace([]endpoint.Info{in}) + + p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 10 * time.Millisecond}, + slog.New(cap)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Sync(ctx) + defer p.Stop() + + // vLLM's names against llama.cpp's output: nothing matches. + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && cap.count(slog.LevelWarn, "does not serve the metrics") == 0 { + time.Sleep(10 * time.Millisecond) + } + if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 { + t.Fatalf("the violation was reported %d times; expected exactly one", n) + } + + // It must not turn into a per-scrape stream either. + time.Sleep(200 * time.Millisecond) + if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 { + t.Errorf("the warning repeated: %d times", n) + } + + // And recovery has to be visible, or an operator cannot tell a fixed + // cluster from a broken one. + mu.Lock() + served = `# TYPE vllm:num_requests_running gauge +vllm:num_requests_running 1 +# TYPE vllm:num_requests_waiting gauge +vllm:num_requests_waiting 2 +# TYPE vllm:kv_cache_usage_perc gauge +vllm:kv_cache_usage_perc 0.5 +` + mu.Unlock() + + deadline = time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && cap.count(slog.LevelInfo, "readable again") == 0 { + time.Sleep(10 * time.Millisecond) + } + if cap.count(slog.LevelInfo, "readable again") == 0 { + t.Error("recovery was never reported") + } +} diff --git a/internal/router/scrape/parse.go b/internal/router/scrape/parse.go index ee68ac9..379bba8 100644 --- a/internal/router/scrape/parse.go +++ b/internal/router/scrape/parse.go @@ -50,15 +50,23 @@ func MappingFor(engine string) Mapping { // exposes when started with --metrics. // // "Deferred" is llama.cpp's word for a request that arrived with every slot -// busy, which is the same thing vLLM calls waiting. The fields left empty are -// metrics llama.cpp does not have: it counts decodes rather than requests, so -// there is no success counter and no cache configuration to read block sizes -// from. +// busy, which is the same thing vLLM calls waiting. +// +// KV cache usage is deliberately absent: current llama.cpp does not publish it +// (b10991 serves fifteen families and none of them is a cache ratio), and +// naming a metric that is not there would fail the contract on every scrape and +// leave the router with no numbers at all rather than two out of three. The +// loss is smaller here than it would be for vLLM — llama.cpp serves a fixed +// number of slots, so its queue depth already says when it is full — but it is +// a loss: saturation falls back to queue depth alone. +// +// The rest are metrics llama.cpp does not have either. It counts decodes rather +// than requests, so there is no success counter and no cache configuration to +// read block sizes from. func LlamaCppMapping() Mapping { return Mapping{ Running: "llamacpp:requests_processing", Waiting: "llamacpp:requests_deferred", - KVCacheUsage: "llamacpp:kv_cache_usage_ratio", PromptTokens: "llamacpp:prompt_tokens_total", GenerationTokens: "llamacpp:tokens_predicted_total", } @@ -122,8 +130,11 @@ func (e *MissingError) Error() string { // Parse converts Prometheus text-format metrics into Stats. // -// If any of the mandatory three (running, waiting, kv usage) is absent it -// returns *MissingError: routing to that endpoint would be blind. +// A metric the mapping names and the engine does not serve is a contract +// violation and returns *MissingError: routing to that endpoint would be blind. +// A metric the mapping leaves unnamed is one the engine is known not to +// publish, and stays zero — the router loses that signal for those endpoints +// rather than losing the endpoints. func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) { m.normalize() @@ -138,6 +149,9 @@ func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) { var s endpoint.Stats var missing []string + // Running and waiting have no such escape: an engine that reports neither + // how much it is doing nor how much is queued cannot be scheduled against + // at all. if v, ok := firstValue(fams, m.Running); ok { s.Running = int(v) } else { @@ -148,10 +162,12 @@ func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) { } else { missing = append(missing, m.Waiting) } - if v, ok := firstValue(fams, m.KVCacheUsage); ok { - s.KVCacheUsage = v - } else { - missing = append(missing, m.KVCacheUsage) + if m.KVCacheUsage != "" { + if v, ok := firstValue(fams, m.KVCacheUsage); ok { + s.KVCacheUsage = v + } else { + missing = append(missing, m.KVCacheUsage) + } } if len(missing) > 0 { return endpoint.Stats{}, &MissingError{Missing: missing} diff --git a/internal/router/scrape/poller.go b/internal/router/scrape/poller.go index 0a3aaea..f9242da 100644 --- a/internal/router/scrape/poller.go +++ b/internal/router/scrape/poller.go @@ -2,6 +2,7 @@ package scrape import ( "context" + "errors" "fmt" "log/slog" "math/rand/v2" @@ -141,30 +142,65 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) { t := time.NewTicker(p.cfg.Interval) defer t.Stop() - p.once(ctx, in) + // One goroutine per endpoint, so this needs no lock. + var reported string + + p.once(ctx, in, &reported) for { select { case <-ctx.Done(): return case <-t.C: - p.once(ctx, in) + p.once(ctx, in, &reported) } } } -func (p *Poller) once(ctx context.Context, in endpoint.Info) { +func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) { stats, err := p.fetch(ctx, in.URL, p.mappingFor(in)) if err != nil { if ctx.Err() != nil { return } p.reg.ScrapeFailed(in.ID) - p.log.Debug("scrape failed", "endpoint", in.ID, "err", err) + p.report(in, err, reported) return } + if *reported != "" { + p.log.Info("endpoint metrics are readable again", "endpoint", in.ID) + *reported = "" + } p.reg.Observe(in.ID, stats) } +// report logs a scrape failure the first time it appears and then stays quiet +// until the reason changes. +// +// Per-scrape debug logging was not enough. A contract violation is a +// compatibility problem, not a blip: every scrape fails, the router has no +// numbers for that endpoint and schedules to it blind, and at the default log +// level nothing says so. A cluster can look entirely healthy while none of its +// routing decisions are informed by anything. +func (p *Poller) report(in endpoint.Info, err error, reported *string) { + var missing *MissingError + if errors.As(err, &missing) { + msg := err.Error() + if *reported == msg { + return + } + *reported = msg + p.log.Warn("engine does not serve the metrics this endpoint is read with; "+ + "routing to it is uninformed", + "endpoint", in.ID, "engine", in.Engine, "missing", missing.Missing) + return + } + + // Ordinary failures — a restarting engine, a timeout — are expected and + // transient, so they stay quiet unless someone is looking. + p.log.Debug("scrape failed", "endpoint", in.ID, "err", err) + *reported = "unreachable" +} + func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) { ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout) defer cancel() diff --git a/llmgw-plan.md b/llmgw-plan.md index af0bfa6..00f3ae0 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -791,16 +791,35 @@ restarts, its index is reset. ### 7.6 The engine metric contract -This is the formal definition of "adding a new engine". If an engine exposes these three metrics +This is the formal definition of "adding a new engine". If an engine exposes these metrics in Prometheus format, llmgw can route to it (GIE's model server protocol): -| Contract field | vLLM metric | Mandatory | -|---|---|---| -| `Waiting` | `vllm:num_requests_waiting` | yes | -| `Running` | `vllm:num_requests_running` | yes | -| `KVCacheUsage` | `vllm:kv_cache_usage_perc` | yes | -| `BlockSize`, `NumGPUBlocks` | `vllm:cache_config_info` | no | -| LoRA state | `vllm:lora_requests_info` (`max_lora`, `running_lora_adapters`, `waiting_lora_adapters`) | no | +| Contract field | vLLM metric | llama.cpp metric | Mandatory | +|---|---|---|---| +| `Waiting` | `vllm:num_requests_waiting` | `llamacpp:requests_deferred` | yes | +| `Running` | `vllm:num_requests_running` | `llamacpp:requests_processing` | yes | +| `KVCacheUsage` | `vllm:kv_cache_usage_perc` | — | where the engine has it | +| `PromptTokens` | `vllm:prompt_tokens_total` | `llamacpp:prompt_tokens_total` | no | +| `GenerationTokens` | `vllm:generation_tokens_total` | `llamacpp:tokens_predicted_total` | no | +| `BlockSize`, `NumGPUBlocks` | `vllm:cache_config_info` | — | no | +| LoRA state | `vllm:lora_requests_info` (`max_lora`, `running_lora_adapters`, `waiting_lora_adapters`) | — | no | + +Mappings are **per engine, not per cluster**. A cluster running llama.cpp on its CPU nodes and +vLLM on its GPU nodes has two vocabularies for the same numbers, and one global mapping would +leave half the fleet unmeasurable. An endpoint that names no engine is read with vLLM's names. + +`Running` and `Waiting` are the two that cannot be omitted: an engine that reports neither how +much it is doing nor how much is queued cannot be scheduled against at all. `KVCacheUsage` is +required *of engines that publish it* — a mapping that names it and does not get it is a contract +violation, while a mapping that leaves it unnamed is a statement that this engine has no such +metric. Current llama.cpp is the second case, and saturation there falls back to queue depth +alone. That is a real loss of signal, mitigated by llama.cpp serving a fixed number of slots, so +its queue depth already says when it is full. + +> Correction: an earlier version of this plan gave llama.cpp's KV metric as +> `llamacpp:kv_cache_usage_ratio`. Build b10991 does not publish it — it serves fifteen metric +> families and none is a cache ratio. Naming it failed the contract on every scrape and left the +> router with no numbers at all rather than two out of three. > Correction: v0.1 said `vllm:gpu_cache_usage_perc`; the correct name is > `vllm:kv_cache_usage_perc`. Metric names drift between versions, so the `ParseMetrics` mapping @@ -811,6 +830,10 @@ Scrape rules: - Every Ready endpoint every 1–2s, one goroutine per endpoint, with jitter. - A failed scrape marks the endpoint stale; its score drops but it is not removed. - Three consecutive failures trip the breaker (§7.7). +- A **contract violation is reported at warn level, once**, and again when it clears. It is a + compatibility problem rather than a blip: every scrape fails, the router schedules to that + endpoint blind, and per-scrape debug logging meant a cluster could look entirely healthy while + none of its routing decisions were informed by anything. - **So that decisions are never made purely on stale data**, the `inflight` scorer (§7.4) stays enabled at all times.