diff --git a/internal/router/scrape/llamacpp_test.go b/internal/router/scrape/llamacpp_test.go index cf0d4ce..a03e23b 100644 --- a/internal/router/scrape/llamacpp_test.go +++ b/internal/router/scrape/llamacpp_test.go @@ -1,9 +1,18 @@ package scrape import ( + "context" "errors" + "io" + "log/slog" + "net/http" + "net/http/httptest" "strings" + "sync" "testing" + "time" + + "gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint" ) // llamaMetrics is what llama-server --metrics actually serves, trimmed to the @@ -14,9 +23,6 @@ llamacpp:prompt_tokens_total 1200 # HELP llamacpp:tokens_predicted_total Number of generation tokens processed. # TYPE llamacpp:tokens_predicted_total counter llamacpp:tokens_predicted_total 3400 -# HELP llamacpp:kv_cache_usage_ratio KV-cache usage. -# TYPE llamacpp:kv_cache_usage_ratio gauge -llamacpp:kv_cache_usage_ratio 0.42 # HELP llamacpp:requests_processing Number of requests processing. # TYPE llamacpp:requests_processing gauge llamacpp:requests_processing 3 @@ -38,7 +44,9 @@ func TestLlamaCppMetricsSatisfyTheContract(t *testing.T) { if s.Waiting != 7 { t.Errorf("waiting=%d", s.Waiting) } - if s.KVCacheUsage != 0.42 { + // Current llama.cpp publishes no cache ratio at all, so this stays zero + // rather than failing the whole scrape. + if s.KVCacheUsage != 0 { t.Errorf("kv=%v", s.KVCacheUsage) } if s.PromptTokens != 1200 || s.GenerationTokens != 3400 { @@ -52,7 +60,7 @@ func TestLlamaCppMappingDoesNotInheritVLLMNames(t *testing.T) { // would hide the fact that this engine simply does not report it. m := LlamaCppMapping() m.normalize() - if m.SuccessTotal != "" || m.CacheConfig != "" { + if m.SuccessTotal != "" || m.CacheConfig != "" || m.KVCacheUsage != "" { t.Errorf("vLLM names leaked into the llama.cpp mapping: %+v", m) } if m.Running != "llamacpp:requests_processing" { @@ -78,7 +86,7 @@ func TestVLLMNamesAreNotReadAsLlamaCpp(t *testing.T) { t.Fatalf("expected the contract to be reported incomplete, got %v", err) } if len(missing.Missing) != 3 { - t.Errorf("all three mandatory metrics should be named: %v", missing.Missing) + t.Errorf("every metric vLLM's mapping expects should be named: %v", missing.Missing) } } @@ -94,3 +102,118 @@ func TestMappingForPicksByEngine(t *testing.T) { } } } + +func TestAMetricTheMappingNamesIsStillMandatory(t *testing.T) { + // Leaving a name out says "this engine does not publish it". Naming one the + // engine does not serve is a contract violation, and must stay one: the + // alternative is an endpoint quietly scheduled against numbers that are + // zero because nothing ever set them. + m := LlamaCppMapping() + m.KVCacheUsage = "llamacpp:kv_cache_usage_ratio" + + var missing *MissingError + if _, err := Parse(strings.NewReader(llamaMetrics), m); !errors.As(err, &missing) { + t.Fatalf("a named but absent metric was tolerated: %v", err) + } +} + +func TestRunningAndWaitingCannotBeOmitted(t *testing.T) { + // There is no engine worth routing to that reports neither how much it is + // doing nor how much is queued. + _, err := Parse(strings.NewReader(llamaMetrics), Mapping{ + Running: "llamacpp:requests_processing", Waiting: "nope:waiting", + }) + if err == nil { + t.Error("an endpoint with no queue signal was accepted") + } +} + +// logCapture collects records so a test can assert what an operator would see. +type logCapture struct { + mu sync.Mutex + records []slog.Record +} + +func (c *logCapture) Enabled(context.Context, slog.Level) bool { return true } +func (c *logCapture) WithAttrs([]slog.Attr) slog.Handler { return c } +func (c *logCapture) WithGroup(string) slog.Handler { return c } +func (c *logCapture) Handle(_ context.Context, r slog.Record) error { + c.mu.Lock() + defer c.mu.Unlock() + c.records = append(c.records, r) + return nil +} + +func (c *logCapture) count(level slog.Level, substr string) int { + c.mu.Lock() + defer c.mu.Unlock() + n := 0 + for _, r := range c.records { + if r.Level == level && strings.Contains(r.Message, substr) { + n++ + } + } + return n +} + +func TestABrokenContractIsReportedOnceAndLoudly(t *testing.T) { + // The failure this guards against is a cluster that looks healthy while + // none of its routing decisions are informed by anything. Debug-level + // logging was not enough: a contract violation fails every scrape forever, + // and at the default level nothing said so. + served := llamaMetrics + var mu sync.Mutex + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + mu.Lock() + defer mu.Unlock() + _, _ = io.WriteString(w, served) + })) + defer srv.Close() + + cap := &logCapture{} + reg := endpoint.NewRegistry(endpoint.Config{}) + in := endpoint.Info{ID: "e1", URL: srv.URL, Engine: "vllm", Models: []string{"m"}} + reg.Replace([]endpoint.Info{in}) + + p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 10 * time.Millisecond}, + slog.New(cap)) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + p.Sync(ctx) + defer p.Stop() + + // vLLM's names against llama.cpp's output: nothing matches. + deadline := time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && cap.count(slog.LevelWarn, "does not serve the metrics") == 0 { + time.Sleep(10 * time.Millisecond) + } + if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 { + t.Fatalf("the violation was reported %d times; expected exactly one", n) + } + + // It must not turn into a per-scrape stream either. + time.Sleep(200 * time.Millisecond) + if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 { + t.Errorf("the warning repeated: %d times", n) + } + + // And recovery has to be visible, or an operator cannot tell a fixed + // cluster from a broken one. + mu.Lock() + served = `# TYPE vllm:num_requests_running gauge +vllm:num_requests_running 1 +# TYPE vllm:num_requests_waiting gauge +vllm:num_requests_waiting 2 +# TYPE vllm:kv_cache_usage_perc gauge +vllm:kv_cache_usage_perc 0.5 +` + mu.Unlock() + + deadline = time.Now().Add(2 * time.Second) + for time.Now().Before(deadline) && cap.count(slog.LevelInfo, "readable again") == 0 { + time.Sleep(10 * time.Millisecond) + } + if cap.count(slog.LevelInfo, "readable again") == 0 { + t.Error("recovery was never reported") + } +} diff --git a/internal/router/scrape/parse.go b/internal/router/scrape/parse.go index ee68ac9..379bba8 100644 --- a/internal/router/scrape/parse.go +++ b/internal/router/scrape/parse.go @@ -50,15 +50,23 @@ func MappingFor(engine string) Mapping { // exposes when started with --metrics. // // "Deferred" is llama.cpp's word for a request that arrived with every slot -// busy, which is the same thing vLLM calls waiting. The fields left empty are -// metrics llama.cpp does not have: it counts decodes rather than requests, so -// there is no success counter and no cache configuration to read block sizes -// from. +// busy, which is the same thing vLLM calls waiting. +// +// KV cache usage is deliberately absent: current llama.cpp does not publish it +// (b10991 serves fifteen families and none of them is a cache ratio), and +// naming a metric that is not there would fail the contract on every scrape and +// leave the router with no numbers at all rather than two out of three. The +// loss is smaller here than it would be for vLLM — llama.cpp serves a fixed +// number of slots, so its queue depth already says when it is full — but it is +// a loss: saturation falls back to queue depth alone. +// +// The rest are metrics llama.cpp does not have either. It counts decodes rather +// than requests, so there is no success counter and no cache configuration to +// read block sizes from. func LlamaCppMapping() Mapping { return Mapping{ Running: "llamacpp:requests_processing", Waiting: "llamacpp:requests_deferred", - KVCacheUsage: "llamacpp:kv_cache_usage_ratio", PromptTokens: "llamacpp:prompt_tokens_total", GenerationTokens: "llamacpp:tokens_predicted_total", } @@ -122,8 +130,11 @@ func (e *MissingError) Error() string { // Parse converts Prometheus text-format metrics into Stats. // -// If any of the mandatory three (running, waiting, kv usage) is absent it -// returns *MissingError: routing to that endpoint would be blind. +// A metric the mapping names and the engine does not serve is a contract +// violation and returns *MissingError: routing to that endpoint would be blind. +// A metric the mapping leaves unnamed is one the engine is known not to +// publish, and stays zero — the router loses that signal for those endpoints +// rather than losing the endpoints. func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) { m.normalize() @@ -138,6 +149,9 @@ func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) { var s endpoint.Stats var missing []string + // Running and waiting have no such escape: an engine that reports neither + // how much it is doing nor how much is queued cannot be scheduled against + // at all. if v, ok := firstValue(fams, m.Running); ok { s.Running = int(v) } else { @@ -148,10 +162,12 @@ func Parse(r io.Reader, m Mapping) (endpoint.Stats, error) { } else { missing = append(missing, m.Waiting) } - if v, ok := firstValue(fams, m.KVCacheUsage); ok { - s.KVCacheUsage = v - } else { - missing = append(missing, m.KVCacheUsage) + if m.KVCacheUsage != "" { + if v, ok := firstValue(fams, m.KVCacheUsage); ok { + s.KVCacheUsage = v + } else { + missing = append(missing, m.KVCacheUsage) + } } if len(missing) > 0 { return endpoint.Stats{}, &MissingError{Missing: missing} diff --git a/internal/router/scrape/poller.go b/internal/router/scrape/poller.go index 0a3aaea..f9242da 100644 --- a/internal/router/scrape/poller.go +++ b/internal/router/scrape/poller.go @@ -2,6 +2,7 @@ package scrape import ( "context" + "errors" "fmt" "log/slog" "math/rand/v2" @@ -141,30 +142,65 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) { t := time.NewTicker(p.cfg.Interval) defer t.Stop() - p.once(ctx, in) + // One goroutine per endpoint, so this needs no lock. + var reported string + + p.once(ctx, in, &reported) for { select { case <-ctx.Done(): return case <-t.C: - p.once(ctx, in) + p.once(ctx, in, &reported) } } } -func (p *Poller) once(ctx context.Context, in endpoint.Info) { +func (p *Poller) once(ctx context.Context, in endpoint.Info, reported *string) { stats, err := p.fetch(ctx, in.URL, p.mappingFor(in)) if err != nil { if ctx.Err() != nil { return } p.reg.ScrapeFailed(in.ID) - p.log.Debug("scrape failed", "endpoint", in.ID, "err", err) + p.report(in, err, reported) return } + if *reported != "" { + p.log.Info("endpoint metrics are readable again", "endpoint", in.ID) + *reported = "" + } p.reg.Observe(in.ID, stats) } +// report logs a scrape failure the first time it appears and then stays quiet +// until the reason changes. +// +// Per-scrape debug logging was not enough. A contract violation is a +// compatibility problem, not a blip: every scrape fails, the router has no +// numbers for that endpoint and schedules to it blind, and at the default log +// level nothing says so. A cluster can look entirely healthy while none of its +// routing decisions are informed by anything. +func (p *Poller) report(in endpoint.Info, err error, reported *string) { + var missing *MissingError + if errors.As(err, &missing) { + msg := err.Error() + if *reported == msg { + return + } + *reported = msg + p.log.Warn("engine does not serve the metrics this endpoint is read with; "+ + "routing to it is uninformed", + "endpoint", in.ID, "engine", in.Engine, "missing", missing.Missing) + return + } + + // Ordinary failures — a restarting engine, a timeout — are expected and + // transient, so they stay quiet unless someone is looking. + p.log.Debug("scrape failed", "endpoint", in.ID, "err", err) + *reported = "unreachable" +} + func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) { ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout) defer cancel() diff --git a/llmgw-plan.md b/llmgw-plan.md index af0bfa6..00f3ae0 100644 --- a/llmgw-plan.md +++ b/llmgw-plan.md @@ -791,16 +791,35 @@ restarts, its index is reset. ### 7.6 The engine metric contract -This is the formal definition of "adding a new engine". If an engine exposes these three metrics +This is the formal definition of "adding a new engine". If an engine exposes these metrics in Prometheus format, llmgw can route to it (GIE's model server protocol): -| Contract field | vLLM metric | Mandatory | -|---|---|---| -| `Waiting` | `vllm:num_requests_waiting` | yes | -| `Running` | `vllm:num_requests_running` | yes | -| `KVCacheUsage` | `vllm:kv_cache_usage_perc` | yes | -| `BlockSize`, `NumGPUBlocks` | `vllm:cache_config_info` | no | -| LoRA state | `vllm:lora_requests_info` (`max_lora`, `running_lora_adapters`, `waiting_lora_adapters`) | no | +| Contract field | vLLM metric | llama.cpp metric | Mandatory | +|---|---|---|---| +| `Waiting` | `vllm:num_requests_waiting` | `llamacpp:requests_deferred` | yes | +| `Running` | `vllm:num_requests_running` | `llamacpp:requests_processing` | yes | +| `KVCacheUsage` | `vllm:kv_cache_usage_perc` | — | where the engine has it | +| `PromptTokens` | `vllm:prompt_tokens_total` | `llamacpp:prompt_tokens_total` | no | +| `GenerationTokens` | `vllm:generation_tokens_total` | `llamacpp:tokens_predicted_total` | no | +| `BlockSize`, `NumGPUBlocks` | `vllm:cache_config_info` | — | no | +| LoRA state | `vllm:lora_requests_info` (`max_lora`, `running_lora_adapters`, `waiting_lora_adapters`) | — | no | + +Mappings are **per engine, not per cluster**. A cluster running llama.cpp on its CPU nodes and +vLLM on its GPU nodes has two vocabularies for the same numbers, and one global mapping would +leave half the fleet unmeasurable. An endpoint that names no engine is read with vLLM's names. + +`Running` and `Waiting` are the two that cannot be omitted: an engine that reports neither how +much it is doing nor how much is queued cannot be scheduled against at all. `KVCacheUsage` is +required *of engines that publish it* — a mapping that names it and does not get it is a contract +violation, while a mapping that leaves it unnamed is a statement that this engine has no such +metric. Current llama.cpp is the second case, and saturation there falls back to queue depth +alone. That is a real loss of signal, mitigated by llama.cpp serving a fixed number of slots, so +its queue depth already says when it is full. + +> Correction: an earlier version of this plan gave llama.cpp's KV metric as +> `llamacpp:kv_cache_usage_ratio`. Build b10991 does not publish it — it serves fifteen metric +> families and none is a cache ratio. Naming it failed the contract on every scrape and left the +> router with no numbers at all rather than two out of three. > Correction: v0.1 said `vllm:gpu_cache_usage_perc`; the correct name is > `vllm:kv_cache_usage_perc`. Metric names drift between versions, so the `ParseMetrics` mapping @@ -811,6 +830,10 @@ Scrape rules: - Every Ready endpoint every 1–2s, one goroutine per endpoint, with jitter. - A failed scrape marks the endpoint stale; its score drops but it is not removed. - Three consecutive failures trip the breaker (§7.7). +- A **contract violation is reported at warn level, once**, and again when it clears. It is a + compatibility problem rather than a blip: every scrape fails, the router schedules to that + endpoint blind, and per-scrape debug logging meant a cluster could look entirely healthy while + none of its routing decisions were informed by anything. - **So that decisions are never made purely on stale data**, the `inflight` scorer (§7.4) stays enabled at all times.