ilkerandClaude Opus 5
b7417e1a69
The code behind plan §7.3-7.6: - breaker: folds cooldown and circuit breaking into one component (FailureThreshold=1 gives LiteLLM-style cooldown), with exponential backoff and half-open probes. - endpoint: live state registry. Inflight counting, staleness detection, drain rate derived from counters. A scrape failure does not trip the breaker — the engine may be serving fine while only its metrics endpoint is down. - scrape: parses the §7.6 metric contract. Errors when any of the mandatory three is missing; the metric name mapping is overridable, since names drift between versions (plan §14). - scheduler: the filter -> score -> pick pipeline. A "strategy" is now just a set of scorer weights; chat, batch and long-context are the same code with different weights. Scorers: queue, kv (hard penalty above the ceiling), inflight, load, weight, session (bounded-load consistent hashing). Pickers: max-score, p2c, softmax, round-robin. Decision carries the rationale — which scorer produced what, which filter dropped how many. Tuning the weights without that is guesswork. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
116 lines
3.4 KiB
Go
116 lines
3.4 KiB
Go
package scrape
|
|
|
|
import (
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
const vllmSample = `
|
|
# HELP vllm:num_requests_running Number of requests currently running on GPU.
|
|
# TYPE vllm:num_requests_running gauge
|
|
vllm:num_requests_running{model_name="llama"} 7
|
|
# TYPE vllm:num_requests_waiting gauge
|
|
vllm:num_requests_waiting{model_name="llama"} 3
|
|
# TYPE vllm:kv_cache_usage_perc gauge
|
|
vllm:kv_cache_usage_perc{model_name="llama"} 0.42
|
|
# TYPE vllm:cache_config_info gauge
|
|
vllm:cache_config_info{block_size="16",num_gpu_blocks="4096"} 1
|
|
# TYPE vllm:generation_tokens_total counter
|
|
vllm:generation_tokens_total{model_name="llama"} 12345
|
|
# TYPE vllm:request_success_total counter
|
|
vllm:request_success_total{model_name="llama",finished_reason="length"} 100
|
|
vllm:request_success_total{model_name="llama",finished_reason="stop"} 50
|
|
`
|
|
|
|
func TestParseContractFields(t *testing.T) {
|
|
s, err := Parse(strings.NewReader(vllmSample), VLLMMapping())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if s.Running != 7 {
|
|
t.Errorf("Running=%d", s.Running)
|
|
}
|
|
if s.Waiting != 3 {
|
|
t.Errorf("Waiting=%d", s.Waiting)
|
|
}
|
|
if s.KVCacheUsage != 0.42 {
|
|
t.Errorf("KVCacheUsage=%v", s.KVCacheUsage)
|
|
}
|
|
if s.BlockSize != 16 || s.NumGPUBlocks != 4096 {
|
|
t.Errorf("cache_config_info not read: block=%d blocks=%d", s.BlockSize, s.NumGPUBlocks)
|
|
}
|
|
// A counter split across labels must be summed.
|
|
if s.SuccessTotal != 150 {
|
|
t.Errorf("SuccessTotal=%v, expected 150 (labels must be summed)", s.SuccessTotal)
|
|
}
|
|
if s.GenerationTokens != 12345 {
|
|
t.Errorf("GenerationTokens=%v", s.GenerationTokens)
|
|
}
|
|
}
|
|
|
|
func TestParseRejectsMissingMandatoryMetric(t *testing.T) {
|
|
// kv_cache_usage is absent: routing would be blind, so this must error.
|
|
input := `
|
|
# TYPE vllm:num_requests_running gauge
|
|
vllm:num_requests_running 1
|
|
# TYPE vllm:num_requests_waiting gauge
|
|
vllm:num_requests_waiting 0
|
|
`
|
|
_, err := Parse(strings.NewReader(input), VLLMMapping())
|
|
var me *MissingError
|
|
if !errors.As(err, &me) {
|
|
t.Fatalf("expected MissingError, got %v", err)
|
|
}
|
|
if len(me.Missing) != 1 || me.Missing[0] != "vllm:kv_cache_usage_perc" {
|
|
t.Errorf("missing list is wrong: %v", me.Missing)
|
|
}
|
|
}
|
|
|
|
func TestParseToleratesMissingOptionals(t *testing.T) {
|
|
input := `
|
|
# TYPE vllm:num_requests_running gauge
|
|
vllm:num_requests_running 2
|
|
# TYPE vllm:num_requests_waiting gauge
|
|
vllm:num_requests_waiting 1
|
|
# TYPE vllm:kv_cache_usage_perc gauge
|
|
vllm:kv_cache_usage_perc 0.1
|
|
`
|
|
s, err := Parse(strings.NewReader(input), VLLMMapping())
|
|
if err != nil {
|
|
t.Fatalf("errored although only optional metrics were missing: %v", err)
|
|
}
|
|
if s.BlockSize != 0 || s.SuccessTotal != 0 {
|
|
t.Error("missing optional fields should have stayed zero")
|
|
}
|
|
}
|
|
|
|
func TestParseWithCustomMapping(t *testing.T) {
|
|
// A different engine with different names (plan §14: names drift across versions).
|
|
input := `
|
|
# TYPE sglang:running gauge
|
|
sglang:running 4
|
|
# TYPE sglang:queued gauge
|
|
sglang:queued 2
|
|
# TYPE sglang:kv_usage gauge
|
|
sglang:kv_usage 0.8
|
|
`
|
|
s, err := Parse(strings.NewReader(input), Mapping{
|
|
Running: "sglang:running",
|
|
Waiting: "sglang:queued",
|
|
KVCacheUsage: "sglang:kv_usage",
|
|
})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if s.Running != 4 || s.Waiting != 2 || s.KVCacheUsage != 0.8 {
|
|
t.Errorf("custom mapping did not work: %+v", s)
|
|
}
|
|
}
|
|
|
|
func TestParseRejectsGarbage(t *testing.T) {
|
|
if _, err := Parse(strings.NewReader("this is not metrics {{{"), VLLMMapping()); err == nil {
|
|
t.Error("expected an error for malformed input")
|
|
}
|
|
}
|