Deployed against real llama.cpp, every scrape failed. The mapping named llamacpp:kv_cache_usage_ratio; build b10991 serves fifteen metric families and none is a cache ratio. Because KV usage was mandatory, that one absent name threw away the two metrics the engine does publish, and the router scheduled to all three replicas blind while the cluster looked healthy. A mapping that leaves a metric unnamed now means the engine does not publish it, and the field stays zero. Naming one the engine does not serve is still a violation, so this does not become a way to tolerate a typo. Running and waiting can never be omitted: an engine that reports neither what it is doing nor what is queued cannot be scheduled against at all. For llama.cpp this costs the KV half of the saturation signal, which is a real loss softened by its fixed slot count — queue depth already says when it is full. The second half is why it took a deployment to notice. A contract violation was logged per scrape at debug level, so at the default level nothing said anything. It is now a warning, once, with a matching line when it clears. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
220 lines
7.3 KiB
Go
220 lines
7.3 KiB
Go
package scrape
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"io"
|
|
"log/slog"
|
|
"net/http"
|
|
"net/http/httptest"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
|
|
"gitea.mynodes.xyz/ilker/llmgw/internal/router/endpoint"
|
|
)
|
|
|
|
// llamaMetrics is what llama-server --metrics actually serves, trimmed to the
|
|
// families that matter.
|
|
const llamaMetrics = `# HELP llamacpp:prompt_tokens_total Number of prompt tokens processed.
|
|
# TYPE llamacpp:prompt_tokens_total counter
|
|
llamacpp:prompt_tokens_total 1200
|
|
# HELP llamacpp:tokens_predicted_total Number of generation tokens processed.
|
|
# TYPE llamacpp:tokens_predicted_total counter
|
|
llamacpp:tokens_predicted_total 3400
|
|
# HELP llamacpp:requests_processing Number of requests processing.
|
|
# TYPE llamacpp:requests_processing gauge
|
|
llamacpp:requests_processing 3
|
|
# HELP llamacpp:requests_deferred Number of requests deferred.
|
|
# TYPE llamacpp:requests_deferred gauge
|
|
llamacpp:requests_deferred 7
|
|
`
|
|
|
|
func TestLlamaCppMetricsSatisfyTheContract(t *testing.T) {
|
|
s, err := Parse(strings.NewReader(llamaMetrics), LlamaCppMapping())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if s.Running != 3 {
|
|
t.Errorf("running=%d", s.Running)
|
|
}
|
|
// "Deferred" is llama.cpp's word for a request that arrived with every slot
|
|
// busy, which is what the scheduler needs to read as queue depth.
|
|
if s.Waiting != 7 {
|
|
t.Errorf("waiting=%d", s.Waiting)
|
|
}
|
|
// Current llama.cpp publishes no cache ratio at all, so this stays zero
|
|
// rather than failing the whole scrape.
|
|
if s.KVCacheUsage != 0 {
|
|
t.Errorf("kv=%v", s.KVCacheUsage)
|
|
}
|
|
if s.PromptTokens != 1200 || s.GenerationTokens != 3400 {
|
|
t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppMappingDoesNotInheritVLLMNames(t *testing.T) {
|
|
// llama.cpp has no request-success counter. Filling that field with vLLM's
|
|
// name would send the router hunting for a metric that cannot be there, and
|
|
// would hide the fact that this engine simply does not report it.
|
|
m := LlamaCppMapping()
|
|
m.normalize()
|
|
if m.SuccessTotal != "" || m.CacheConfig != "" || m.KVCacheUsage != "" {
|
|
t.Errorf("vLLM names leaked into the llama.cpp mapping: %+v", m)
|
|
}
|
|
if m.Running != "llamacpp:requests_processing" {
|
|
t.Errorf("running=%q", m.Running)
|
|
}
|
|
}
|
|
|
|
func TestEmptyMappingStillDefaultsToVLLM(t *testing.T) {
|
|
var m Mapping
|
|
m.normalize()
|
|
if m.Running != "vllm:num_requests_running" || m.SuccessTotal == "" {
|
|
t.Errorf("an unconfigured mapping should be vLLM's: %+v", m)
|
|
}
|
|
}
|
|
|
|
func TestVLLMNamesAreNotReadAsLlamaCpp(t *testing.T) {
|
|
// The failure this guards against is silent: reading a vLLM endpoint with
|
|
// llama.cpp's vocabulary finds nothing, and an endpoint with no numbers
|
|
// must be reported as unmeasurable rather than as idle.
|
|
var missing *MissingError
|
|
_, err := Parse(strings.NewReader(llamaMetrics), VLLMMapping())
|
|
if !errors.As(err, &missing) {
|
|
t.Fatalf("expected the contract to be reported incomplete, got %v", err)
|
|
}
|
|
if len(missing.Missing) != 3 {
|
|
t.Errorf("every metric vLLM's mapping expects should be named: %v", missing.Missing)
|
|
}
|
|
}
|
|
|
|
func TestMappingForPicksByEngine(t *testing.T) {
|
|
if MappingFor("llamacpp").Running != "llamacpp:requests_processing" {
|
|
t.Error("llamacpp did not get its own mapping")
|
|
}
|
|
// An endpoint that names no engine, or one this build has never heard of,
|
|
// is read with vLLM's names rather than left unreadable.
|
|
for _, engine := range []string{"", "vllm", "something-new"} {
|
|
if MappingFor(engine).Running != "vllm:num_requests_running" {
|
|
t.Errorf("engine %q did not fall back to vLLM", engine)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestAMetricTheMappingNamesIsStillMandatory(t *testing.T) {
|
|
// Leaving a name out says "this engine does not publish it". Naming one the
|
|
// engine does not serve is a contract violation, and must stay one: the
|
|
// alternative is an endpoint quietly scheduled against numbers that are
|
|
// zero because nothing ever set them.
|
|
m := LlamaCppMapping()
|
|
m.KVCacheUsage = "llamacpp:kv_cache_usage_ratio"
|
|
|
|
var missing *MissingError
|
|
if _, err := Parse(strings.NewReader(llamaMetrics), m); !errors.As(err, &missing) {
|
|
t.Fatalf("a named but absent metric was tolerated: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestRunningAndWaitingCannotBeOmitted(t *testing.T) {
|
|
// There is no engine worth routing to that reports neither how much it is
|
|
// doing nor how much is queued.
|
|
_, err := Parse(strings.NewReader(llamaMetrics), Mapping{
|
|
Running: "llamacpp:requests_processing", Waiting: "nope:waiting",
|
|
})
|
|
if err == nil {
|
|
t.Error("an endpoint with no queue signal was accepted")
|
|
}
|
|
}
|
|
|
|
// logCapture collects records so a test can assert what an operator would see.
|
|
type logCapture struct {
|
|
mu sync.Mutex
|
|
records []slog.Record
|
|
}
|
|
|
|
func (c *logCapture) Enabled(context.Context, slog.Level) bool { return true }
|
|
func (c *logCapture) WithAttrs([]slog.Attr) slog.Handler { return c }
|
|
func (c *logCapture) WithGroup(string) slog.Handler { return c }
|
|
func (c *logCapture) Handle(_ context.Context, r slog.Record) error {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
c.records = append(c.records, r)
|
|
return nil
|
|
}
|
|
|
|
func (c *logCapture) count(level slog.Level, substr string) int {
|
|
c.mu.Lock()
|
|
defer c.mu.Unlock()
|
|
n := 0
|
|
for _, r := range c.records {
|
|
if r.Level == level && strings.Contains(r.Message, substr) {
|
|
n++
|
|
}
|
|
}
|
|
return n
|
|
}
|
|
|
|
func TestABrokenContractIsReportedOnceAndLoudly(t *testing.T) {
|
|
// The failure this guards against is a cluster that looks healthy while
|
|
// none of its routing decisions are informed by anything. Debug-level
|
|
// logging was not enough: a contract violation fails every scrape forever,
|
|
// and at the default level nothing said so.
|
|
served := llamaMetrics
|
|
var mu sync.Mutex
|
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
|
mu.Lock()
|
|
defer mu.Unlock()
|
|
_, _ = io.WriteString(w, served)
|
|
}))
|
|
defer srv.Close()
|
|
|
|
cap := &logCapture{}
|
|
reg := endpoint.NewRegistry(endpoint.Config{})
|
|
in := endpoint.Info{ID: "e1", URL: srv.URL, Engine: "vllm", Models: []string{"m"}}
|
|
reg.Replace([]endpoint.Info{in})
|
|
|
|
p := NewPoller(reg, Config{Interval: 20 * time.Millisecond, Timeout: 10 * time.Millisecond},
|
|
slog.New(cap))
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
defer cancel()
|
|
p.Sync(ctx)
|
|
defer p.Stop()
|
|
|
|
// vLLM's names against llama.cpp's output: nothing matches.
|
|
deadline := time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) && cap.count(slog.LevelWarn, "does not serve the metrics") == 0 {
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 {
|
|
t.Fatalf("the violation was reported %d times; expected exactly one", n)
|
|
}
|
|
|
|
// It must not turn into a per-scrape stream either.
|
|
time.Sleep(200 * time.Millisecond)
|
|
if n := cap.count(slog.LevelWarn, "does not serve the metrics"); n != 1 {
|
|
t.Errorf("the warning repeated: %d times", n)
|
|
}
|
|
|
|
// And recovery has to be visible, or an operator cannot tell a fixed
|
|
// cluster from a broken one.
|
|
mu.Lock()
|
|
served = `# TYPE vllm:num_requests_running gauge
|
|
vllm:num_requests_running 1
|
|
# TYPE vllm:num_requests_waiting gauge
|
|
vllm:num_requests_waiting 2
|
|
# TYPE vllm:kv_cache_usage_perc gauge
|
|
vllm:kv_cache_usage_perc 0.5
|
|
`
|
|
mu.Unlock()
|
|
|
|
deadline = time.Now().Add(2 * time.Second)
|
|
for time.Now().Before(deadline) && cap.count(slog.LevelInfo, "readable again") == 0 {
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
if cap.count(slog.LevelInfo, "readable again") == 0 {
|
|
t.Error("recovery was never reported")
|
|
}
|
|
}
|