llama.cpp is what makes a cluster with no cards useful: it serves the same OpenAI API on CPU, so the whole path — placement, supervision, scraping, scheduling, admission — can be exercised end to end on ordinary machines. Three things had to give way for it. A profile may now omit its GPU class, and that is how it says it runs on the CPU. Only engines that can actually serve without a card may omit it; for vLLM an omitted class stays an error, because otherwise a typo becomes a replica placed on a machine with no hardware and diagnosed minutes later during model loading. Placement spreads CPU replicas instead of bin packing them, since unlike GPUs there is no allocation to stop two of them sharing a machine's cores. Metric mappings are now per engine rather than per cluster. llama.cpp calls the same three numbers requests_processing, requests_deferred and kv_cache_usage_ratio, and a single global mapping would leave half a mixed fleet unmeasurable. An empty mapping still defaults to vLLM's names; one that names its own Running metric is taken as a complete statement, so the fields it leaves blank stay blank rather than being filled with names that engine cannot have. The adapter refuses multi-node and tensor-parallel profiles. llama.cpp splits across local GPUs by a different mechanism than this adapter drives, and accepting either would produce a replica that starts and then serves from one machine while the catalog believes it has several. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
97 lines
3.3 KiB
Go
97 lines
3.3 KiB
Go
package scrape
|
|
|
|
import (
|
|
"errors"
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
// llamaMetrics is what llama-server --metrics actually serves, trimmed to the
|
|
// families that matter.
|
|
const llamaMetrics = `# HELP llamacpp:prompt_tokens_total Number of prompt tokens processed.
|
|
# TYPE llamacpp:prompt_tokens_total counter
|
|
llamacpp:prompt_tokens_total 1200
|
|
# HELP llamacpp:tokens_predicted_total Number of generation tokens processed.
|
|
# TYPE llamacpp:tokens_predicted_total counter
|
|
llamacpp:tokens_predicted_total 3400
|
|
# HELP llamacpp:kv_cache_usage_ratio KV-cache usage.
|
|
# TYPE llamacpp:kv_cache_usage_ratio gauge
|
|
llamacpp:kv_cache_usage_ratio 0.42
|
|
# HELP llamacpp:requests_processing Number of requests processing.
|
|
# TYPE llamacpp:requests_processing gauge
|
|
llamacpp:requests_processing 3
|
|
# HELP llamacpp:requests_deferred Number of requests deferred.
|
|
# TYPE llamacpp:requests_deferred gauge
|
|
llamacpp:requests_deferred 7
|
|
`
|
|
|
|
func TestLlamaCppMetricsSatisfyTheContract(t *testing.T) {
|
|
s, err := Parse(strings.NewReader(llamaMetrics), LlamaCppMapping())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if s.Running != 3 {
|
|
t.Errorf("running=%d", s.Running)
|
|
}
|
|
// "Deferred" is llama.cpp's word for a request that arrived with every slot
|
|
// busy, which is what the scheduler needs to read as queue depth.
|
|
if s.Waiting != 7 {
|
|
t.Errorf("waiting=%d", s.Waiting)
|
|
}
|
|
if s.KVCacheUsage != 0.42 {
|
|
t.Errorf("kv=%v", s.KVCacheUsage)
|
|
}
|
|
if s.PromptTokens != 1200 || s.GenerationTokens != 3400 {
|
|
t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens)
|
|
}
|
|
}
|
|
|
|
func TestLlamaCppMappingDoesNotInheritVLLMNames(t *testing.T) {
|
|
// llama.cpp has no request-success counter. Filling that field with vLLM's
|
|
// name would send the router hunting for a metric that cannot be there, and
|
|
// would hide the fact that this engine simply does not report it.
|
|
m := LlamaCppMapping()
|
|
m.normalize()
|
|
if m.SuccessTotal != "" || m.CacheConfig != "" {
|
|
t.Errorf("vLLM names leaked into the llama.cpp mapping: %+v", m)
|
|
}
|
|
if m.Running != "llamacpp:requests_processing" {
|
|
t.Errorf("running=%q", m.Running)
|
|
}
|
|
}
|
|
|
|
func TestEmptyMappingStillDefaultsToVLLM(t *testing.T) {
|
|
var m Mapping
|
|
m.normalize()
|
|
if m.Running != "vllm:num_requests_running" || m.SuccessTotal == "" {
|
|
t.Errorf("an unconfigured mapping should be vLLM's: %+v", m)
|
|
}
|
|
}
|
|
|
|
func TestVLLMNamesAreNotReadAsLlamaCpp(t *testing.T) {
|
|
// The failure this guards against is silent: reading a vLLM endpoint with
|
|
// llama.cpp's vocabulary finds nothing, and an endpoint with no numbers
|
|
// must be reported as unmeasurable rather than as idle.
|
|
var missing *MissingError
|
|
_, err := Parse(strings.NewReader(llamaMetrics), VLLMMapping())
|
|
if !errors.As(err, &missing) {
|
|
t.Fatalf("expected the contract to be reported incomplete, got %v", err)
|
|
}
|
|
if len(missing.Missing) != 3 {
|
|
t.Errorf("all three mandatory metrics should be named: %v", missing.Missing)
|
|
}
|
|
}
|
|
|
|
func TestMappingForPicksByEngine(t *testing.T) {
|
|
if MappingFor("llamacpp").Running != "llamacpp:requests_processing" {
|
|
t.Error("llamacpp did not get its own mapping")
|
|
}
|
|
// An endpoint that names no engine, or one this build has never heard of,
|
|
// is read with vLLM's names rather than left unreadable.
|
|
for _, engine := range []string{"", "vllm", "something-new"} {
|
|
if MappingFor(engine).Running != "vllm:num_requests_running" {
|
|
t.Errorf("engine %q did not fall back to vLLM", engine)
|
|
}
|
|
}
|
|
}
|