Ilker ManapandClaude Opus 5
501db3d3e8
llama.cpp is what makes a cluster with no cards useful: it serves the same OpenAI API on CPU, so the whole path — placement, supervision, scraping, scheduling, admission — can be exercised end to end on ordinary machines. Three things had to give way for it. A profile may now omit its GPU class, and that is how it says it runs on the CPU. Only engines that can actually serve without a card may omit it; for vLLM an omitted class stays an error, because otherwise a typo becomes a replica placed on a machine with no hardware and diagnosed minutes later during model loading. Placement spreads CPU replicas instead of bin packing them, since unlike GPUs there is no allocation to stop two of them sharing a machine's cores. Metric mappings are now per engine rather than per cluster. llama.cpp calls the same three numbers requests_processing, requests_deferred and kv_cache_usage_ratio, and a single global mapping would leave half a mixed fleet unmeasurable. An empty mapping still defaults to vLLM's names; one that names its own Running metric is taken as a complete statement, so the fields it leaves blank stay blank rather than being filled with names that engine cannot have. The adapter refuses multi-node and tensor-parallel profiles. llama.cpp splits across local GPUs by a different mechanism than this adapter drives, and accepting either would produce a replica that starts and then serves from one machine while the catalog believes it has several. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
78 lines
2.2 KiB
Go
78 lines
2.2 KiB
Go
package catalog
|
|
|
|
import (
|
|
"strings"
|
|
"testing"
|
|
)
|
|
|
|
func TestCPUProfileMayOmitGPUClass(t *testing.T) {
|
|
c, err := New([]Model{{
|
|
Name: "qwen-0.5b",
|
|
Source: "/models/qwen.gguf",
|
|
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp"}},
|
|
}})
|
|
if err != nil {
|
|
t.Fatalf("a llama.cpp profile with no GPU class was rejected: %v", err)
|
|
}
|
|
|
|
m, _ := c.Resolve("qwen-0.5b")
|
|
p := m.Profiles[0]
|
|
if !p.CPUOnly() {
|
|
t.Error("the profile was not recognized as CPU-only")
|
|
}
|
|
if p.GPUsPerNode != 0 {
|
|
t.Errorf("a CPU profile asks for %d GPUs per node", p.GPUsPerNode)
|
|
}
|
|
if pp := m.PlacementProfiles()[0]; !pp.CPUOnly() || pp.GPUsPerNode != 0 {
|
|
t.Errorf("placement sees %+v", pp)
|
|
}
|
|
}
|
|
|
|
func TestGPUEngineStillRequiresItsClass(t *testing.T) {
|
|
// Omitting the class for vLLM is a mistake, and one that would otherwise
|
|
// surface as a replica placed on a machine with no cards, minutes later.
|
|
_, err := New([]Model{{
|
|
Name: "llama-70b",
|
|
Source: "meta/llama",
|
|
Profiles: []Profile{{Name: "tp4", Engine: "vllm", TP: 4}},
|
|
}})
|
|
if err == nil || !strings.Contains(err.Error(), "gpu_class is required") {
|
|
t.Errorf("expected the class to be required for vllm: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCPUProfileCannotSpanNodes(t *testing.T) {
|
|
// llama.cpp has no equivalent of a replica spanning machines, so accepting
|
|
// this would place shards that never form one replica.
|
|
_, err := New([]Model{{
|
|
Name: "qwen-0.5b",
|
|
Source: "/models/qwen.gguf",
|
|
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp", Nodes: 2}},
|
|
}})
|
|
if err == nil || !strings.Contains(err.Error(), "cannot span") {
|
|
t.Errorf("expected a multi-node CPU profile to be refused: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestCPUProfileReachesTheEngineWithoutGPUs(t *testing.T) {
|
|
c, err := New([]Model{{
|
|
Name: "qwen-0.5b",
|
|
Source: "/models/qwen.gguf",
|
|
Profiles: []Profile{{Name: "cpu", Engine: "llamacpp", Args: []string{"--ctx-size", "4096"}}},
|
|
}})
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m, _ := c.Resolve("qwen-0.5b")
|
|
ep, ok := m.EngineProfile("cpu")
|
|
if !ok {
|
|
t.Fatal("the engine profile is missing")
|
|
}
|
|
if ep.Engine != "llamacpp" || ep.ModelPath != "/models/qwen.gguf" {
|
|
t.Errorf("engine profile %+v", ep)
|
|
}
|
|
if ep.ServedModelName != "qwen-0.5b" {
|
|
t.Errorf("served as %q", ep.ServedModelName)
|
|
}
|
|
}
|