diff --git a/internal/agent/engines/engine.go b/internal/agent/engines/engine.go index c20cb4f..cdc3929 100644 --- a/internal/agent/engines/engine.go +++ b/internal/agent/engines/engine.go @@ -141,6 +141,7 @@ func NewRegistry(cfg Config) *Registry { r := &Registry{engines: map[string]Engine{}} r.Add(&VLLM{cfg: cfg.VLLM}) r.Add(&SGLang{cfg: cfg.SGLang}) + r.Add(&LlamaCpp{cfg: cfg.LlamaCpp}) return r } @@ -168,8 +169,9 @@ func (r *Registry) Names() []string { // Config configures the engine adapters. type Config struct { - VLLM VLLMConfig `koanf:"vllm"` - SGLang SGLangConfig `koanf:"sglang"` + VLLM VLLMConfig `koanf:"vllm"` + SGLang SGLangConfig `koanf:"sglang"` + LlamaCpp LlamaCppConfig `koanf:"llamacpp"` } // --- shared helpers --- diff --git a/internal/agent/engines/engines_test.go b/internal/agent/engines/engines_test.go index ede59c8..f3cd948 100644 --- a/internal/agent/engines/engines_test.go +++ b/internal/agent/engines/engines_test.go @@ -397,14 +397,14 @@ func TestValidateRequiresModelPath(t *testing.T) { func TestRegistry(t *testing.T) { r := NewRegistry(Config{}) - if got := r.Names(); len(got) != 2 || got[0] != "sglang" || got[1] != "vllm" { + if got := r.Names(); !slices.Equal(got, []string{"llamacpp", "sglang", "vllm"}) { t.Errorf("registry contents: %v", got) } if _, err := r.Get("vllm"); err != nil { t.Errorf("vllm missing: %v", err) } err := func() error { _, err := r.Get("nope"); return err }() - if err == nil || !strings.Contains(err.Error(), "sglang, vllm") { + if err == nil || !strings.Contains(err.Error(), "llamacpp, sglang, vllm") { t.Errorf("an unknown engine should list what is available: %v", err) } } diff --git a/internal/agent/engines/llamacpp.go b/internal/agent/engines/llamacpp.go new file mode 100644 index 0000000..4fecb09 --- /dev/null +++ b/internal/agent/engines/llamacpp.go @@ -0,0 +1,117 @@ +package engines + +import ( + "fmt" + "strconv" + "strings" +) + +// LlamaCppConfig locates the llama.cpp installation. +type LlamaCppConfig struct { + // Bin is the llama-server binary. It is the server, not the CLI: the + // gateway talks OpenAI over HTTP and needs the one that serves it. + Bin string `koanf:"bin"` + // Parallel is how many requests a server handles at once, when a profile + // does not say. llama.cpp defaults to one, which would make every second + // request wait behind the first no matter what the router decided. + Parallel int `koanf:"parallel"` +} + +// LlamaCpp builds llama.cpp launch commands. +// +// It is the engine that makes a cluster without GPUs useful: llama.cpp runs a +// small model on CPU at a speed that is fine for testing the whole path, and it +// serves the same OpenAI API as the others, so nothing downstream can tell the +// difference. +type LlamaCpp struct{ cfg LlamaCppConfig } + +// Name identifies the engine in the catalog. +func (*LlamaCpp) Name() string { return "llamacpp" } + +// HealthPath is llama.cpp's readiness endpoint. It returns 503 while the model +// is still loading, which is exactly what the supervisor needs. +func (*LlamaCpp) HealthPath() string { return "/health" } + +// MetricsPath is llama.cpp's Prometheus endpoint, which only exists when the +// server was started with --metrics. Commands always passes it. +func (*LlamaCpp) MetricsPath() string { return "/metrics" } + +// ServesHTTP reports whether this rank answers the API. Every llama.cpp shard +// does, because a llama.cpp replica is always exactly one shard. +func (*LlamaCpp) ServesHTTP(ShardSpec) bool { return true } + +// Validate checks the profile against the GPUs it would run on. +// +// The two refusals here are about not pretending: llama.cpp has no equivalent +// of a tensor-parallel group spanning machines, and its multi-GPU support is a +// different mechanism (--split-mode, --tensor-split) that this adapter does not +// drive. Accepting either would produce a replica that starts and then serves +// from one machine while the catalog believes it has several. +func (*LlamaCpp) Validate(p Profile, gpus []GPUInfo) error { + if err := validateCommon(p, gpus); err != nil { + return err + } + if p.Nodes > 1 { + return fmt.Errorf("profile %q: llamacpp does not span nodes; use vllm or sglang for a %d-node replica", + p.Name, p.Nodes) + } + if p.TP > 1 { + return fmt.Errorf("profile %q: llamacpp does not do tensor parallelism; "+ + "it splits a model across local GPUs with --tensor-split, which this adapter does not drive", + p.Name) + } + return nil +} + +// Commands returns the processes for one shard, which is always one process. +func (l *LlamaCpp) Commands(s ShardSpec) ([]Command, error) { + bin := l.cfg.Bin + if bin == "" { + bin = "llama-server" + } + p := s.Profile + + args := []string{ + "--model", p.ModelPath, + "--host", "0.0.0.0", + "--port", strconv.Itoa(s.HTTPPort), + // Without this there is no /metrics, and an endpoint the router cannot + // measure is one it has to route to blind. + "--metrics", + } + if p.ServedModelName != "" { + args = append(args, "--alias", p.ServedModelName) + } + if !hasFlag(p.Args, "--parallel", "-np") { + n := l.cfg.Parallel + if n <= 0 { + n = 4 + } + args = append(args, "--parallel", strconv.Itoa(n)) + } + // With GPUs assigned, offload everything; llama.cpp keeps the model on the + // CPU otherwise, which is not what a GPU placement asked for. Without them + // the flag is omitted entirely rather than set to zero, so a build with no + // GPU support does not see an argument it rejects. + if len(s.GPUIndices) > 0 && !hasFlag(p.Args, "--n-gpu-layers", "-ngl") { + args = append(args, "--n-gpu-layers", "999") + } + args = append(args, p.Args...) + + return []Command{{ + Name: "llama-server", Path: bin, Args: args, Env: materializeEnv(baseEnv(s)), + }}, nil +} + +// hasFlag reports whether the profile already sets one of these flags, in +// either "--flag value" or "--flag=value" form. +func hasFlag(args []string, names ...string) bool { + for _, a := range args { + for _, n := range names { + if a == n || strings.HasPrefix(a, n+"=") { + return true + } + } + } + return false +} diff --git a/internal/agent/engines/llamacpp_test.go b/internal/agent/engines/llamacpp_test.go new file mode 100644 index 0000000..3b17ab0 --- /dev/null +++ b/internal/agent/engines/llamacpp_test.go @@ -0,0 +1,156 @@ +package engines + +import ( + "slices" + "strings" + "testing" +) + +func llamaSpec(gpus []int) ShardSpec { + return ShardSpec{ + ReplicaID: "rep-1", ShardID: "sh-1", Rank: 0, NNodes: 1, + GPUIndices: gpus, HTTPPort: 8000, MgmtInterface: "eth0", + Profile: Profile{ + Name: "cpu", Engine: "llamacpp", TP: 1, + ModelPath: "/models/qwen.gguf", ServedModelName: "qwen-0.5b", + }, + } +} + +func TestLlamaCppCommand(t *testing.T) { + cmds, err := (&LlamaCpp{}).Commands(llamaSpec(nil)) + if err != nil { + t.Fatal(err) + } + if len(cmds) != 1 { + t.Fatalf("a llama.cpp shard is one process, got %d", len(cmds)) + } + c := cmds[0] + if c.Path != "llama-server" { + t.Errorf("path %q", c.Path) + } + + // --metrics is not optional: llama-server serves no /metrics without it, and + // an endpoint the router cannot measure is one it routes to blind. + if !slices.Contains(c.Args, "--metrics") { + t.Error("the server was started without --metrics") + } + if v, _ := flagValue(c.Args, "--model"); v != "/models/qwen.gguf" { + t.Errorf("--model %q", v) + } + if v, _ := flagValue(c.Args, "--port"); v != "8000" { + t.Errorf("--port %q", v) + } + if v, _ := flagValue(c.Args, "--alias"); v != "qwen-0.5b" { + t.Errorf("--alias %q", v) + } + // llama.cpp serves one request at a time unless told otherwise, which would + // make every second request wait no matter what the router decided. + if v, ok := flagValue(c.Args, "--parallel"); !ok || v == "1" { + t.Errorf("--parallel %q", v) + } + if slices.Contains(c.Args, "--n-gpu-layers") { + t.Error("a CPU placement was given GPU layers") + } +} + +func TestLlamaCppOffloadsWhenGivenGPUs(t *testing.T) { + cmds, _ := (&LlamaCpp{}).Commands(llamaSpec([]int{2})) + if v, ok := flagValue(cmds[0].Args, "--n-gpu-layers"); !ok || v != "999" { + t.Errorf("a GPU placement should offload the model, got %q", v) + } + // The shard must still be pinned to its own card. + if !slices.Contains(cmds[0].Env, "CUDA_VISIBLE_DEVICES=2") { + t.Error("the shard was not pinned to its GPU") + } +} + +func TestProfileArgumentsWinOverDefaults(t *testing.T) { + // The escape hatch has to actually work, or an operator tuning slot count + // ends up with two --parallel flags and whichever llama.cpp happens to + // prefer. + s := llamaSpec(nil) + s.Profile.Args = []string{"--parallel", "16", "--ctx-size", "4096"} + + cmds, _ := (&LlamaCpp{}).Commands(s) + n := 0 + for _, a := range cmds[0].Args { + if a == "--parallel" { + n++ + } + } + if n != 1 { + t.Errorf("--parallel appears %d times", n) + } + if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "16" { + t.Errorf("the profile's value lost: %q", v) + } +} + +func TestShortFormArgumentsAreRecognized(t *testing.T) { + s := llamaSpec([]int{0}) + s.Profile.Args = []string{"-np", "8", "-ngl", "20"} + + cmds, _ := (&LlamaCpp{}).Commands(s) + for _, flag := range []string{"--parallel", "--n-gpu-layers"} { + if slices.Contains(cmds[0].Args, flag) { + t.Errorf("%s was added although the profile set its short form", flag) + } + } +} + +func TestLlamaCppRefusesWhatItCannotDo(t *testing.T) { + // Accepting either of these would produce a replica that starts and then + // serves from one machine while the catalog believes it has several. + multi := Profile{Name: "multi", Engine: "llamacpp", TP: 1, Nodes: 2, GPUsPerNode: 1, ModelPath: "/m"} + if err := (&LlamaCpp{}).Validate(multi, nil); err == nil { + t.Error("a multi-node llama.cpp profile was accepted") + } + + tp := Profile{Name: "tp", Engine: "llamacpp", TP: 2, GPUsPerNode: 2, ModelPath: "/m"} + if err := (&LlamaCpp{}).Validate(tp, h100(2)); err == nil { + t.Error("a tensor-parallel llama.cpp profile was accepted") + } +} + +func TestCPUProfileValidates(t *testing.T) { + p := Profile{Name: "cpu", Engine: "llamacpp", TP: 1, ModelPath: "/models/qwen.gguf"} + if err := (&LlamaCpp{}).Validate(p, nil); err != nil { + t.Errorf("a CPU profile on a node with no GPUs was rejected: %v", err) + } +} + +func TestLlamaCppAlwaysServesHTTP(t *testing.T) { + // Unlike the others there is no worker rank to get this wrong for. + if !(&LlamaCpp{}).ServesHTTP(llamaSpec(nil)) { + t.Error("a llama.cpp shard must answer the API") + } +} + +func TestBinaryIsConfigurable(t *testing.T) { + l := &LlamaCpp{cfg: LlamaCppConfig{Bin: "/opt/llama.cpp/bin/llama-server", Parallel: 2}} + cmds, _ := l.Commands(llamaSpec(nil)) + if cmds[0].Path != "/opt/llama.cpp/bin/llama-server" { + t.Errorf("path %q", cmds[0].Path) + } + if v, _ := flagValue(cmds[0].Args, "--parallel"); v != "2" { + t.Errorf("--parallel %q", v) + } +} + +func TestUnknownEngineNamesLlamaCpp(t *testing.T) { + _, err := NewRegistry(Config{}).Get("llama.cpp") + if err == nil || !strings.Contains(err.Error(), "llamacpp") { + t.Errorf("the suggestion should name the real engine: %v", err) + } +} + +// flagValue reads a "--flag value" pair out of a command line. +func flagValue(args []string, flag string) (string, bool) { + for i, a := range args { + if a == flag && i+1 < len(args) { + return args[i+1], true + } + } + return "", false +} diff --git a/internal/config/config.go b/internal/config/config.go index f05a83e..d92a3a0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -143,12 +143,15 @@ type Scoring struct { // Endpoint is one engine replica. Static in phase 1. type Endpoint struct { - ID string `koanf:"id"` - URL string `koanf:"url"` - Models []string `koanf:"models"` - Pool string `koanf:"pool"` - Weight float64 `koanf:"weight"` - Priority int `koanf:"priority"` + ID string `koanf:"id"` + URL string `koanf:"url"` + Models []string `koanf:"models"` + // Engine names the inference engine, which decides what its metrics are + // called. Empty means vLLM's names. + Engine string `koanf:"engine"` + Pool string `koanf:"pool"` + Weight float64 `koanf:"weight"` + Priority int `koanf:"priority"` } // Default returns the configuration used when nothing is specified. diff --git a/internal/control/catalog/catalog.go b/internal/control/catalog/catalog.go index 1313c9f..9f64255 100644 --- a/internal/control/catalog/catalog.go +++ b/internal/control/catalog/catalog.go @@ -51,6 +51,20 @@ type Profile struct { StartupTimeout time.Duration `koanf:"startup_timeout"` } +// CPUOnly reports whether this profile needs no GPUs. +// +// Naming no GPU class is how a profile says so, and only engines that can +// actually serve without a card are allowed to leave it out — otherwise an +// omitted class would quietly become a replica placed on a machine with no +// hardware to run it. +func (p Profile) CPUOnly() bool { return p.GPUClass == "" && cpuCapable(p.Engine) } + +// cpuCapable reports whether an engine can serve a model without a GPU. +// +// llama.cpp is the one that can, and it is what makes a cluster with no cards +// testable end to end. +func cpuCapable(engine string) bool { return engine == "llamacpp" } + // Model is a catalog entry as clients see it. type Model struct { Name string `koanf:"name"` @@ -136,7 +150,11 @@ func (m Model) normalize() Model { if p.Nodes <= 0 { p.Nodes = 1 } - if p.GPUsPerNode <= 0 { + switch { + case p.CPUOnly(): + // Nothing to divide up: this profile asks for no cards at all. + p.GPUsPerNode = 0 + case p.GPUsPerNode <= 0: // The common case: a single-node profile uses TP GPUs on one node. p.GPUsPerNode = p.TP / p.Nodes if p.GPUsPerNode <= 0 { @@ -177,8 +195,16 @@ func (m Model) validate() error { } seen[p.Name] = struct{}{} - if p.GPUClass == "" { - return fmt.Errorf("model %q profile %q: gpu_class is required", m.Name, p.Name) + if p.GPUClass == "" && !cpuCapable(p.Engine) { + return fmt.Errorf("model %q profile %q: gpu_class is required for engine %q", + m.Name, p.Name, p.Engine) + } + if p.CPUOnly() { + if p.Nodes > 1 { + return fmt.Errorf("model %q profile %q: a CPU profile cannot span %d nodes", + m.Name, p.Name, p.Nodes) + } + continue } // The arithmetic has to add up, or the engine is handed a shard spec it // cannot satisfy and fails minutes later during model loading. diff --git a/internal/control/catalog/cpu_test.go b/internal/control/catalog/cpu_test.go new file mode 100644 index 0000000..477ce22 --- /dev/null +++ b/internal/control/catalog/cpu_test.go @@ -0,0 +1,77 @@ +package catalog + +import ( + "strings" + "testing" +) + +func TestCPUProfileMayOmitGPUClass(t *testing.T) { + c, err := New([]Model{{ + Name: "qwen-0.5b", + Source: "/models/qwen.gguf", + Profiles: []Profile{{Name: "cpu", Engine: "llamacpp"}}, + }}) + if err != nil { + t.Fatalf("a llama.cpp profile with no GPU class was rejected: %v", err) + } + + m, _ := c.Resolve("qwen-0.5b") + p := m.Profiles[0] + if !p.CPUOnly() { + t.Error("the profile was not recognized as CPU-only") + } + if p.GPUsPerNode != 0 { + t.Errorf("a CPU profile asks for %d GPUs per node", p.GPUsPerNode) + } + if pp := m.PlacementProfiles()[0]; !pp.CPUOnly() || pp.GPUsPerNode != 0 { + t.Errorf("placement sees %+v", pp) + } +} + +func TestGPUEngineStillRequiresItsClass(t *testing.T) { + // Omitting the class for vLLM is a mistake, and one that would otherwise + // surface as a replica placed on a machine with no cards, minutes later. + _, err := New([]Model{{ + Name: "llama-70b", + Source: "meta/llama", + Profiles: []Profile{{Name: "tp4", Engine: "vllm", TP: 4}}, + }}) + if err == nil || !strings.Contains(err.Error(), "gpu_class is required") { + t.Errorf("expected the class to be required for vllm: %v", err) + } +} + +func TestCPUProfileCannotSpanNodes(t *testing.T) { + // llama.cpp has no equivalent of a replica spanning machines, so accepting + // this would place shards that never form one replica. + _, err := New([]Model{{ + Name: "qwen-0.5b", + Source: "/models/qwen.gguf", + Profiles: []Profile{{Name: "cpu", Engine: "llamacpp", Nodes: 2}}, + }}) + if err == nil || !strings.Contains(err.Error(), "cannot span") { + t.Errorf("expected a multi-node CPU profile to be refused: %v", err) + } +} + +func TestCPUProfileReachesTheEngineWithoutGPUs(t *testing.T) { + c, err := New([]Model{{ + Name: "qwen-0.5b", + Source: "/models/qwen.gguf", + Profiles: []Profile{{Name: "cpu", Engine: "llamacpp", Args: []string{"--ctx-size", "4096"}}}, + }}) + if err != nil { + t.Fatal(err) + } + m, _ := c.Resolve("qwen-0.5b") + ep, ok := m.EngineProfile("cpu") + if !ok { + t.Fatal("the engine profile is missing") + } + if ep.Engine != "llamacpp" || ep.ModelPath != "/models/qwen.gguf" { + t.Errorf("engine profile %+v", ep) + } + if ep.ServedModelName != "qwen-0.5b" { + t.Errorf("served as %q", ep.ServedModelName) + } +} diff --git a/internal/control/control.go b/internal/control/control.go index f1b046b..e308857 100644 --- a/internal/control/control.go +++ b/internal/control/control.go @@ -164,6 +164,7 @@ func (c *Controller) endpoints() []endpoint.Info { ID: rep.ID, URL: strings.TrimRight(rep.Endpoint, "/"), Models: names, + Engine: prof.Engine, Pool: rep.Pool, Weight: rep.Weight, Priority: prof.Priority, @@ -176,7 +177,7 @@ func (c *Controller) endpoints() []endpoint.Info { func fingerprint(infos []endpoint.Info) string { var b strings.Builder for _, i := range infos { - fmt.Fprintf(&b, "%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Pool, i.Models, i.Weight, i.Priority) + fmt.Fprintf(&b, "%s|%s|%s|%s|%v|%g|%d;", i.ID, i.URL, i.Engine, i.Pool, i.Models, i.Weight, i.Priority) } return b.String() } diff --git a/internal/control/placement/cpu_test.go b/internal/control/placement/cpu_test.go new file mode 100644 index 0000000..d5c4891 --- /dev/null +++ b/internal/control/placement/cpu_test.go @@ -0,0 +1,102 @@ +package placement + +import ( + "testing" + + "gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory" +) + +// cpuProfile is what the catalog produces for a llama.cpp model: no GPU class, +// because there is no card to ask for. +func cpuProfile(name string) Profile { + return Profile{Name: name, Engine: "llamacpp"} +} + +func TestCPUProfilePlacesOnANodeWithNoGPUs(t *testing.T) { + // The whole point: a machine with no cards is useless to every other engine + // and perfectly fine for this one. + s := New(store(node("n1", "", 0))) + + res := s.Place([]Profile{cpuProfile("cpu")}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if len(res.Placement.Nodes) != 1 || res.Placement.Nodes[0].NodeID != "n1" { + t.Fatalf("placed on %+v", res.Placement.Nodes) + } + if len(res.Placement.Nodes[0].GPUIndices) != 0 { + t.Errorf("a CPU replica was given GPUs: %v", res.Placement.Nodes[0].GPUIndices) + } + if res.Placement.Pool != "none/cpu" { + t.Errorf("pool %q", res.Placement.Pool) + } +} + +func TestCPUReplicasSpreadRatherThanStack(t *testing.T) { + // There is no allocation to stop two CPU replicas sharing a machine's + // cores, so placement has to do it. Bin packing, which is right for GPUs, + // would put every replica on the same node. + st := store(node("n1", "", 0), node("n2", "", 0), node("n3", "", 0)) + s := New(st) + + seen := map[string]int{} + for i := range 3 { + res := s.Place([]Profile{cpuProfile("cpu")}, nil) + if res.Placement == nil { + t.Fatalf("iteration %d:\n%s", i, res.Explain()) + } + id := res.Placement.Nodes[0].NodeID + seen[id]++ + // Record it the way the reconciler would, so the next decision sees it. + st.Allocate(id, inventory.Allocation{ShardID: id + "-sh", ReplicaID: id + "-rep"}) + } + if len(seen) != 3 { + t.Errorf("three replicas landed on %d nodes: %v", len(seen), seen) + } +} + +func TestCPUProfileIgnoresGPUsItDoesNotNeed(t *testing.T) { + // A GPU node is still a valid host, and taking it must not consume cards + // that a GPU profile would need. + st := store(node("gpu", "h100-80g", 8)) + s := New(st) + + res := s.Place([]Profile{cpuProfile("cpu")}, nil) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if n := len(res.Placement.Nodes[0].GPUIndices); n != 0 { + t.Errorf("a CPU replica claimed %d GPUs", n) + } + if free := st.FreeGPUs("gpu"); len(free) != 8 { + t.Errorf("GPUs were consumed: %d free", len(free)) + } +} + +func TestGPUProfileStillNeedsItsClass(t *testing.T) { + // The CPU path must not become a way for a GPU profile to be placed on a + // machine that cannot run it. + s := New(store(node("n1", "", 0))) + + res := s.Place([]Profile{{Name: "tp1", GPUClass: "h100-80g", GPUsPerNode: 1}}, nil) + if res.Placement != nil { + t.Fatal("a GPU profile was placed on a node with no GPUs") + } + if len(res.Rejections) == 0 { + t.Error("the rejection should say why") + } +} + +func TestCPUProfileRespectsAvoid(t *testing.T) { + // Spreading replicas of one model across failure domains matters the same + // way it does for GPU replicas. + s := New(store(node("n1", "", 0), node("n2", "", 0))) + + res := s.Place([]Profile{cpuProfile("cpu")}, []string{"n1"}) + if res.Placement == nil { + t.Fatalf("no placement:\n%s", res.Explain()) + } + if got := res.Placement.Nodes[0].NodeID; got != "n2" { + t.Errorf("placed on %q despite being told to avoid it", got) + } +} diff --git a/internal/control/placement/placement.go b/internal/control/placement/placement.go index 10a77be..724dcc1 100644 --- a/internal/control/placement/placement.go +++ b/internal/control/placement/placement.go @@ -34,12 +34,27 @@ type Profile struct { Priority int } +// CPUOnly reports whether this profile needs no GPUs. The catalog only lets a +// profile omit its GPU class when its engine can serve without one. +func (p Profile) CPUOnly() bool { return p.GPUClass == "" } + +// classLabel names the profile's hardware for pools and messages. +func (p Profile) classLabel() string { + if p.CPUOnly() { + return "cpu" + } + return p.GPUClass +} + // normalize fills in the defaults a catalog entry may omit. func (p Profile) normalize() Profile { if p.Nodes <= 0 { p.Nodes = 1 } - if p.GPUsPerNode <= 0 { + switch { + case p.CPUOnly(): + p.GPUsPerNode = 0 + case p.GPUsPerNode <= 0: p.GPUsPerNode = 1 } return p @@ -166,6 +181,21 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re return len(candidates[i].free) < len(candidates[j].free) }) + if p.CPUOnly() { + // Nothing to bin pack, so spread instead: two CPU replicas on one + // machine compete for the same cores, and unlike GPUs there is no + // allocation to stop them. + sort.SliceStable(candidates, func(i, j int) bool { + return len(s.inv.Allocations(candidates[i].node.ID)) < + len(s.inv.Allocations(candidates[j].node.ID)) + }) + return &Placement{ + Profile: p, + Pool: "none/cpu", + Nodes: []NodeAssignment{{NodeID: candidates[0].node.ID, Rank: 0}}, + }, rejections + } + best := candidates[0] gpus := s.pickGPUs(best.node, best.free, p.GPUsPerNode) if gpus == nil { @@ -178,7 +208,7 @@ func (s *Scheduler) placeSingleNode(p Profile, avoid []string) (*Placement, []Re return &Placement{ Profile: p, - Pool: "none/" + p.GPUClass, + Pool: "none/" + p.classLabel(), Nodes: []NodeAssignment{{NodeID: best.node.ID, Rank: 0, GPUIndices: gpus}}, }, rejections } @@ -201,7 +231,7 @@ func (s *Scheduler) placeMultiNode(p Profile, avoid []string) (*Placement, []Rej } for _, fabric := range fabrics { - pool := fabric + "/" + p.GPUClass + pool := fabric + "/" + p.classLabel() candidates := s.eligibleNodes(p, fabric, avoid, &rejections) if len(candidates) < p.Nodes { @@ -262,7 +292,7 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje if fabric != "" { pool = fabric } - pool += "/" + p.GPUClass + pool += "/" + p.classLabel() var out []candidate for _, n := range s.inv.Nodes() { @@ -290,6 +320,12 @@ func (s *Scheduler) eligibleNodes(p Profile, fabric string, avoid []string, reje } } + if p.CPUOnly() { + // Every ready node can run it; there is no card to be free. + out = append(out, candidate{node: n}) + continue + } + freeIdx := s.inv.FreeGPUs(n.ID) var free []discovery.GPU for _, g := range n.GPUs { diff --git a/internal/router/endpoint/endpoint.go b/internal/router/endpoint/endpoint.go index 012d229..a47f3f7 100644 --- a/internal/router/endpoint/endpoint.go +++ b/internal/router/endpoint/endpoint.go @@ -24,6 +24,9 @@ type Info struct { URL string // Models lists the model names this replica serves. Models []string + // Engine names the inference engine serving here, which decides what its + // metrics are called. Empty means vLLM's names. + Engine string // Pool names the (fabric, gpu_class) pool. Used for observability and pool // preference only; phase 1 may well have a single pool. Pool string diff --git a/internal/router/router.go b/internal/router/router.go index eae0ab3..2247967 100644 --- a/internal/router/router.go +++ b/internal/router/router.go @@ -168,6 +168,7 @@ func toInfos(eps []config.Endpoint) []endpoint.Info { ID: e.ID, URL: strings.TrimRight(e.URL, "/"), Models: e.Models, + Engine: e.Engine, Pool: e.Pool, Weight: e.Weight, Priority: e.Priority, diff --git a/internal/router/scrape/llamacpp_test.go b/internal/router/scrape/llamacpp_test.go new file mode 100644 index 0000000..cf0d4ce --- /dev/null +++ b/internal/router/scrape/llamacpp_test.go @@ -0,0 +1,96 @@ +package scrape + +import ( + "errors" + "strings" + "testing" +) + +// llamaMetrics is what llama-server --metrics actually serves, trimmed to the +// families that matter. +const llamaMetrics = `# HELP llamacpp:prompt_tokens_total Number of prompt tokens processed. +# TYPE llamacpp:prompt_tokens_total counter +llamacpp:prompt_tokens_total 1200 +# HELP llamacpp:tokens_predicted_total Number of generation tokens processed. +# TYPE llamacpp:tokens_predicted_total counter +llamacpp:tokens_predicted_total 3400 +# HELP llamacpp:kv_cache_usage_ratio KV-cache usage. +# TYPE llamacpp:kv_cache_usage_ratio gauge +llamacpp:kv_cache_usage_ratio 0.42 +# HELP llamacpp:requests_processing Number of requests processing. +# TYPE llamacpp:requests_processing gauge +llamacpp:requests_processing 3 +# HELP llamacpp:requests_deferred Number of requests deferred. +# TYPE llamacpp:requests_deferred gauge +llamacpp:requests_deferred 7 +` + +func TestLlamaCppMetricsSatisfyTheContract(t *testing.T) { + s, err := Parse(strings.NewReader(llamaMetrics), LlamaCppMapping()) + if err != nil { + t.Fatal(err) + } + if s.Running != 3 { + t.Errorf("running=%d", s.Running) + } + // "Deferred" is llama.cpp's word for a request that arrived with every slot + // busy, which is what the scheduler needs to read as queue depth. + if s.Waiting != 7 { + t.Errorf("waiting=%d", s.Waiting) + } + if s.KVCacheUsage != 0.42 { + t.Errorf("kv=%v", s.KVCacheUsage) + } + if s.PromptTokens != 1200 || s.GenerationTokens != 3400 { + t.Errorf("tokens=%v/%v", s.PromptTokens, s.GenerationTokens) + } +} + +func TestLlamaCppMappingDoesNotInheritVLLMNames(t *testing.T) { + // llama.cpp has no request-success counter. Filling that field with vLLM's + // name would send the router hunting for a metric that cannot be there, and + // would hide the fact that this engine simply does not report it. + m := LlamaCppMapping() + m.normalize() + if m.SuccessTotal != "" || m.CacheConfig != "" { + t.Errorf("vLLM names leaked into the llama.cpp mapping: %+v", m) + } + if m.Running != "llamacpp:requests_processing" { + t.Errorf("running=%q", m.Running) + } +} + +func TestEmptyMappingStillDefaultsToVLLM(t *testing.T) { + var m Mapping + m.normalize() + if m.Running != "vllm:num_requests_running" || m.SuccessTotal == "" { + t.Errorf("an unconfigured mapping should be vLLM's: %+v", m) + } +} + +func TestVLLMNamesAreNotReadAsLlamaCpp(t *testing.T) { + // The failure this guards against is silent: reading a vLLM endpoint with + // llama.cpp's vocabulary finds nothing, and an endpoint with no numbers + // must be reported as unmeasurable rather than as idle. + var missing *MissingError + _, err := Parse(strings.NewReader(llamaMetrics), VLLMMapping()) + if !errors.As(err, &missing) { + t.Fatalf("expected the contract to be reported incomplete, got %v", err) + } + if len(missing.Missing) != 3 { + t.Errorf("all three mandatory metrics should be named: %v", missing.Missing) + } +} + +func TestMappingForPicksByEngine(t *testing.T) { + if MappingFor("llamacpp").Running != "llamacpp:requests_processing" { + t.Error("llamacpp did not get its own mapping") + } + // An endpoint that names no engine, or one this build has never heard of, + // is read with vLLM's names rather than left unreadable. + for _, engine := range []string{"", "vllm", "something-new"} { + if MappingFor(engine).Running != "vllm:num_requests_running" { + t.Errorf("engine %q did not fall back to vLLM", engine) + } + } +} diff --git a/internal/router/scrape/parse.go b/internal/router/scrape/parse.go index a697129..ee68ac9 100644 --- a/internal/router/scrape/parse.go +++ b/internal/router/scrape/parse.go @@ -30,6 +30,40 @@ type Mapping struct { SuccessTotal string } +// MappingFor returns the metric names for an engine. +// +// Per engine rather than per cluster: a cluster running llama.cpp on the CPU +// nodes and vLLM on the GPU nodes has two vocabularies for the same three +// numbers, and one global mapping would leave half the fleet unmeasurable. +// An unknown engine falls back to vLLM's names, which is also what an endpoint +// that names no engine gets. +func MappingFor(engine string) Mapping { + switch engine { + case "llamacpp": + return LlamaCppMapping() + default: + return VLLMMapping() + } +} + +// LlamaCppMapping returns llama.cpp's metric names, which llama-server only +// exposes when started with --metrics. +// +// "Deferred" is llama.cpp's word for a request that arrived with every slot +// busy, which is the same thing vLLM calls waiting. The fields left empty are +// metrics llama.cpp does not have: it counts decodes rather than requests, so +// there is no success counter and no cache configuration to read block sizes +// from. +func LlamaCppMapping() Mapping { + return Mapping{ + Running: "llamacpp:requests_processing", + Waiting: "llamacpp:requests_deferred", + KVCacheUsage: "llamacpp:kv_cache_usage_ratio", + PromptTokens: "llamacpp:prompt_tokens_total", + GenerationTokens: "llamacpp:tokens_predicted_total", + } +} + // VLLMMapping returns vLLM's current metric names. func VLLMMapping() Mapping { return Mapping{ @@ -43,7 +77,16 @@ func VLLMMapping() Mapping { } } +// normalize fills in vLLM's names for an empty mapping. +// +// Only for an empty one. A mapping that names its own Running metric is a +// complete statement about some other engine, and the fields it leaves blank +// are metrics that engine does not expose — filling those with vLLM's names +// would have the router hunting for metrics that cannot be there. func (m *Mapping) normalize() { + if m.Running != "" { + return + } d := VLLMMapping() if m.Running == "" { m.Running = d.Running diff --git a/internal/router/scrape/poller.go b/internal/router/scrape/poller.go index d40dee2..0a3aaea 100644 --- a/internal/router/scrape/poller.go +++ b/internal/router/scrape/poller.go @@ -20,7 +20,8 @@ type Config struct { Timeout time.Duration // Path is the metrics endpoint. Path string - // Mapping selects the metric names. + // Mapping selects the metric names for endpoints that name no engine. An + // endpoint that names one is read with that engine's mapping instead. Mapping Mapping } @@ -152,7 +153,7 @@ func (p *Poller) loop(ctx context.Context, in endpoint.Info) { } func (p *Poller) once(ctx context.Context, in endpoint.Info) { - stats, err := p.fetch(ctx, in.URL) + stats, err := p.fetch(ctx, in.URL, p.mappingFor(in)) if err != nil { if ctx.Err() != nil { return @@ -164,7 +165,7 @@ func (p *Poller) once(ctx context.Context, in endpoint.Info) { p.reg.Observe(in.ID, stats) } -func (p *Poller) fetch(ctx context.Context, base string) (endpoint.Stats, error) { +func (p *Poller) fetch(ctx context.Context, base string, mapping Mapping) (endpoint.Stats, error) { ctx, cancel := context.WithTimeout(ctx, p.cfg.Timeout) defer cancel() @@ -181,5 +182,16 @@ func (p *Poller) fetch(ctx context.Context, base string) (endpoint.Stats, error) if resp.StatusCode != http.StatusOK { return endpoint.Stats{}, fmt.Errorf("unexpected status: %s", resp.Status) } - return Parse(resp.Body, p.cfg.Mapping) + return Parse(resp.Body, mapping) +} + +// mappingFor picks the vocabulary this endpoint's metrics are written in. +// +// The configured mapping wins for endpoints that name no engine, so an operator +// can still override the names when a version renames them (plan §14). +func (p *Poller) mappingFor(in endpoint.Info) Mapping { + if in.Engine == "" { + return p.cfg.Mapping + } + return MappingFor(in.Engine) }