Merge C24 KV-index routing from vLLM's cache events

Conflicts with the LoRA work: vLLM's arguments, the engine profile, router
metrics and the scheduler's request all keep both; agent.pb.go regenerated
from the merged proto; the two test fakes of vLLM given separate names.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:24:25 +02:00
commit 9e46ee83e4
43 files changed
+3498 -265

No files matched your search

+31
View File
@@ -250,6 +250,20 @@ type Profile struct {
// StartupTimeout bounds model loading. A 70B model across two nodes takes
// tens of minutes, so this is per profile rather than global.
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
// KVEvents has vLLM publish what its prefix cache holds, and the router
// route this model's requests by it: to the replica holding the most of
// a prompt's leading blocks (plan §7.5 tier 2). vLLM only.
//
// Off by default, because it costs something on every request: the
// router asks a replica to tokenize the prompt before placing it, a
// round trip and the chat template run twice. It pays where prompts
// share long prefixes that the approximate tree guesses wrong — many
// replicas, conversations that move between them, evictions. The engine
// is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,
// which the router's hashing assumes; arguments overriding either leave
// the index empty, and the agent says so in its log.
KVEvents bool `json:"kv_events,omitempty" koanf:"kv_events"`
}
// CPUOnly reports whether this profile needs no GPUs.
@@ -739,6 +753,10 @@ func (m Model) validate() error {
if err := engines.UnsafeArgs(p.Engine, p.Args); err != nil {
return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err)
}
if p.KVEvents && p.Engine != "vllm" {
return fmt.Errorf("model %q profile %q: kv_events is vLLM's; engine %q publishes no cache events",
m.Name, p.Name, p.Engine)
}
if p.PowerWatts < 0 {
return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts)
@@ -906,11 +924,24 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
ModelPath: m.Source,
ServedModelName: m.Name,
Adapters: m.engineAdapters(),
KVEvents: p.KVEvents,
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
}
return engines.Profile{}, false
}
// KVIndex reports whether the router should route this model by the KV
// cache its replicas report: whether any profile publishes events. A replica
// of another profile is simply one the index has not heard from.
func (m Model) KVIndex() bool {
for _, p := range m.Profiles {
if p.KVEvents {
return true
}
}
return false
}
// ProfileByName returns one of a model's profiles.
func (m Model) ProfileByName(name string) (Profile, bool) {
for _, p := range m.Profiles {
+40
View File
@@ -0,0 +1,40 @@
package catalog
import (
"strings"
"testing"
)
func TestKVEventsReachTheEngineAndTheRouter(t *testing.T) {
c, err := New([]Model{model("m", Profile{Name: "p", GPUClass: "h100-80g", TP: 1, KVEvents: true})})
if err != nil {
t.Fatal(err)
}
m, _ := c.Resolve("m")
if !m.KVIndex() {
t.Fatal("a model with a profile publishing events should be routed by them")
}
if ep, _ := m.EngineProfile("p"); !ep.KVEvents {
t.Fatal("the engine has to be told to publish")
}
plain, _ := New([]Model{model("n")})
if n, _ := plain.Resolve("n"); n.KVIndex() {
t.Fatal("off by default")
}
}
func TestKVEventsAreVLLMsOnly(t *testing.T) {
_, err := New([]Model{model("m", Profile{Name: "p", Engine: "llamacpp", KVEvents: true})})
if err == nil || !strings.Contains(err.Error(), "kv_events") {
t.Fatalf("err %v", err)
}
}
func TestTurningKVEventsOnMarksRunningReplicasStale(t *testing.T) {
p := Profile{Name: "p", GPUClass: "h100-80g", TP: 1}
q := p
q.KVEvents = true
if p.Fingerprint() == q.Fingerprint() {
t.Fatal("replicas started without events do not publish until replaced, and should be shown as stale")
}
}