Merge C24 KV-index routing from vLLM's cache events
Conflicts with the LoRA work: vLLM's arguments, the engine profile, router metrics and the scheduler's request all keep both; agent.pb.go regenerated from the merged proto; the two test fakes of vLLM given separate names. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
9e46ee83e4
43 files changed
+3498
-265
No files matched your search
@@ -250,6 +250,20 @@ type Profile struct {
|
||||
// StartupTimeout bounds model loading. A 70B model across two nodes takes
|
||||
// tens of minutes, so this is per profile rather than global.
|
||||
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
|
||||
|
||||
// KVEvents has vLLM publish what its prefix cache holds, and the router
|
||||
// route this model's requests by it: to the replica holding the most of
|
||||
// a prompt's leading blocks (plan §7.5 tier 2). vLLM only.
|
||||
//
|
||||
// Off by default, because it costs something on every request: the
|
||||
// router asks a replica to tokenize the prompt before placing it, a
|
||||
// round trip and the chat template run twice. It pays where prompts
|
||||
// share long prefixes that the approximate tree guesses wrong — many
|
||||
// replicas, conversations that move between them, evictions. The engine
|
||||
// is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,
|
||||
// which the router's hashing assumes; arguments overriding either leave
|
||||
// the index empty, and the agent says so in its log.
|
||||
KVEvents bool `json:"kv_events,omitempty" koanf:"kv_events"`
|
||||
}
|
||||
|
||||
// CPUOnly reports whether this profile needs no GPUs.
|
||||
@@ -739,6 +753,10 @@ func (m Model) validate() error {
|
||||
if err := engines.UnsafeArgs(p.Engine, p.Args); err != nil {
|
||||
return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err)
|
||||
}
|
||||
if p.KVEvents && p.Engine != "vllm" {
|
||||
return fmt.Errorf("model %q profile %q: kv_events is vLLM's; engine %q publishes no cache events",
|
||||
m.Name, p.Name, p.Engine)
|
||||
}
|
||||
|
||||
if p.PowerWatts < 0 {
|
||||
return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts)
|
||||
@@ -906,11 +924,24 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
||||
ModelPath: m.Source,
|
||||
ServedModelName: m.Name,
|
||||
Adapters: m.engineAdapters(),
|
||||
KVEvents: p.KVEvents,
|
||||
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
|
||||
}
|
||||
return engines.Profile{}, false
|
||||
}
|
||||
|
||||
// KVIndex reports whether the router should route this model by the KV
|
||||
// cache its replicas report: whether any profile publishes events. A replica
|
||||
// of another profile is simply one the index has not heard from.
|
||||
func (m Model) KVIndex() bool {
|
||||
for _, p := range m.Profiles {
|
||||
if p.KVEvents {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ProfileByName returns one of a model's profiles.
|
||||
func (m Model) ProfileByName(name string) (Profile, bool) {
|
||||
for _, p := range m.Profiles {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestKVEventsReachTheEngineAndTheRouter(t *testing.T) {
|
||||
c, err := New([]Model{model("m", Profile{Name: "p", GPUClass: "h100-80g", TP: 1, KVEvents: true})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := c.Resolve("m")
|
||||
if !m.KVIndex() {
|
||||
t.Fatal("a model with a profile publishing events should be routed by them")
|
||||
}
|
||||
if ep, _ := m.EngineProfile("p"); !ep.KVEvents {
|
||||
t.Fatal("the engine has to be told to publish")
|
||||
}
|
||||
plain, _ := New([]Model{model("n")})
|
||||
if n, _ := plain.Resolve("n"); n.KVIndex() {
|
||||
t.Fatal("off by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVEventsAreVLLMsOnly(t *testing.T) {
|
||||
_, err := New([]Model{model("m", Profile{Name: "p", Engine: "llamacpp", KVEvents: true})})
|
||||
if err == nil || !strings.Contains(err.Error(), "kv_events") {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTurningKVEventsOnMarksRunningReplicasStale(t *testing.T) {
|
||||
p := Profile{Name: "p", GPUClass: "h100-80g", TP: 1}
|
||||
q := p
|
||||
q.KVEvents = true
|
||||
if p.Fingerprint() == q.Fingerprint() {
|
||||
t.Fatal("replicas started without events do not publish until replaced, and should be shown as stale")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user