Precise KV-index routing: vLLM KV cache events through the agent, routed by leading blocks held
Plan §7.5 tier 2, opt-in per profile with kv_events. The agent starts vLLM publishing KV cache events to a loopback port it listens on (pure-Go zmq4, msgpack in both of vLLM's encodings), with sha256_cbor hashing and a fixed PYTHONHASHSEED; it mirrors each engine's blocks and sends net changes up the existing stream, with a full copy on every new one and a reset on gaps, restarts and clears. The control plane hands them to the router, for the sending node's own replicas only. The router tokenizes through a replica's /tokenize, hashes the tokens as vLLM does (checked against cbor2/hashlib golden values), and scores replicas by the share of leading blocks held, falling back to the approximate tree. New metrics beside llmgw_prefix_decisions_total. Untested against a running vLLM. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
d41104a452
commit
117f417ef5
43 files changed
+3496
-263
No files matched your search
@@ -45,6 +45,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/guard"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/mirror"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
@@ -684,6 +685,11 @@ func startControlPlane(ctx context.Context, cfg config.Server, cat *catalog.Live
|
||||
svc.SetEnginesFor(func(n inventory.Node) map[string]string {
|
||||
return cfg.EnginesFor(n.ID, slices.Sorted(maps.Keys(n.GPUsByClass())), n.Labels)
|
||||
})
|
||||
// The KV cache events agents forward go to the router's precise index,
|
||||
// when there is a router in this process to have one.
|
||||
if k, ok := sink.(interface{ ApplyKV(kvevents.Update) }); ok {
|
||||
svc.SetKVSink(k.ApplyKV)
|
||||
}
|
||||
log.Info("nodes install these engines", "engines", cfg.WantedEngines(), "overrides", len(cfg.EngineOverrides))
|
||||
if ch := ctl.EngineChoices(); ch != nil {
|
||||
svc.SetEngineChoices(ch.For)
|
||||
|
||||
Reference in new issue
Block a user