Merge C24 KV-index routing from vLLM's cache events

Conflicts with the LoRA work: vLLM's arguments, the engine profile, router
metrics and the scheduler's request all keep both; agent.pb.go regenerated
from the merged proto; the two test fakes of vLLM given separate names.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:24:25 +02:00
commit 9e46ee83e4
43 files changed
+3498 -265

No files matched your search

File diff suppressed because it is too large. Load diff
+30
View File
@@ -27,9 +27,36 @@ message AgentMessage {
FabricScanReply fabric_scan = 5;
FabricTestReply fabric_test = 6;
UpgradeReply upgrade = 7;
KVEvents kv_events = 8;
}
}
// KVEvents is a change to what one vLLM replica's prefix cache holds, read
// by the agent from the engine's KV cache events and sent on for the router's
// precise index (plan §7.5 tier 2).
//
// A net effect since the last one, not the events themselves: a block stored
// and evicted between two sends is not worth the bytes. On every new stream
// the agent first sends each replica in full, with clear_first set, because a
// control plane that has just started or taken over knows nothing.
message KVEvents {
string replica_id = 1;
// block_size is how many tokens make one of the replica's blocks.
int32 block_size = 2;
// clear_first says to forget what was known of the replica before applying the
// rest: its cache was cleared, its engine restarted, events were lost, or
// this is the first piece of a full copy.
bool clear_first = 3;
// stored and removed are block hashes as vLLM publishes them: the low 64
// bits of the sha256_cbor digest.
repeated fixed64 stored = 4;
repeated fixed64 removed = 5;
// problem, when set, is why the replica's events cannot be used — most
// likely an engine hashing another way than llmgw configured. Its index is
// empty while it stands.
string problem = 6;
}
// ControlMessage is anything the control plane tells a node.
message ControlMessage {
oneof msg {
@@ -659,6 +686,9 @@ message EngineProfile {
// adapters are LoRA adapters loaded beside the model, in order: the index
// is the id a request applies one by.
repeated LoraAdapter adapters = 12;
// kv_events has a vLLM engine publish its KV cache events to the agent, for
// the router's precise index.
bool kv_events = 13;
}
// RemoveWeights deletes cached model files a node no longer needs.