Merge C24 KV-index routing from vLLM's cache events
Conflicts with the LoRA work: vLLM's arguments, the engine profile, router metrics and the scheduler's request all keep both; agent.pb.go regenerated from the merged proto; the two test fakes of vLLM given separate names. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
9e46ee83e4
43 files changed
+3498
-265
No files matched your search
+370
-228
File diff suppressed because it is too large.
Load diff
@@ -27,9 +27,36 @@ message AgentMessage {
|
||||
FabricScanReply fabric_scan = 5;
|
||||
FabricTestReply fabric_test = 6;
|
||||
UpgradeReply upgrade = 7;
|
||||
KVEvents kv_events = 8;
|
||||
}
|
||||
}
|
||||
|
||||
// KVEvents is a change to what one vLLM replica's prefix cache holds, read
|
||||
// by the agent from the engine's KV cache events and sent on for the router's
|
||||
// precise index (plan §7.5 tier 2).
|
||||
//
|
||||
// A net effect since the last one, not the events themselves: a block stored
|
||||
// and evicted between two sends is not worth the bytes. On every new stream
|
||||
// the agent first sends each replica in full, with clear_first set, because a
|
||||
// control plane that has just started or taken over knows nothing.
|
||||
message KVEvents {
|
||||
string replica_id = 1;
|
||||
// block_size is how many tokens make one of the replica's blocks.
|
||||
int32 block_size = 2;
|
||||
// clear_first says to forget what was known of the replica before applying the
|
||||
// rest: its cache was cleared, its engine restarted, events were lost, or
|
||||
// this is the first piece of a full copy.
|
||||
bool clear_first = 3;
|
||||
// stored and removed are block hashes as vLLM publishes them: the low 64
|
||||
// bits of the sha256_cbor digest.
|
||||
repeated fixed64 stored = 4;
|
||||
repeated fixed64 removed = 5;
|
||||
// problem, when set, is why the replica's events cannot be used — most
|
||||
// likely an engine hashing another way than llmgw configured. Its index is
|
||||
// empty while it stands.
|
||||
string problem = 6;
|
||||
}
|
||||
|
||||
// ControlMessage is anything the control plane tells a node.
|
||||
message ControlMessage {
|
||||
oneof msg {
|
||||
@@ -659,6 +686,9 @@ message EngineProfile {
|
||||
// adapters are LoRA adapters loaded beside the model, in order: the index
|
||||
// is the id a request applies one by.
|
||||
repeated LoraAdapter adapters = 12;
|
||||
// kv_events has a vLLM engine publish its KV cache events to the agent, for
|
||||
// the router's precise index.
|
||||
bool kv_events = 13;
|
||||
}
|
||||
|
||||
// RemoveWeights deletes cached model files a node no longer needs.
|
||||
|
||||
Reference in new issue
Block a user