Merge C24 KV-index routing from vLLM's cache events

Conflicts with the LoRA work: vLLM's arguments, the engine profile, router
metrics and the scheduler's request all keep both; agent.pb.go regenerated
from the merged proto; the two test fakes of vLLM given separate names.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:24:25 +02:00
commit 9e46ee83e4
43 files changed
+3498 -265

No files matched your search

+2
View File
@@ -198,6 +198,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
Args: p.Args, Env: p.Env,
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
Adapters: adaptersToWire(p.Adapters),
KvEvents: p.KVEvents,
},
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
Pool: cmd.Pool,
@@ -234,6 +235,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
Args: p.GetArgs(), Env: p.GetEnv(),
ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(),
Adapters: adaptersFromWire(p.GetAdapters()),
KVEvents: p.GetKvEvents(),
},
}
return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond
+103
View File
@@ -0,0 +1,103 @@
package agentsvc
import (
"sync"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
)
// SetKVSink has the KV cache events agents forward delivered to fn — the
// router's precise index, in a process that serves both. Unset, they are
// dropped, which is right for a control plane with no router beside it.
func (s *Service) SetKVSink(fn func(kvevents.Update)) {
s.mu.Lock()
defer s.mu.Unlock()
s.kv = fn
}
func (s *Service) kvSink() func(kvevents.Update) {
s.mu.RLock()
defer s.mu.RUnlock()
return s.kv
}
// kvState is what one stream has said about KV caches: which replicas the
// node runs, and so may describe, and which it has described.
type kvState struct {
mu sync.Mutex
owned map[string]bool
described map[string]bool
}
// own records a replica the node runs a shard of.
func (c *conn) own(replicaID string) {
if replicaID == "" {
return
}
c.kv.mu.Lock()
defer c.kv.mu.Unlock()
if c.kv.owned == nil {
c.kv.owned = map[string]bool{}
}
c.kv.owned[replicaID] = true
}
// kvEvents passes one replica's update on.
//
// Only for a replica the node has said it runs. A node describing another's
// cache — misconfigured, or worse — could otherwise draw that replica's
// traffic, or starve it, from anywhere in the cluster.
func (s *Service) kvEvents(c *conn, m *llmgwv1.KVEvents) {
sink := s.kvSink()
if sink == nil {
return
}
id := m.GetReplicaId()
c.kv.mu.Lock()
ok := c.kv.owned[id]
if ok {
if c.kv.described == nil {
c.kv.described = map[string]bool{}
}
c.kv.described[id] = true
}
c.kv.mu.Unlock()
if !ok {
s.log.Debug("ignoring kv cache events for a replica the node does not run",
"node", c.nodeID, "replica", id)
return
}
sink(kvevents.Update{
Replica: id, BlockSize: int(m.GetBlockSize()), Reset: m.GetClearFirst(),
Stored: m.GetStored(), Removed: m.GetRemoved(), Problem: m.GetProblem(),
})
}
// kvGone empties what a closed stream described. Events stop arriving with
// it, evictions among them, and an index that kept the blocks would steer
// requests to caches that may have moved on; the next stream starts with a
// full copy anyway.
func (s *Service) kvGone(c *conn) {
sink := s.kvSink()
if sink == nil {
return
}
c.kv.mu.Lock()
ids := make([]string, 0, len(c.kv.described))
for id := range c.kv.described {
ids = append(ids, id)
}
c.kv.mu.Unlock()
for _, id := range ids {
sink(kvevents.Update{Replica: id, Reset: true})
}
}
// KVEventsMessage wraps an update for the stream, as the agent sends it.
func KVEventsMessage(u kvevents.Update) *llmgwv1.AgentMessage {
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_KvEvents{KvEvents: &llmgwv1.KVEvents{
ReplicaId: u.Replica, BlockSize: int32(u.BlockSize), ClearFirst: u.Reset,
Stored: u.Stored, Removed: u.Removed, Problem: u.Problem,
}}}
}
+29
View File
@@ -0,0 +1,29 @@
package agentsvc
import (
"log/slog"
"testing"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
)
func TestKVEventsAreTakenOnlyForTheNodesOwnReplicas(t *testing.T) {
s := &Service{log: slog.New(slog.DiscardHandler)}
var got []kvevents.Update
s.SetKVSink(func(u kvevents.Update) { got = append(got, u) })
c := &conn{nodeID: "n1"}
c.own("r1")
s.kvEvents(c, KVEventsMessage(kvevents.Update{Replica: "r1", BlockSize: 16, Reset: true, Stored: []uint64{7}}).GetKvEvents())
s.kvEvents(c, KVEventsMessage(kvevents.Update{Replica: "r-elsewhere", Stored: []uint64{8}}).GetKvEvents())
if len(got) != 1 || got[0].Replica != "r1" || !got[0].Reset || got[0].Stored[0] != 7 || got[0].BlockSize != 16 {
t.Fatalf("got %+v; a node may describe its own replicas' caches and no others", got)
}
// The stream closing empties what it described: no more evictions will
// arrive by it.
s.kvGone(c)
if len(got) != 2 || got[1].Replica != "r1" || !got[1].Reset || len(got[1].Stored) != 0 {
t.Fatalf("got %+v", got)
}
}
+16
View File
@@ -24,6 +24,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
@@ -58,6 +59,8 @@ type Service struct {
mu sync.RWMutex
conns map[string]*conn
// kv receives the KV cache events agents forward; see SetKVSink.
kv func(kvevents.Update)
// enginesFor is what a node is told to install, set once at start: the
// cluster's engines, with any override that picks the node laid over.
@@ -126,6 +129,8 @@ type conn struct {
send chan *llmgwv1.ControlMessage
closed chan struct{}
once sync.Once
// kv is what this stream has said about KV caches.
kv kvState
}
func (c *conn) close() { c.once.Do(func() { close(c.closed) }) }
@@ -245,6 +250,10 @@ func (s *Service) Connect(
c := s.attach(ctx, nodeID)
defer s.detach(nodeID, c)
for _, rs := range reg.GetRunningShards() {
c.own(rs.GetReplicaId())
}
defer s.kvGone(c)
if err := stream.Send(&llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Ack{Ack: &llmgwv1.Ack{MessageId: "register", Ok: true}},
@@ -304,6 +313,13 @@ func (s *Service) Connect(
s.log.Warn("agent stream failed", "node", nodeID, "err", err)
return nil
}
if kv := msg.GetKvEvents(); kv != nil {
s.kvEvents(c, kv)
continue
}
if sh := msg.GetShard(); sh != nil {
c.own(sh.GetReplicaId())
}
s.handle(nodeID, msg)
}
}