Merge C24 KV-index routing from vLLM's cache events

Conflicts with the LoRA work: vLLM's arguments, the engine profile, router
metrics and the scheduler's request all keep both; agent.pb.go regenerated
from the merged proto; the two test fakes of vLLM given separate names.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:24:25 +02:00
commit 9e46ee83e4
43 files changed
+3498 -265

No files matched your search

+20
View File
@@ -35,6 +35,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
@@ -120,6 +121,12 @@ type Agent struct {
// — eight shards starting together produce one download — and all eight
// are waiting, so all eight should say so.
fetchWatchers map[string][]func(weights.Progress)
// kv mirrors the KV caches of the engines that publish events, for the
// router's precise index; kvSubs are their listeners by shard, guarded
// by mu.
kv *kvevents.Mirror
kvSubs map[string]*kvevents.Subscriber
}
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
@@ -162,6 +169,8 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
sensors: discovery.NewSensorSampler(),
wear: discovery.NewHealthSampler(),
serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
kv: kvevents.NewMirror(),
kvSubs: map[string]*kvevents.Subscriber{},
}
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
return a, nil
@@ -280,6 +289,12 @@ func (a *Agent) recoverOrphans() {
a.mu.Lock()
a.shardReplica[o.ShardID] = o.ReplicaID
a.mu.Unlock()
// The engine is still publishing to the port the last agent
// listened on; listen there again.
if o.Spec.KVEventsPort > 0 {
spec := o.Spec
a.listenKV(&spec, o.Spec.KVEventsPort)
}
a.log.Info("adopted a running engine",
"shard", o.ShardID, "replica", o.ReplicaID, "pid", o.PID)
go a.settleAdopted(sh, o)
@@ -533,6 +548,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
}()
defer wg.Wait()
go a.watchExits(ctx, out)
go a.forwardKV(ctx, out)
for {
msg, err := stream.Receive()
@@ -1063,8 +1079,10 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
}
}
a.listenKV(&spec, 0)
cmds, err := eng.Commands(spec)
if err != nil {
a.stopKV(shardID, replicaID)
report(supervisor.Failed, err.Error(), nil)
return
}
@@ -1077,6 +1095,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
report(sh.Status().State, "", nil)
return
}
a.stopKV(shardID, replicaID)
report(supervisor.Failed, err.Error(), sh.Logs(10))
return
}
@@ -1173,6 +1192,7 @@ func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgw
if err != nil {
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
}
a.stopKV(shardID, replicaID)
obs.End(span, err)
a.sup.Remove(shardID)
+8
View File
@@ -44,6 +44,9 @@ type Profile struct {
// Adapters are LoRA adapters to load beside the model, in order: an
// adapter's index is the id requests apply it by.
Adapters []Adapter
// KVEvents has the engine publish its KV cache events to the agent,
// for the router's precise index. vLLM only.
KVEvents bool `json:"kv_events,omitempty"`
}
// ShardSpec is one shard of one replica on this node.
@@ -90,6 +93,11 @@ type ShardSpec struct {
// and accounting. Given to the engine in its environment rather than on
// its command line, which every user on the node can read.
APIKey string `json:"api_key,omitempty"`
// KVEventsPort is the loopback port the agent receives this shard's KV
// cache events on, chosen by the agent; zero when it publishes none.
// Kept with the spec so an agent that restarts listens where the engine
// it adopts is still sending.
KVEventsPort int `json:"kv_events_port,omitempty"`
// Threads is the CPU cores placement reserved for a shard with no GPU,
// and so what its engine computes on. Zero leaves it to the engine's own
// configuration.
+47
View File
@@ -0,0 +1,47 @@
package engines
import (
"slices"
"strings"
"testing"
)
func TestAVLLMShardWithKVEventsPublishesToTheAgentOnLoopback(t *testing.T) {
s := singleNodeSpec()
s.Profile.KVEvents = true
s.KVEventsPort = 5999
s.Profile.Args = []string{"--max-model-len", "8192"}
cmd := buildOne(t, &VLLM{}, s)
i := slices.Index(cmd.Args, "--kv-events-config")
if i < 0 || !strings.Contains(cmd.Args[i+1], `"endpoint":"tcp://127.0.0.1:5999"`) {
t.Fatalf("args %v should name the agent's loopback port", cmd.Args)
}
if j := slices.Index(cmd.Args, "--prefix-caching-hash-algo"); j < 0 || cmd.Args[j+1] != "sha256_cbor" {
t.Fatalf("args %v should pin the hashing the router computes", cmd.Args)
}
if slices.Index(cmd.Args, "--max-model-len") < i {
t.Fatal("profile arguments come last, so an operator can still override")
}
if !slices.Contains(cmd.Env, "PYTHONHASHSEED=1729") {
t.Fatalf("env %v should fix the hash seed", cmd.Env)
}
}
func TestNoKVEventsWithoutAPortOrOffTheLeader(t *testing.T) {
s := singleNodeSpec()
s.Profile.KVEvents = true // but the agent could not listen
if cmd := buildOne(t, &VLLM{}, s); slices.Contains(cmd.Args, "--kv-events-config") {
t.Fatal("an engine told to publish where nobody listens only fills a queue")
}
w := multiNodeSpec(1)
w.Profile.KVEvents, w.KVEventsPort = true, 5999
if cmd := buildEngine(t, &VLLM{}, w); slices.Contains(cmd.Args, "--kv-events-config") {
t.Fatal("a headless worker has no scheduler to publish from")
}
off := singleNodeSpec()
off.KVEventsPort = 5999
if cmd := buildOne(t, &VLLM{}, off); slices.Contains(cmd.Args, "--kv-events-config") {
t.Fatal("kv_events is opt-in")
}
}
+11
View File
@@ -2,8 +2,11 @@ package engines
import (
"fmt"
"maps"
"strconv"
"sync/atomic"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
)
// VLLMConfig locates the vLLM installation and picks how it is launched.
@@ -161,6 +164,14 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
}
}
// The KV cache events, for the router's precise index: published to the
// agent on loopback, hashed the way the router hashes. Only the leader
// has a scheduler to publish from.
if p.KVEvents && s.KVEventsPort > 0 && s.Leader() {
args = append(args, kvevents.VLLMArgs(fmt.Sprintf("tcp://127.0.0.1:%d", s.KVEventsPort))...)
maps.Copy(env, kvevents.VLLMEnv())
}
// Profile arguments last, so an operator can override anything above.
args = append(args, p.Args...)
+86
View File
@@ -0,0 +1,86 @@
package agent
import (
"context"
"time"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
)
// kvFlushInterval is how often KV cache changes go up the stream. Often
// enough that a prefix cached by one request is known before the next few
// arrive; seldom enough that a busy engine's churn is sent as its net effect.
const kvFlushInterval = 250 * time.Millisecond
// listenKV starts receiving a shard's KV cache events, when its profile asks
// for them and this rank publishes any, and records the port in the spec for
// the engine's arguments. port is the one to reuse, or 0 for any.
//
// A failure is logged and the engine started without events: the router then
// routes the replica by tier 1, which is what it did before, rather than the
// replica not starting at all for want of an optimisation.
func (a *Agent) listenKV(spec *engines.ShardSpec, port int) {
if !spec.Profile.KVEvents || spec.Profile.Engine != "vllm" || !spec.Leader() {
return
}
a.mu.Lock()
if sub, ok := a.kvSubs[spec.ShardID]; ok {
a.mu.Unlock()
// A repeated start command: the shard already has its listener.
spec.KVEventsPort = sub.Port()
return
}
a.mu.Unlock()
sub, err := kvevents.Listen(spec.ReplicaID, port, a.kv, a.log)
if err != nil {
a.log.Warn("cannot receive kv cache events; the router will route this replica without them",
"shard", spec.ShardID, "replica", spec.ReplicaID, "err", err)
return
}
a.mu.Lock()
a.kvSubs[spec.ShardID] = sub
a.mu.Unlock()
spec.KVEventsPort = sub.Port()
}
// stopKV stops a shard's listener and tells the control plane its replica
// holds nothing any more.
func (a *Agent) stopKV(shardID, replicaID string) {
a.mu.Lock()
sub, ok := a.kvSubs[shardID]
delete(a.kvSubs, shardID)
a.mu.Unlock()
if !ok {
return
}
sub.Close()
a.kv.Remove(replicaID)
}
// forwardKV sends KV cache changes up one session's stream until it ends,
// beginning with a full copy of every replica: the control plane at the other
// end may have just started, or just taken over, and know nothing.
func (a *Agent) forwardKV(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
s := a.kv.Open()
defer s.Close()
t := time.NewTicker(kvFlushInterval)
defer t.Stop()
for {
for _, u := range s.Drain() {
select {
case out <- agentsvc.KVEventsMessage(u):
case <-ctx.Done():
return
}
}
select {
case <-ctx.Done():
return
case <-t.C:
}
}
}
+103
View File
@@ -0,0 +1,103 @@
package agent
import (
"context"
"encoding/binary"
"fmt"
"log/slog"
"testing"
"time"
"github.com/go-zeromq/zmq4"
"github.com/vmihailenco/msgpack/v5"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
)
func kvAgent() *Agent {
return &Agent{log: slog.New(slog.DiscardHandler), kv: kvevents.NewMirror(),
kvSubs: map[string]*kvevents.Subscriber{}}
}
// TestAnEnginesEventsReachTheControlPlaneStream runs the agent's half end to
// end: a listener for the shard, a publisher standing in for vLLM connecting
// to the port the engine would be given, and the stream to the control plane
// carrying the result.
func TestAnEnginesEventsReachTheControlPlaneStream(t *testing.T) {
a := kvAgent()
spec := engines.ShardSpec{ReplicaID: "r1", ShardID: "r1-0", NNodes: 1,
Profile: engines.Profile{Engine: "vllm", KVEvents: true}}
a.listenKV(&spec, 0)
if spec.KVEventsPort == 0 {
t.Fatal("the shard should have been given a port to publish to")
}
defer a.stopKV(spec.ShardID, spec.ReplicaID)
pub := zmq4.NewPub(context.Background())
defer func() { _ = pub.Close() }()
if err := pub.Dial(fmt.Sprintf("tcp://127.0.0.1:%d", spec.KVEventsPort)); err != nil {
t.Fatal(err)
}
tokens := make([]int, 32)
for i := range tokens {
tokens[i] = 500 + i
}
hs := kvevents.BlockHashes(tokens, 16, 0)
payload, _ := msgpack.Marshal([]any{1.0, []any{map[string]any{
"type": "BlockStored", "block_hashes": []any{hs[0], hs[1]}, "parent_block_hash": nil,
"token_ids": tokens, "block_size": 16, "lora_id": nil, "medium": "GPU",
}}})
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
out := make(chan *llmgwv1.AgentMessage, 16)
go a.forwardKV(ctx, out)
deadline := time.After(5 * time.Second)
var seq uint64
for {
// Sent until it arrives: a subscription reaches the publisher after
// the connection, and what is published before it is dropped.
var s [8]byte
binary.BigEndian.PutUint64(s[:], seq)
seq++
_ = pub.Send(zmq4.NewMsgFrom([]byte("kv-events"), s[:], payload))
select {
case m := <-out:
kv := m.GetKvEvents()
if kv == nil || kv.GetReplicaId() != "r1" || kv.GetBlockSize() != 16 || len(kv.GetStored()) != 2 {
t.Fatalf("message %v", m)
}
return
case <-deadline:
t.Fatal("nothing reached the stream")
case <-time.After(50 * time.Millisecond):
}
}
}
func TestAShardWithoutKVEventsGetsNoListener(t *testing.T) {
a := kvAgent()
spec := engines.ShardSpec{ReplicaID: "r1", ShardID: "r1-0", NNodes: 1,
Profile: engines.Profile{Engine: "vllm"}}
a.listenKV(&spec, 0)
if spec.KVEventsPort != 0 || len(a.kvSubs) != 0 {
t.Fatal("kv_events is opt-in")
}
}
func TestARepeatedStartReusesTheShardsListener(t *testing.T) {
a := kvAgent()
spec := engines.ShardSpec{ReplicaID: "r1", ShardID: "r1-0", NNodes: 1,
Profile: engines.Profile{Engine: "vllm", KVEvents: true}}
a.listenKV(&spec, 0)
defer a.stopKV(spec.ShardID, spec.ReplicaID)
again := spec
again.KVEventsPort = 0
a.listenKV(&again, 0)
if again.KVEventsPort != spec.KVEventsPort || len(a.kvSubs) != 1 {
t.Fatal("a start command repeated must not open a second listener")
}
}