Merge C24 KV-index routing from vLLM's cache events
Conflicts with the LoRA work: vLLM's arguments, the engine profile, router metrics and the scheduler's request all keep both; agent.pb.go regenerated from the merged proto; the two test fakes of vLLM given separate names. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
9e46ee83e4
43 files changed
+3498
-265
No files matched your search
@@ -35,6 +35,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
@@ -120,6 +121,12 @@ type Agent struct {
|
||||
// — eight shards starting together produce one download — and all eight
|
||||
// are waiting, so all eight should say so.
|
||||
fetchWatchers map[string][]func(weights.Progress)
|
||||
|
||||
// kv mirrors the KV caches of the engines that publish events, for the
|
||||
// router's precise index; kvSubs are their listeners by shard, guarded
|
||||
// by mu.
|
||||
kv *kvevents.Mirror
|
||||
kvSubs map[string]*kvevents.Subscriber
|
||||
}
|
||||
|
||||
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
|
||||
@@ -162,6 +169,8 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
|
||||
sensors: discovery.NewSensorSampler(),
|
||||
wear: discovery.NewHealthSampler(),
|
||||
serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
|
||||
kv: kvevents.NewMirror(),
|
||||
kvSubs: map[string]*kvevents.Subscriber{},
|
||||
}
|
||||
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
|
||||
return a, nil
|
||||
@@ -280,6 +289,12 @@ func (a *Agent) recoverOrphans() {
|
||||
a.mu.Lock()
|
||||
a.shardReplica[o.ShardID] = o.ReplicaID
|
||||
a.mu.Unlock()
|
||||
// The engine is still publishing to the port the last agent
|
||||
// listened on; listen there again.
|
||||
if o.Spec.KVEventsPort > 0 {
|
||||
spec := o.Spec
|
||||
a.listenKV(&spec, o.Spec.KVEventsPort)
|
||||
}
|
||||
a.log.Info("adopted a running engine",
|
||||
"shard", o.ShardID, "replica", o.ReplicaID, "pid", o.PID)
|
||||
go a.settleAdopted(sh, o)
|
||||
@@ -533,6 +548,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
|
||||
}()
|
||||
defer wg.Wait()
|
||||
go a.watchExits(ctx, out)
|
||||
go a.forwardKV(ctx, out)
|
||||
|
||||
for {
|
||||
msg, err := stream.Receive()
|
||||
@@ -1063,8 +1079,10 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
}
|
||||
}
|
||||
|
||||
a.listenKV(&spec, 0)
|
||||
cmds, err := eng.Commands(spec)
|
||||
if err != nil {
|
||||
a.stopKV(shardID, replicaID)
|
||||
report(supervisor.Failed, err.Error(), nil)
|
||||
return
|
||||
}
|
||||
@@ -1077,6 +1095,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
report(sh.Status().State, "", nil)
|
||||
return
|
||||
}
|
||||
a.stopKV(shardID, replicaID)
|
||||
report(supervisor.Failed, err.Error(), sh.Logs(10))
|
||||
return
|
||||
}
|
||||
@@ -1173,6 +1192,7 @@ func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgw
|
||||
if err != nil {
|
||||
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
|
||||
}
|
||||
a.stopKV(shardID, replicaID)
|
||||
obs.End(span, err)
|
||||
a.sup.Remove(shardID)
|
||||
|
||||
|
||||
@@ -44,6 +44,9 @@ type Profile struct {
|
||||
// Adapters are LoRA adapters to load beside the model, in order: an
|
||||
// adapter's index is the id requests apply it by.
|
||||
Adapters []Adapter
|
||||
// KVEvents has the engine publish its KV cache events to the agent,
|
||||
// for the router's precise index. vLLM only.
|
||||
KVEvents bool `json:"kv_events,omitempty"`
|
||||
}
|
||||
|
||||
// ShardSpec is one shard of one replica on this node.
|
||||
@@ -90,6 +93,11 @@ type ShardSpec struct {
|
||||
// and accounting. Given to the engine in its environment rather than on
|
||||
// its command line, which every user on the node can read.
|
||||
APIKey string `json:"api_key,omitempty"`
|
||||
// KVEventsPort is the loopback port the agent receives this shard's KV
|
||||
// cache events on, chosen by the agent; zero when it publishes none.
|
||||
// Kept with the spec so an agent that restarts listens where the engine
|
||||
// it adopts is still sending.
|
||||
KVEventsPort int `json:"kv_events_port,omitempty"`
|
||||
// Threads is the CPU cores placement reserved for a shard with no GPU,
|
||||
// and so what its engine computes on. Zero leaves it to the engine's own
|
||||
// configuration.
|
||||
|
||||
@@ -0,0 +1,47 @@
|
||||
package engines
|
||||
|
||||
import (
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestAVLLMShardWithKVEventsPublishesToTheAgentOnLoopback(t *testing.T) {
|
||||
s := singleNodeSpec()
|
||||
s.Profile.KVEvents = true
|
||||
s.KVEventsPort = 5999
|
||||
s.Profile.Args = []string{"--max-model-len", "8192"}
|
||||
cmd := buildOne(t, &VLLM{}, s)
|
||||
|
||||
i := slices.Index(cmd.Args, "--kv-events-config")
|
||||
if i < 0 || !strings.Contains(cmd.Args[i+1], `"endpoint":"tcp://127.0.0.1:5999"`) {
|
||||
t.Fatalf("args %v should name the agent's loopback port", cmd.Args)
|
||||
}
|
||||
if j := slices.Index(cmd.Args, "--prefix-caching-hash-algo"); j < 0 || cmd.Args[j+1] != "sha256_cbor" {
|
||||
t.Fatalf("args %v should pin the hashing the router computes", cmd.Args)
|
||||
}
|
||||
if slices.Index(cmd.Args, "--max-model-len") < i {
|
||||
t.Fatal("profile arguments come last, so an operator can still override")
|
||||
}
|
||||
if !slices.Contains(cmd.Env, "PYTHONHASHSEED=1729") {
|
||||
t.Fatalf("env %v should fix the hash seed", cmd.Env)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoKVEventsWithoutAPortOrOffTheLeader(t *testing.T) {
|
||||
s := singleNodeSpec()
|
||||
s.Profile.KVEvents = true // but the agent could not listen
|
||||
if cmd := buildOne(t, &VLLM{}, s); slices.Contains(cmd.Args, "--kv-events-config") {
|
||||
t.Fatal("an engine told to publish where nobody listens only fills a queue")
|
||||
}
|
||||
w := multiNodeSpec(1)
|
||||
w.Profile.KVEvents, w.KVEventsPort = true, 5999
|
||||
if cmd := buildEngine(t, &VLLM{}, w); slices.Contains(cmd.Args, "--kv-events-config") {
|
||||
t.Fatal("a headless worker has no scheduler to publish from")
|
||||
}
|
||||
off := singleNodeSpec()
|
||||
off.KVEventsPort = 5999
|
||||
if cmd := buildOne(t, &VLLM{}, off); slices.Contains(cmd.Args, "--kv-events-config") {
|
||||
t.Fatal("kv_events is opt-in")
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,11 @@ package engines
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"maps"
|
||||
"strconv"
|
||||
"sync/atomic"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
)
|
||||
|
||||
// VLLMConfig locates the vLLM installation and picks how it is launched.
|
||||
@@ -161,6 +164,14 @@ func (v *VLLM) serverCommand(s ShardSpec) (Command, error) {
|
||||
}
|
||||
}
|
||||
|
||||
// The KV cache events, for the router's precise index: published to the
|
||||
// agent on loopback, hashed the way the router hashes. Only the leader
|
||||
// has a scheduler to publish from.
|
||||
if p.KVEvents && s.KVEventsPort > 0 && s.Leader() {
|
||||
args = append(args, kvevents.VLLMArgs(fmt.Sprintf("tcp://127.0.0.1:%d", s.KVEventsPort))...)
|
||||
maps.Copy(env, kvevents.VLLMEnv())
|
||||
}
|
||||
|
||||
// Profile arguments last, so an operator can override anything above.
|
||||
args = append(args, p.Args...)
|
||||
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
)
|
||||
|
||||
// kvFlushInterval is how often KV cache changes go up the stream. Often
|
||||
// enough that a prefix cached by one request is known before the next few
|
||||
// arrive; seldom enough that a busy engine's churn is sent as its net effect.
|
||||
const kvFlushInterval = 250 * time.Millisecond
|
||||
|
||||
// listenKV starts receiving a shard's KV cache events, when its profile asks
|
||||
// for them and this rank publishes any, and records the port in the spec for
|
||||
// the engine's arguments. port is the one to reuse, or 0 for any.
|
||||
//
|
||||
// A failure is logged and the engine started without events: the router then
|
||||
// routes the replica by tier 1, which is what it did before, rather than the
|
||||
// replica not starting at all for want of an optimisation.
|
||||
func (a *Agent) listenKV(spec *engines.ShardSpec, port int) {
|
||||
if !spec.Profile.KVEvents || spec.Profile.Engine != "vllm" || !spec.Leader() {
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
if sub, ok := a.kvSubs[spec.ShardID]; ok {
|
||||
a.mu.Unlock()
|
||||
// A repeated start command: the shard already has its listener.
|
||||
spec.KVEventsPort = sub.Port()
|
||||
return
|
||||
}
|
||||
a.mu.Unlock()
|
||||
|
||||
sub, err := kvevents.Listen(spec.ReplicaID, port, a.kv, a.log)
|
||||
if err != nil {
|
||||
a.log.Warn("cannot receive kv cache events; the router will route this replica without them",
|
||||
"shard", spec.ShardID, "replica", spec.ReplicaID, "err", err)
|
||||
return
|
||||
}
|
||||
a.mu.Lock()
|
||||
a.kvSubs[spec.ShardID] = sub
|
||||
a.mu.Unlock()
|
||||
spec.KVEventsPort = sub.Port()
|
||||
}
|
||||
|
||||
// stopKV stops a shard's listener and tells the control plane its replica
|
||||
// holds nothing any more.
|
||||
func (a *Agent) stopKV(shardID, replicaID string) {
|
||||
a.mu.Lock()
|
||||
sub, ok := a.kvSubs[shardID]
|
||||
delete(a.kvSubs, shardID)
|
||||
a.mu.Unlock()
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
sub.Close()
|
||||
a.kv.Remove(replicaID)
|
||||
}
|
||||
|
||||
// forwardKV sends KV cache changes up one session's stream until it ends,
|
||||
// beginning with a full copy of every replica: the control plane at the other
|
||||
// end may have just started, or just taken over, and know nothing.
|
||||
func (a *Agent) forwardKV(ctx context.Context, out chan<- *llmgwv1.AgentMessage) {
|
||||
s := a.kv.Open()
|
||||
defer s.Close()
|
||||
t := time.NewTicker(kvFlushInterval)
|
||||
defer t.Stop()
|
||||
for {
|
||||
for _, u := range s.Drain() {
|
||||
select {
|
||||
case out <- agentsvc.KVEventsMessage(u):
|
||||
case <-ctx.Done():
|
||||
return
|
||||
}
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-t.C:
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,103 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"log/slog"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/go-zeromq/zmq4"
|
||||
"github.com/vmihailenco/msgpack/v5"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
)
|
||||
|
||||
func kvAgent() *Agent {
|
||||
return &Agent{log: slog.New(slog.DiscardHandler), kv: kvevents.NewMirror(),
|
||||
kvSubs: map[string]*kvevents.Subscriber{}}
|
||||
}
|
||||
|
||||
// TestAnEnginesEventsReachTheControlPlaneStream runs the agent's half end to
|
||||
// end: a listener for the shard, a publisher standing in for vLLM connecting
|
||||
// to the port the engine would be given, and the stream to the control plane
|
||||
// carrying the result.
|
||||
func TestAnEnginesEventsReachTheControlPlaneStream(t *testing.T) {
|
||||
a := kvAgent()
|
||||
spec := engines.ShardSpec{ReplicaID: "r1", ShardID: "r1-0", NNodes: 1,
|
||||
Profile: engines.Profile{Engine: "vllm", KVEvents: true}}
|
||||
a.listenKV(&spec, 0)
|
||||
if spec.KVEventsPort == 0 {
|
||||
t.Fatal("the shard should have been given a port to publish to")
|
||||
}
|
||||
defer a.stopKV(spec.ShardID, spec.ReplicaID)
|
||||
|
||||
pub := zmq4.NewPub(context.Background())
|
||||
defer func() { _ = pub.Close() }()
|
||||
if err := pub.Dial(fmt.Sprintf("tcp://127.0.0.1:%d", spec.KVEventsPort)); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
tokens := make([]int, 32)
|
||||
for i := range tokens {
|
||||
tokens[i] = 500 + i
|
||||
}
|
||||
hs := kvevents.BlockHashes(tokens, 16, 0)
|
||||
payload, _ := msgpack.Marshal([]any{1.0, []any{map[string]any{
|
||||
"type": "BlockStored", "block_hashes": []any{hs[0], hs[1]}, "parent_block_hash": nil,
|
||||
"token_ids": tokens, "block_size": 16, "lora_id": nil, "medium": "GPU",
|
||||
}}})
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
out := make(chan *llmgwv1.AgentMessage, 16)
|
||||
go a.forwardKV(ctx, out)
|
||||
|
||||
deadline := time.After(5 * time.Second)
|
||||
var seq uint64
|
||||
for {
|
||||
// Sent until it arrives: a subscription reaches the publisher after
|
||||
// the connection, and what is published before it is dropped.
|
||||
var s [8]byte
|
||||
binary.BigEndian.PutUint64(s[:], seq)
|
||||
seq++
|
||||
_ = pub.Send(zmq4.NewMsgFrom([]byte("kv-events"), s[:], payload))
|
||||
select {
|
||||
case m := <-out:
|
||||
kv := m.GetKvEvents()
|
||||
if kv == nil || kv.GetReplicaId() != "r1" || kv.GetBlockSize() != 16 || len(kv.GetStored()) != 2 {
|
||||
t.Fatalf("message %v", m)
|
||||
}
|
||||
return
|
||||
case <-deadline:
|
||||
t.Fatal("nothing reached the stream")
|
||||
case <-time.After(50 * time.Millisecond):
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestAShardWithoutKVEventsGetsNoListener(t *testing.T) {
|
||||
a := kvAgent()
|
||||
spec := engines.ShardSpec{ReplicaID: "r1", ShardID: "r1-0", NNodes: 1,
|
||||
Profile: engines.Profile{Engine: "vllm"}}
|
||||
a.listenKV(&spec, 0)
|
||||
if spec.KVEventsPort != 0 || len(a.kvSubs) != 0 {
|
||||
t.Fatal("kv_events is opt-in")
|
||||
}
|
||||
}
|
||||
|
||||
func TestARepeatedStartReusesTheShardsListener(t *testing.T) {
|
||||
a := kvAgent()
|
||||
spec := engines.ShardSpec{ReplicaID: "r1", ShardID: "r1-0", NNodes: 1,
|
||||
Profile: engines.Profile{Engine: "vllm", KVEvents: true}}
|
||||
a.listenKV(&spec, 0)
|
||||
defer a.stopKV(spec.ShardID, spec.ReplicaID)
|
||||
again := spec
|
||||
again.KVEventsPort = 0
|
||||
a.listenKV(&again, 0)
|
||||
if again.KVEventsPort != spec.KVEventsPort || len(a.kvSubs) != 1 {
|
||||
t.Fatal("a start command repeated must not open a second listener")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user