Merge C24 KV-index routing from vLLM's cache events
Conflicts with the LoRA work: vLLM's arguments, the engine profile, router metrics and the scheduler's request all keep both; agent.pb.go regenerated from the merged proto; the two test fakes of vLLM given separate names. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
9e46ee83e4
43 files changed
+3498
-265
No files matched your search
@@ -198,6 +198,7 @@ func StartShardFromCommand(cmd reconciler.ShardCommand) *llmgwv1.StartShard {
|
||||
Args: p.Args, Env: p.Env,
|
||||
ModelPath: p.ModelPath, ServedModelName: p.ServedModelName,
|
||||
Adapters: adaptersToWire(p.Adapters),
|
||||
KvEvents: p.KVEvents,
|
||||
},
|
||||
StartupTimeoutMs: cmd.StartupTimeout.Milliseconds(),
|
||||
Pool: cmd.Pool,
|
||||
@@ -234,6 +235,7 @@ func ShardSpecFromStart(m *llmgwv1.StartShard) (engines.ShardSpec, time.Duration
|
||||
Args: p.GetArgs(), Env: p.GetEnv(),
|
||||
ModelPath: p.GetModelPath(), ServedModelName: p.GetServedModelName(),
|
||||
Adapters: adaptersFromWire(p.GetAdapters()),
|
||||
KVEvents: p.GetKvEvents(),
|
||||
},
|
||||
}
|
||||
return spec, time.Duration(m.GetStartupTimeoutMs()) * time.Millisecond
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"sync"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
)
|
||||
|
||||
// SetKVSink has the KV cache events agents forward delivered to fn — the
|
||||
// router's precise index, in a process that serves both. Unset, they are
|
||||
// dropped, which is right for a control plane with no router beside it.
|
||||
func (s *Service) SetKVSink(fn func(kvevents.Update)) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
s.kv = fn
|
||||
}
|
||||
|
||||
func (s *Service) kvSink() func(kvevents.Update) {
|
||||
s.mu.RLock()
|
||||
defer s.mu.RUnlock()
|
||||
return s.kv
|
||||
}
|
||||
|
||||
// kvState is what one stream has said about KV caches: which replicas the
|
||||
// node runs, and so may describe, and which it has described.
|
||||
type kvState struct {
|
||||
mu sync.Mutex
|
||||
owned map[string]bool
|
||||
described map[string]bool
|
||||
}
|
||||
|
||||
// own records a replica the node runs a shard of.
|
||||
func (c *conn) own(replicaID string) {
|
||||
if replicaID == "" {
|
||||
return
|
||||
}
|
||||
c.kv.mu.Lock()
|
||||
defer c.kv.mu.Unlock()
|
||||
if c.kv.owned == nil {
|
||||
c.kv.owned = map[string]bool{}
|
||||
}
|
||||
c.kv.owned[replicaID] = true
|
||||
}
|
||||
|
||||
// kvEvents passes one replica's update on.
|
||||
//
|
||||
// Only for a replica the node has said it runs. A node describing another's
|
||||
// cache — misconfigured, or worse — could otherwise draw that replica's
|
||||
// traffic, or starve it, from anywhere in the cluster.
|
||||
func (s *Service) kvEvents(c *conn, m *llmgwv1.KVEvents) {
|
||||
sink := s.kvSink()
|
||||
if sink == nil {
|
||||
return
|
||||
}
|
||||
id := m.GetReplicaId()
|
||||
c.kv.mu.Lock()
|
||||
ok := c.kv.owned[id]
|
||||
if ok {
|
||||
if c.kv.described == nil {
|
||||
c.kv.described = map[string]bool{}
|
||||
}
|
||||
c.kv.described[id] = true
|
||||
}
|
||||
c.kv.mu.Unlock()
|
||||
if !ok {
|
||||
s.log.Debug("ignoring kv cache events for a replica the node does not run",
|
||||
"node", c.nodeID, "replica", id)
|
||||
return
|
||||
}
|
||||
sink(kvevents.Update{
|
||||
Replica: id, BlockSize: int(m.GetBlockSize()), Reset: m.GetClearFirst(),
|
||||
Stored: m.GetStored(), Removed: m.GetRemoved(), Problem: m.GetProblem(),
|
||||
})
|
||||
}
|
||||
|
||||
// kvGone empties what a closed stream described. Events stop arriving with
|
||||
// it, evictions among them, and an index that kept the blocks would steer
|
||||
// requests to caches that may have moved on; the next stream starts with a
|
||||
// full copy anyway.
|
||||
func (s *Service) kvGone(c *conn) {
|
||||
sink := s.kvSink()
|
||||
if sink == nil {
|
||||
return
|
||||
}
|
||||
c.kv.mu.Lock()
|
||||
ids := make([]string, 0, len(c.kv.described))
|
||||
for id := range c.kv.described {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
c.kv.mu.Unlock()
|
||||
for _, id := range ids {
|
||||
sink(kvevents.Update{Replica: id, Reset: true})
|
||||
}
|
||||
}
|
||||
|
||||
// KVEventsMessage wraps an update for the stream, as the agent sends it.
|
||||
func KVEventsMessage(u kvevents.Update) *llmgwv1.AgentMessage {
|
||||
return &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_KvEvents{KvEvents: &llmgwv1.KVEvents{
|
||||
ReplicaId: u.Replica, BlockSize: int32(u.BlockSize), ClearFirst: u.Reset,
|
||||
Stored: u.Stored, Removed: u.Removed, Problem: u.Problem,
|
||||
}}}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package agentsvc
|
||||
|
||||
import (
|
||||
"log/slog"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
)
|
||||
|
||||
func TestKVEventsAreTakenOnlyForTheNodesOwnReplicas(t *testing.T) {
|
||||
s := &Service{log: slog.New(slog.DiscardHandler)}
|
||||
var got []kvevents.Update
|
||||
s.SetKVSink(func(u kvevents.Update) { got = append(got, u) })
|
||||
c := &conn{nodeID: "n1"}
|
||||
c.own("r1")
|
||||
|
||||
s.kvEvents(c, KVEventsMessage(kvevents.Update{Replica: "r1", BlockSize: 16, Reset: true, Stored: []uint64{7}}).GetKvEvents())
|
||||
s.kvEvents(c, KVEventsMessage(kvevents.Update{Replica: "r-elsewhere", Stored: []uint64{8}}).GetKvEvents())
|
||||
if len(got) != 1 || got[0].Replica != "r1" || !got[0].Reset || got[0].Stored[0] != 7 || got[0].BlockSize != 16 {
|
||||
t.Fatalf("got %+v; a node may describe its own replicas' caches and no others", got)
|
||||
}
|
||||
|
||||
// The stream closing empties what it described: no more evictions will
|
||||
// arrive by it.
|
||||
s.kvGone(c)
|
||||
if len(got) != 2 || got[1].Replica != "r1" || !got[1].Reset || len(got[1].Stored) != 0 {
|
||||
t.Fatalf("got %+v", got)
|
||||
}
|
||||
}
|
||||
@@ -24,6 +24,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
@@ -58,6 +59,8 @@ type Service struct {
|
||||
|
||||
mu sync.RWMutex
|
||||
conns map[string]*conn
|
||||
// kv receives the KV cache events agents forward; see SetKVSink.
|
||||
kv func(kvevents.Update)
|
||||
|
||||
// enginesFor is what a node is told to install, set once at start: the
|
||||
// cluster's engines, with any override that picks the node laid over.
|
||||
@@ -126,6 +129,8 @@ type conn struct {
|
||||
send chan *llmgwv1.ControlMessage
|
||||
closed chan struct{}
|
||||
once sync.Once
|
||||
// kv is what this stream has said about KV caches.
|
||||
kv kvState
|
||||
}
|
||||
|
||||
func (c *conn) close() { c.once.Do(func() { close(c.closed) }) }
|
||||
@@ -245,6 +250,10 @@ func (s *Service) Connect(
|
||||
|
||||
c := s.attach(ctx, nodeID)
|
||||
defer s.detach(nodeID, c)
|
||||
for _, rs := range reg.GetRunningShards() {
|
||||
c.own(rs.GetReplicaId())
|
||||
}
|
||||
defer s.kvGone(c)
|
||||
|
||||
if err := stream.Send(&llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Ack{Ack: &llmgwv1.Ack{MessageId: "register", Ok: true}},
|
||||
@@ -304,6 +313,13 @@ func (s *Service) Connect(
|
||||
s.log.Warn("agent stream failed", "node", nodeID, "err", err)
|
||||
return nil
|
||||
}
|
||||
if kv := msg.GetKvEvents(); kv != nil {
|
||||
s.kvEvents(c, kv)
|
||||
continue
|
||||
}
|
||||
if sh := msg.GetShard(); sh != nil {
|
||||
c.own(sh.GetReplicaId())
|
||||
}
|
||||
s.handle(nodeID, msg)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -250,6 +250,20 @@ type Profile struct {
|
||||
// StartupTimeout bounds model loading. A 70B model across two nodes takes
|
||||
// tens of minutes, so this is per profile rather than global.
|
||||
StartupTimeout time.Duration `json:"startup_timeout,omitempty" koanf:"startup_timeout"`
|
||||
|
||||
// KVEvents has vLLM publish what its prefix cache holds, and the router
|
||||
// route this model's requests by it: to the replica holding the most of
|
||||
// a prompt's leading blocks (plan §7.5 tier 2). vLLM only.
|
||||
//
|
||||
// Off by default, because it costs something on every request: the
|
||||
// router asks a replica to tokenize the prompt before placing it, a
|
||||
// round trip and the chat template run twice. It pays where prompts
|
||||
// share long prefixes that the approximate tree guesses wrong — many
|
||||
// replicas, conversations that move between them, evictions. The engine
|
||||
// is started with sha256_cbor block hashing and a fixed PYTHONHASHSEED,
|
||||
// which the router's hashing assumes; arguments overriding either leave
|
||||
// the index empty, and the agent says so in its log.
|
||||
KVEvents bool `json:"kv_events,omitempty" koanf:"kv_events"`
|
||||
}
|
||||
|
||||
// CPUOnly reports whether this profile needs no GPUs.
|
||||
@@ -739,6 +753,10 @@ func (m Model) validate() error {
|
||||
if err := engines.UnsafeArgs(p.Engine, p.Args); err != nil {
|
||||
return fmt.Errorf("model %q profile %q: %w", m.Name, p.Name, err)
|
||||
}
|
||||
if p.KVEvents && p.Engine != "vllm" {
|
||||
return fmt.Errorf("model %q profile %q: kv_events is vLLM's; engine %q publishes no cache events",
|
||||
m.Name, p.Name, p.Engine)
|
||||
}
|
||||
|
||||
if p.PowerWatts < 0 {
|
||||
return fmt.Errorf("model %q profile %q: power_watts %v is negative", m.Name, p.Name, p.PowerWatts)
|
||||
@@ -906,11 +924,24 @@ func (m Model) EngineProfile(name string) (engines.Profile, bool) {
|
||||
ModelPath: m.Source,
|
||||
ServedModelName: m.Name,
|
||||
Adapters: m.engineAdapters(),
|
||||
KVEvents: p.KVEvents,
|
||||
}.WithRuntimeLoRA(m.MaxLoadedAdapters), true
|
||||
}
|
||||
return engines.Profile{}, false
|
||||
}
|
||||
|
||||
// KVIndex reports whether the router should route this model by the KV
|
||||
// cache its replicas report: whether any profile publishes events. A replica
|
||||
// of another profile is simply one the index has not heard from.
|
||||
func (m Model) KVIndex() bool {
|
||||
for _, p := range m.Profiles {
|
||||
if p.KVEvents {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// ProfileByName returns one of a model's profiles.
|
||||
func (m Model) ProfileByName(name string) (Profile, bool) {
|
||||
for _, p := range m.Profiles {
|
||||
|
||||
@@ -0,0 +1,40 @@
|
||||
package catalog
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestKVEventsReachTheEngineAndTheRouter(t *testing.T) {
|
||||
c, err := New([]Model{model("m", Profile{Name: "p", GPUClass: "h100-80g", TP: 1, KVEvents: true})})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m, _ := c.Resolve("m")
|
||||
if !m.KVIndex() {
|
||||
t.Fatal("a model with a profile publishing events should be routed by them")
|
||||
}
|
||||
if ep, _ := m.EngineProfile("p"); !ep.KVEvents {
|
||||
t.Fatal("the engine has to be told to publish")
|
||||
}
|
||||
plain, _ := New([]Model{model("n")})
|
||||
if n, _ := plain.Resolve("n"); n.KVIndex() {
|
||||
t.Fatal("off by default")
|
||||
}
|
||||
}
|
||||
|
||||
func TestKVEventsAreVLLMsOnly(t *testing.T) {
|
||||
_, err := New([]Model{model("m", Profile{Name: "p", Engine: "llamacpp", KVEvents: true})})
|
||||
if err == nil || !strings.Contains(err.Error(), "kv_events") {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTurningKVEventsOnMarksRunningReplicasStale(t *testing.T) {
|
||||
p := Profile{Name: "p", GPUClass: "h100-80g", TP: 1}
|
||||
q := p
|
||||
q.KVEvents = true
|
||||
if p.Fingerprint() == q.Fingerprint() {
|
||||
t.Fatal("replicas started without events do not publish until replaced, and should be shown as stale")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user