Merge C24 KV-index routing from vLLM's cache events

Conflicts with the LoRA work: vLLM's arguments, the engine profile, router
metrics and the scheduler's request all keep both; agent.pb.go regenerated
from the merged proto; the two test fakes of vLLM given separate names.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 01:24:25 +02:00
commit 9e46ee83e4
43 files changed
+3498 -265

No files matched your search

+20
View File
@@ -35,6 +35,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
"gitea.mynodes.xyz/ilker/llmgw/internal/kvevents"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
)
@@ -120,6 +121,12 @@ type Agent struct {
// — eight shards starting together produce one download — and all eight
// are waiting, so all eight should say so.
fetchWatchers map[string][]func(weights.Progress)
// kv mirrors the KV caches of the engines that publish events, for the
// router's precise index; kvSubs are their listeners by shard, guarded
// by mu.
kv *kvevents.Mirror
kvSubs map[string]*kvevents.Subscriber
}
// connectHTTPClient is the HTTP client connect uses; an interface so tests can
@@ -162,6 +169,8 @@ func New(cfg Config, httpClient connectHTTPClient, log *slog.Logger) (*Agent, er
sensors: discovery.NewSensorSampler(),
wear: discovery.NewHealthSampler(),
serverList: loadControlPlanes(cfg.Servers, cfg.TLS.Enabled()),
kv: kvevents.NewMirror(),
kvSubs: map[string]*kvevents.Subscriber{},
}
a.prov = &provision.Installer{Root: cfg.EnginesDir, Log: log, OnStatus: a.provisionStatus}
return a, nil
@@ -280,6 +289,12 @@ func (a *Agent) recoverOrphans() {
a.mu.Lock()
a.shardReplica[o.ShardID] = o.ReplicaID
a.mu.Unlock()
// The engine is still publishing to the port the last agent
// listened on; listen there again.
if o.Spec.KVEventsPort > 0 {
spec := o.Spec
a.listenKV(&spec, o.Spec.KVEventsPort)
}
a.log.Info("adopted a running engine",
"shard", o.ShardID, "replica", o.ReplicaID, "pid", o.PID)
go a.settleAdopted(sh, o)
@@ -533,6 +548,7 @@ func (a *Agent) session(ctx context.Context, server string) error {
}()
defer wg.Wait()
go a.watchExits(ctx, out)
go a.forwardKV(ctx, out)
for {
msg, err := stream.Receive()
@@ -1063,8 +1079,10 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
}
}
a.listenKV(&spec, 0)
cmds, err := eng.Commands(spec)
if err != nil {
a.stopKV(shardID, replicaID)
report(supervisor.Failed, err.Error(), nil)
return
}
@@ -1077,6 +1095,7 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
report(sh.Status().State, "", nil)
return
}
a.stopKV(shardID, replicaID)
report(supervisor.Failed, err.Error(), sh.Logs(10))
return
}
@@ -1173,6 +1192,7 @@ func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgw
if err != nil {
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
}
a.stopKV(shardID, replicaID)
obs.End(span, err)
a.sup.Remove(shardID)