Merge C20 tracing beyond the router

Conflicts: config imports and docgen sources (both kept); agent.pb.go
regenerated with buf from the merged proto (MIG devices and trace context).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:05:29 +02:00
commit 801e2d51c0
33 files changed
+1608 -144

No files matched your search

+34 -7
View File
@@ -14,6 +14,8 @@ import (
"time"
"connectrpc.com/connect"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
@@ -22,6 +24,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// StatusSink receives shard transitions and recovers replica state from what
@@ -311,7 +314,20 @@ func (s *Service) Connect(
// unreachable in production while its unit tests, which call the reconciler
// directly, kept passing.
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
// A trace of its own, and only for a node that reported something: a
// node running nothing registering is not an event anyone looks for.
tctx, span := ctx, trace.SpanFromContext(context.Background()) // the no-op span
if len(shards) > 0 {
tctx, span = obs.Tracer(tracerName).Start(ctx, "adopt",
trace.WithNewRoot(),
trace.WithAttributes(
attribute.String("llmgw.node", nodeID),
attribute.Int("llmgw.shards", len(shards)),
))
}
defer span.End()
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
span.SetAttributes(attribute.Int("llmgw.unclaimed", len(unclaimed)))
for _, u := range unclaimed {
// The model or profile left the catalog while this was running. Nobody
// will ever route to it, and it is holding GPUs.
@@ -319,7 +335,8 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
select {
case c.send <- &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
Trace: obs.Inject(tctx),
}:
case <-ctx.Done():
return
@@ -715,27 +732,37 @@ type Leader interface{ IsLeader() bool }
// and what every deployment before leadership had.
func (s *Service) SetLeadership(l Leader) { s.leader = l }
// tracerName names the spans this package opens.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
// ErrNotConnected means no agent stream is open for that node.
var ErrNotConnected = errors.New("no agent connected")
// StartShard implements reconciler.Dispatcher.
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
//
// The command carries the trace of ctx, so the agent's spans for this start
// — the weights, the engine's launch, its readiness — are in the trace of
// the replica they are starting rather than in one of their own.
func (s *Service) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
Trace: obs.Inject(ctx),
})
}
// StopShard implements reconciler.Dispatcher.
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
func (s *Service) StopShard(ctx context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
Trace: obs.Inject(ctx),
})
}
// DrainShard implements reconciler.Dispatcher.
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
func (s *Service) DrainShard(ctx context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
Trace: obs.Inject(ctx),
})
}