Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node

A reconcile pass that does something is a trace, with placement (where it
went, or every candidate turned down), scale-up/down, drain, stop, eviction,
preemption, reaping and restarts beneath it. A replica's start is a span open
from placement until it serves; the start command carries its context in a
new ControlMessage.trace field, and the agent continues the trace through
the weights fetch, engine launch, rendezvous and readiness wait. Stops,
drains, adoption and engine installs are traced on the node, model jobs are
a trace with a span per step linked to the replica's start, and agent.yaml
takes the same tracing block as server.yaml. Off, every span is the no-op one
and commands carry nothing.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:03:17 +02:00
1 parent e59ff52932
commit e725adc2f0
33 files changed
+1607 -143

No files matched your search

+34 -7
View File
@@ -14,6 +14,8 @@ import (
"time"
"connectrpc.com/connect"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
@@ -22,6 +24,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// StatusSink receives shard transitions and recovers replica state from what
@@ -311,7 +314,20 @@ func (s *Service) Connect(
// unreachable in production while its unit tests, which call the reconciler
// directly, kept passing.
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
// A trace of its own, and only for a node that reported something: a
// node running nothing registering is not an event anyone looks for.
tctx, span := ctx, trace.SpanFromContext(context.Background()) // the no-op span
if len(shards) > 0 {
tctx, span = obs.Tracer(tracerName).Start(ctx, "adopt",
trace.WithNewRoot(),
trace.WithAttributes(
attribute.String("llmgw.node", nodeID),
attribute.Int("llmgw.shards", len(shards)),
))
}
defer span.End()
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
span.SetAttributes(attribute.Int("llmgw.unclaimed", len(unclaimed)))
for _, u := range unclaimed {
// The model or profile left the catalog while this was running. Nobody
// will ever route to it, and it is holding GPUs.
@@ -319,7 +335,8 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
select {
case c.send <- &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
Trace: obs.Inject(tctx),
}:
case <-ctx.Done():
return
@@ -715,27 +732,37 @@ type Leader interface{ IsLeader() bool }
// and what every deployment before leadership had.
func (s *Service) SetLeadership(l Leader) { s.leader = l }
// tracerName names the spans this package opens.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
// ErrNotConnected means no agent stream is open for that node.
var ErrNotConnected = errors.New("no agent connected")
// StartShard implements reconciler.Dispatcher.
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
//
// The command carries the trace of ctx, so the agent's spans for this start
// — the weights, the engine's launch, its readiness — are in the trace of
// the replica they are starting rather than in one of their own.
func (s *Service) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
Trace: obs.Inject(ctx),
})
}
// StopShard implements reconciler.Dispatcher.
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
func (s *Service) StopShard(ctx context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
Trace: obs.Inject(ctx),
})
}
// DrainShard implements reconciler.Dispatcher.
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
func (s *Service) DrainShard(ctx context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
Trace: obs.Inject(ctx),
})
}