Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
e725adc2f0
33 files changed
+1607
-143
No files matched your search
@@ -14,6 +14,8 @@ import (
|
||||
"time"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
|
||||
@@ -22,6 +24,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// StatusSink receives shard transitions and recovers replica state from what
|
||||
@@ -311,7 +314,20 @@ func (s *Service) Connect(
|
||||
// unreachable in production while its unit tests, which call the reconciler
|
||||
// directly, kept passing.
|
||||
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
|
||||
// A trace of its own, and only for a node that reported something: a
|
||||
// node running nothing registering is not an event anyone looks for.
|
||||
tctx, span := ctx, trace.SpanFromContext(context.Background()) // the no-op span
|
||||
if len(shards) > 0 {
|
||||
tctx, span = obs.Tracer(tracerName).Start(ctx, "adopt",
|
||||
trace.WithNewRoot(),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", nodeID),
|
||||
attribute.Int("llmgw.shards", len(shards)),
|
||||
))
|
||||
}
|
||||
defer span.End()
|
||||
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
|
||||
span.SetAttributes(attribute.Int("llmgw.unclaimed", len(unclaimed)))
|
||||
for _, u := range unclaimed {
|
||||
// The model or profile left the catalog while this was running. Nobody
|
||||
// will ever route to it, and it is holding GPUs.
|
||||
@@ -319,7 +335,8 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
|
||||
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
|
||||
select {
|
||||
case c.send <- &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
|
||||
Trace: obs.Inject(tctx),
|
||||
}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -715,27 +732,37 @@ type Leader interface{ IsLeader() bool }
|
||||
// and what every deployment before leadership had.
|
||||
func (s *Service) SetLeadership(l Leader) { s.leader = l }
|
||||
|
||||
// tracerName names the spans this package opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
|
||||
// ErrNotConnected means no agent stream is open for that node.
|
||||
var ErrNotConnected = errors.New("no agent connected")
|
||||
|
||||
// StartShard implements reconciler.Dispatcher.
|
||||
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
|
||||
//
|
||||
// The command carries the trace of ctx, so the agent's spans for this start
|
||||
// — the weights, the engine's launch, its readiness — are in the trace of
|
||||
// the replica they are starting rather than in one of their own.
|
||||
func (s *Service) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
|
||||
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
// StopShard implements reconciler.Dispatcher.
|
||||
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
|
||||
func (s *Service) StopShard(ctx context.Context, nodeID, shardID string) error {
|
||||
return s.send(nodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
// DrainShard implements reconciler.Dispatcher.
|
||||
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
|
||||
func (s *Service) DrainShard(ctx context.Context, nodeID, shardID string) error {
|
||||
return s.send(nodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
Reference in new issue
Block a user