Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
e725adc2f0
33 files changed
+1607
-143
No files matched your search
+57
-4
@@ -23,6 +23,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
@@ -33,6 +35,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
@@ -419,6 +422,17 @@ func int32Slice(v []int) []int32 {
|
||||
return out
|
||||
}
|
||||
|
||||
func int32Ints(v []int32) []int {
|
||||
out := make([]int, len(v))
|
||||
for i, x := range v {
|
||||
out[i] = int(x)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tracerName names the spans the agent opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent"
|
||||
|
||||
// scan refreshes the node's hardware inventory.
|
||||
func (a *Agent) scan(ctx context.Context) error {
|
||||
// Carried from the agent's own configuration rather than duplicated into
|
||||
@@ -786,15 +800,23 @@ func (a *Agent) removeWeights(paths []string) {
|
||||
}
|
||||
|
||||
func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) {
|
||||
// The control plane's span that sent this, when it is tracing: what is
|
||||
// done here for a start or a stop goes into the same trace.
|
||||
tctx := obs.Extract(ctx, msg.GetTrace())
|
||||
switch m := msg.Msg.(type) {
|
||||
case *llmgwv1.ControlMessage_Start:
|
||||
go a.startShard(ctx, m.Start, out)
|
||||
go a.startShard(tctx, m.Start, out)
|
||||
case *llmgwv1.ControlMessage_Stop:
|
||||
go a.stopShard(ctx, m.Stop.GetShardId(), out)
|
||||
go a.stopShard(tctx, m.Stop.GetShardId(), out)
|
||||
case *llmgwv1.ControlMessage_Drain:
|
||||
if err := a.sup.Drain(m.Drain.GetShardId()); err != nil {
|
||||
_, span := obs.Tracer(tracerName).Start(tctx, "drain shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(attribute.String("llmgw.shard", m.Drain.GetShardId())))
|
||||
err := a.sup.Drain(m.Drain.GetShardId())
|
||||
if err != nil {
|
||||
a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err)
|
||||
}
|
||||
obs.End(span, err)
|
||||
case *llmgwv1.ControlMessage_RemoveWeights:
|
||||
a.removeWeights(m.RemoveWeights.GetPaths())
|
||||
case *llmgwv1.ControlMessage_Provision:
|
||||
@@ -927,6 +949,25 @@ func localAddr(ip netip.Addr) bool {
|
||||
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
|
||||
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
|
||||
|
||||
// This node's whole part of the start, from the command arriving to the
|
||||
// engine answering, beneath the control plane's span that sent it. The
|
||||
// steps — weights, launch, readiness — are its children.
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "start shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", a.cfg.NodeID),
|
||||
attribute.String("llmgw.replica", replicaID),
|
||||
attribute.String("llmgw.shard", shardID),
|
||||
attribute.Int("llmgw.rank", int(m.GetRank())),
|
||||
attribute.String("llmgw.model", m.GetProfile().GetServedModelName()),
|
||||
attribute.String("llmgw.profile", m.GetProfile().GetName()),
|
||||
attribute.String("llmgw.engine", m.GetProfile().GetEngine()),
|
||||
attribute.String("llmgw.gpu_class", m.GetProfile().GetGpuClass()),
|
||||
attribute.IntSlice("llmgw.gpus", int32Ints(m.GetGpuIndices())),
|
||||
))
|
||||
var failed error
|
||||
defer func() { obs.End(span, failed) }()
|
||||
|
||||
a.mu.Lock()
|
||||
a.shardReplica[shardID] = replicaID
|
||||
a.mu.Unlock()
|
||||
@@ -939,6 +980,9 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
}
|
||||
}
|
||||
report := func(st supervisor.State, errMsg string, tail []string) {
|
||||
if st == supervisor.Failed {
|
||||
failed = errors.New(errMsg)
|
||||
}
|
||||
send(&llmgwv1.ShardStatus{
|
||||
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
|
||||
})
|
||||
@@ -1109,9 +1153,18 @@ func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgw
|
||||
delete(a.shardReplica, shardID)
|
||||
a.mu.Unlock()
|
||||
|
||||
if err := a.sup.Stop(ctx, shardID); err != nil {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", a.cfg.NodeID),
|
||||
attribute.String("llmgw.replica", replicaID),
|
||||
attribute.String("llmgw.shard", shardID),
|
||||
))
|
||||
err := a.sup.Stop(ctx, shardID)
|
||||
if err != nil {
|
||||
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
|
||||
}
|
||||
obs.End(span, err)
|
||||
a.sup.Remove(shardID)
|
||||
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
|
||||
Reference in new issue
Block a user