Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node

A reconcile pass that does something is a trace, with placement (where it
went, or every candidate turned down), scale-up/down, drain, stop, eviction,
preemption, reaping and restarts beneath it. A replica's start is a span open
from placement until it serves; the start command carries its context in a
new ControlMessage.trace field, and the agent continues the trace through
the weights fetch, engine launch, rendezvous and readiness wait. Stops,
drains, adoption and engine installs are traced on the node, model jobs are
a trace with a span per step linked to the replica's start, and agent.yaml
takes the same tracing block as server.yaml. Off, every span is the no-op one
and commands carry nothing.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:03:17 +02:00
1 parent e59ff52932
commit e725adc2f0
33 files changed
+1607 -143

No files matched your search

+75 -6
View File
@@ -18,9 +18,19 @@ import (
"syscall"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// tracerName names the spans the supervisor opens. There is no restart span
// here because the supervisor never restarts an engine on its own: a restart
// is the control plane's decision, traced there, and reaches a node as a stop
// and a start.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
// Config configures the supervisor.
type Config struct {
// WorkDir holds pid files and per-shard scratch space.
@@ -195,7 +205,25 @@ var ErrAlreadyRunning = errors.New("shard is already running")
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
// rather than a second process, because the control plane retries commands
// (plan §5.6).
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (*Shard, error) {
//
// Traced as "launch engine": from the first process to the engine's being
// started, which for a multi-node worker includes waiting for its leader's
// rendezvous point. Loading the model is the readiness wait's, not this.
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (_ *Shard, err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "launch engine", trace.WithAttributes(
attribute.String("llmgw.shard", id),
attribute.String("llmgw.engine", spec.Profile.Engine),
attribute.Int("llmgw.processes", len(cmds)),
))
defer func() {
if errors.Is(err, ErrAlreadyRunning) {
// A repeated command, which is not this start failing.
span.SetAttributes(attribute.Bool("llmgw.already_running", true))
span.End()
return
}
obs.End(span, err)
}()
if len(cmds) == 0 {
return nil, fmt.Errorf("shard %s: no commands to run", id)
}
@@ -301,6 +329,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
sh.pid = c.Process.Pid
sh.state = Starting
sh.mu.Unlock()
trace.SpanFromContext(ctx).SetAttributes(attribute.Int("llmgw.pid", c.Process.Pid))
if err := s.writeState(sh); err != nil {
s.log.Warn("could not write pid file; adoption after a restart will not work",
@@ -337,7 +366,13 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
}
// waitForPort blocks until addr accepts a connection.
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
//
// Traced: a worker's start that took ten minutes because its leader was
// still loading reads as exactly that.
func waitForPort(ctx context.Context, addr string, timeout time.Duration) (err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for rendezvous",
trace.WithAttributes(attribute.String("llmgw.rendezvous", addr)))
defer func() { obs.End(span, err) }()
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
@@ -416,7 +451,26 @@ func (s *Supervisor) wait(sh *Shard) {
// The timeout comes from the profile because it is model-dependent: loading a
// 70B model across two nodes legitimately takes tens of minutes, while a small
// model failing to load for ten minutes means something is wrong.
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
//
// Traced as "wait for engine": the model loading, which is usually most of a
// start. The span says how many times the engine was asked and what it last
// answered, so one that never came up says why.
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) (err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
attribute.String("llmgw.shard", sh.ID),
attribute.String("llmgw.readiness", "health check"),
attribute.String("llmgw.timeout", timeout.String()),
))
polls := 0
var answer error
defer func() {
span.SetAttributes(attribute.Int("llmgw.polls", polls))
if answer != nil && err != nil {
span.SetAttributes(attribute.String("llmgw.last_answer", answer.Error()))
}
obs.End(span, err)
}()
ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
@@ -432,7 +486,8 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
last := strings.Join(sh.Logs(10), "\n")
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
case <-t.C:
if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
polls++
if answer = s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); answer != nil {
continue
}
sh.setState(Ready, nil)
@@ -449,7 +504,13 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
// leader answering, which only happens once every worker has joined. Polling a
// worker's port instead would time out and fail a replica that is coming up
// perfectly well.
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) error {
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) (err error) {
_, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
attribute.String("llmgw.shard", sh.ID),
attribute.String("llmgw.readiness", "stays alive"),
attribute.String("llmgw.timeout", s.cfg.AliveGrace.String()),
))
defer func() { obs.End(span, err) }()
sh.setState(Loading, nil)
select {
@@ -483,7 +544,14 @@ func (s *Supervisor) Drain(id string) error {
// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
// StopTimeout.
func (s *Supervisor) Stop(ctx context.Context, id string) error {
//
// Traced as "stop engine", saying whether it had to be killed: an engine
// that ignores SIGTERM for a minute is a minute its replacement waits for
// the GPU.
func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop engine",
trace.WithAttributes(attribute.String("llmgw.shard", id)))
defer func() { obs.End(span, err) }()
sh, err := s.Get(id)
if err != nil {
return err
@@ -527,6 +595,7 @@ func (s *Supervisor) Stop(ctx context.Context, id string) error {
// force kills the group outright. exec.Cmd's cancel does this for shards we
// started; an adopted shard has none, so signal its group directly.
force := func() {
span.SetAttributes(attribute.Bool("llmgw.killed", true))
if sh.cancel != nil {
sh.cancel()
return