Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
e725adc2f0
33 files changed
+1607
-143
No files matched your search
@@ -18,9 +18,19 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// tracerName names the spans the supervisor opens. There is no restart span
|
||||
// here because the supervisor never restarts an engine on its own: a restart
|
||||
// is the control plane's decision, traced there, and reaches a node as a stop
|
||||
// and a start.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
|
||||
// Config configures the supervisor.
|
||||
type Config struct {
|
||||
// WorkDir holds pid files and per-shard scratch space.
|
||||
@@ -195,7 +205,25 @@ var ErrAlreadyRunning = errors.New("shard is already running")
|
||||
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
|
||||
// rather than a second process, because the control plane retries commands
|
||||
// (plan §5.6).
|
||||
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (*Shard, error) {
|
||||
//
|
||||
// Traced as "launch engine": from the first process to the engine's being
|
||||
// started, which for a multi-node worker includes waiting for its leader's
|
||||
// rendezvous point. Loading the model is the readiness wait's, not this.
|
||||
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (_ *Shard, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "launch engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", id),
|
||||
attribute.String("llmgw.engine", spec.Profile.Engine),
|
||||
attribute.Int("llmgw.processes", len(cmds)),
|
||||
))
|
||||
defer func() {
|
||||
if errors.Is(err, ErrAlreadyRunning) {
|
||||
// A repeated command, which is not this start failing.
|
||||
span.SetAttributes(attribute.Bool("llmgw.already_running", true))
|
||||
span.End()
|
||||
return
|
||||
}
|
||||
obs.End(span, err)
|
||||
}()
|
||||
if len(cmds) == 0 {
|
||||
return nil, fmt.Errorf("shard %s: no commands to run", id)
|
||||
}
|
||||
@@ -301,6 +329,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
sh.pid = c.Process.Pid
|
||||
sh.state = Starting
|
||||
sh.mu.Unlock()
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.Int("llmgw.pid", c.Process.Pid))
|
||||
|
||||
if err := s.writeState(sh); err != nil {
|
||||
s.log.Warn("could not write pid file; adoption after a restart will not work",
|
||||
@@ -337,7 +366,13 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
|
||||
}
|
||||
|
||||
// waitForPort blocks until addr accepts a connection.
|
||||
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
|
||||
//
|
||||
// Traced: a worker's start that took ten minutes because its leader was
|
||||
// still loading reads as exactly that.
|
||||
func waitForPort(ctx context.Context, addr string, timeout time.Duration) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for rendezvous",
|
||||
trace.WithAttributes(attribute.String("llmgw.rendezvous", addr)))
|
||||
defer func() { obs.End(span, err) }()
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -416,7 +451,26 @@ func (s *Supervisor) wait(sh *Shard) {
|
||||
// The timeout comes from the profile because it is model-dependent: loading a
|
||||
// 70B model across two nodes legitimately takes tens of minutes, while a small
|
||||
// model failing to load for ten minutes means something is wrong.
|
||||
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
|
||||
//
|
||||
// Traced as "wait for engine": the model loading, which is usually most of a
|
||||
// start. The span says how many times the engine was asked and what it last
|
||||
// answered, so one that never came up says why.
|
||||
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", sh.ID),
|
||||
attribute.String("llmgw.readiness", "health check"),
|
||||
attribute.String("llmgw.timeout", timeout.String()),
|
||||
))
|
||||
polls := 0
|
||||
var answer error
|
||||
defer func() {
|
||||
span.SetAttributes(attribute.Int("llmgw.polls", polls))
|
||||
if answer != nil && err != nil {
|
||||
span.SetAttributes(attribute.String("llmgw.last_answer", answer.Error()))
|
||||
}
|
||||
obs.End(span, err)
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -432,7 +486,8 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
||||
last := strings.Join(sh.Logs(10), "\n")
|
||||
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
||||
case <-t.C:
|
||||
if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
|
||||
polls++
|
||||
if answer = s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); answer != nil {
|
||||
continue
|
||||
}
|
||||
sh.setState(Ready, nil)
|
||||
@@ -449,7 +504,13 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
||||
// leader answering, which only happens once every worker has joined. Polling a
|
||||
// worker's port instead would time out and fail a replica that is coming up
|
||||
// perfectly well.
|
||||
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) error {
|
||||
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) (err error) {
|
||||
_, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", sh.ID),
|
||||
attribute.String("llmgw.readiness", "stays alive"),
|
||||
attribute.String("llmgw.timeout", s.cfg.AliveGrace.String()),
|
||||
))
|
||||
defer func() { obs.End(span, err) }()
|
||||
sh.setState(Loading, nil)
|
||||
|
||||
select {
|
||||
@@ -483,7 +544,14 @@ func (s *Supervisor) Drain(id string) error {
|
||||
|
||||
// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
|
||||
// StopTimeout.
|
||||
func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
||||
//
|
||||
// Traced as "stop engine", saying whether it had to be killed: an engine
|
||||
// that ignores SIGTERM for a minute is a minute its replacement waits for
|
||||
// the GPU.
|
||||
func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop engine",
|
||||
trace.WithAttributes(attribute.String("llmgw.shard", id)))
|
||||
defer func() { obs.End(span, err) }()
|
||||
sh, err := s.Get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -527,6 +595,7 @@ func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
||||
// force kills the group outright. exec.Cmd's cancel does this for shards we
|
||||
// started; an adopted shard has none, so signal its group directly.
|
||||
force := func() {
|
||||
span.SetAttributes(attribute.Bool("llmgw.killed", true))
|
||||
if sh.cancel != nil {
|
||||
sh.cancel()
|
||||
return
|
||||
|
||||
Reference in new issue
Block a user