Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node

A reconcile pass that does something is a trace, with placement (where it
went, or every candidate turned down), scale-up/down, drain, stop, eviction,
preemption, reaping and restarts beneath it. A replica's start is a span open
from placement until it serves; the start command carries its context in a
new ControlMessage.trace field, and the agent continues the trace through
the weights fetch, engine launch, rendezvous and readiness wait. Stops,
drains, adoption and engine installs are traced on the node, model jobs are
a trace with a span per step linked to the replica's start, and agent.yaml
takes the same tracing block as server.yaml. Off, every span is the no-op one
and commands carry nothing.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:03:17 +02:00
1 parent e59ff52932
commit e725adc2f0
33 files changed
+1607 -143

No files matched your search

+98
View File
@@ -0,0 +1,98 @@
// Package obstest records spans for tests that assert what was traced.
package obstest
import (
"testing"
"time"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/propagation"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"go.opentelemetry.io/otel/trace"
)
// Recorder is every span ended while a test runs.
type Recorder struct{ *tracetest.SpanRecorder }
// Record installs a provider that keeps every span, and the W3C propagator,
// for the length of the test, putting back what was there after.
//
// Global, as the code under test finds it, so no test that records may run
// in parallel with another that traces.
func Record(t *testing.T) *Recorder {
t.Helper()
rec := tracetest.NewSpanRecorder()
prevTP, prevProp := otel.GetTracerProvider(), otel.GetTextMapPropagator()
otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)))
otel.SetTextMapPropagator(propagation.TraceContext{})
t.Cleanup(func() { otel.SetTracerProvider(prevTP); otel.SetTextMapPropagator(prevProp) })
return &Recorder{rec}
}
// Named is the ended spans called name, oldest first.
func (r *Recorder) Named(name string) []sdktrace.ReadOnlySpan {
var out []sdktrace.ReadOnlySpan
for _, s := range r.Ended() {
if s.Name() == name {
out = append(out, s)
}
}
return out
}
// One is the single ended span called name in trace tid, failing the test
// when there is not exactly one. A zero tid is any trace.
func (r *Recorder) One(t *testing.T, tid trace.TraceID, name string) sdktrace.ReadOnlySpan {
t.Helper()
var found []sdktrace.ReadOnlySpan
for _, s := range r.Named(name) {
if !tid.IsValid() || s.SpanContext().TraceID() == tid {
found = append(found, s)
}
}
if len(found) != 1 {
t.Fatalf("%d spans called %q in the trace; ended: %v", len(found), name, r.names())
}
return found[0]
}
// Await is One, waiting up to limit for the span to end: the code that
// ends it may run after whatever the test waited on.
func (r *Recorder) Await(t *testing.T, tid trace.TraceID, name string, limit time.Duration) sdktrace.ReadOnlySpan {
t.Helper()
deadline := time.Now().Add(limit)
for time.Now().Before(deadline) {
for _, s := range r.Named(name) {
if s.SpanContext().TraceID() == tid {
return r.One(t, tid, name)
}
}
time.Sleep(10 * time.Millisecond)
}
t.Fatalf("no span called %q ended within %v; ended: %v", name, limit, r.names())
return nil
}
func (r *Recorder) names() []string {
var out []string
for _, s := range r.Ended() {
out = append(out, s.Name())
}
return out
}
// Attrs is a span's attributes, as strings.
func Attrs(s sdktrace.ReadOnlySpan) map[string]string {
out := map[string]string{}
for _, a := range s.Attributes() {
out[string(a.Key)] = a.Value.Emit()
}
return out
}
// ChildOf reports whether child's parent is parent.
func ChildOf(child, parent sdktrace.ReadOnlySpan) bool {
return child.Parent().SpanID() == parent.SpanContext().SpanID() &&
child.SpanContext().TraceID() == parent.SpanContext().TraceID()
}
+74 -3
View File
@@ -9,18 +9,47 @@ import (
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp"
"go.opentelemetry.io/otel/propagation"
"go.opentelemetry.io/otel/sdk/resource"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/trace"
)
// Tracing is where traces go, over OTLP/HTTP.
//
// In server.yaml: one span per inference request, with admission, scheduling
// and each attempt at an engine beneath it, and the trace carried on to the
// engine in a traceparent header — the request's X-Request-Id is on the span,
// so a trace and `llmgw ctl request` name the same request. A control plane
// also traces every reconcile pass that does something, each replica from
// its placement until it serves, and each job `ctl model serve` and `ctl
// model update` start.
//
// In agent.yaml: the node's part of starting a replica — fetching the
// weights, launching the engine, waiting for it to answer — inside the
// control plane's trace of that start. Point both at the same collector.
type Tracing struct {
// Endpoint is the collector, for example http://otel.internal:4318.
// Empty falls back to OTEL_EXPORTER_OTLP_ENDPOINT, and to no tracing.
Endpoint string `koanf:"endpoint"`
// SampleRatio is the share of traces kept, 0 to 1, of those nobody
// upstream already decided. Zero takes 1. A request with a sampled
// traceparent of its own is kept, and an agent keeps every start the
// control plane kept, whatever their own ratio says.
SampleRatio float64 `koanf:"sample_ratio"`
}
// TracingOptions says where traces go.
type TracingOptions struct {
Endpoint string // http(s)://host:4318; empty reads OTEL_EXPORTER_OTLP_ENDPOINT
SampleRatio float64 // of traces the caller did not already decide; zero takes 1
Service string
Version string
// Instance tells apart the processes of one service: the node, for an
// agent. Empty leaves it out.
Instance string
}
// SetupTracing installs the global tracer provider and the W3C propagator,
@@ -55,13 +84,16 @@ func SetupTracing(ctx context.Context, o TracingOptions) (func(context.Context)
if ratio <= 0 || ratio > 1 {
ratio = 1
}
res := resource.NewSchemaless(
attrs := []attribute.KeyValue{
attribute.String("service.name", o.Service),
attribute.String("service.version", o.Version),
)
}
if o.Instance != "" {
attrs = append(attrs, attribute.String("service.instance.id", o.Instance))
}
tp := sdktrace.NewTracerProvider(
sdktrace.WithBatcher(exp, sdktrace.WithBatchTimeout(5*time.Second)),
sdktrace.WithResource(res),
sdktrace.WithResource(resource.NewSchemaless(attrs...)),
// A caller that sent a sampled traceparent has already decided; the
// ratio is for requests that arrive without one.
sdktrace.WithSampler(sdktrace.ParentBased(sdktrace.TraceIDRatioBased(ratio))),
@@ -70,3 +102,42 @@ func SetupTracing(ctx context.Context, o TracingOptions) (func(context.Context)
otel.SetTextMapPropagator(propagation.TraceContext{})
return tp.Shutdown, nil
}
// Tracer is the named tracer of whatever provider is installed when it is
// called. Looked up per use rather than kept, so a provider installed after a
// component was built — as tests do — is the one its spans go to. With none
// installed it is the no-op tracer, and a span costs a function call.
func Tracer(name string) trace.Tracer { return otel.Tracer(name) }
// End closes a span, marking it failed when err is set.
func End(span trace.Span, err error) {
if err != nil {
span.RecordError(err)
span.SetStatus(codes.Error, err.Error())
}
span.End()
}
// Inject is the trace context of ctx as a message carries it: traceparent
// and, when there is one, tracestate. Nil when ctx holds no span, which is
// every context while tracing is off, so a message then carries nothing.
func Inject(ctx context.Context) map[string]string {
if !trace.SpanContextFromContext(ctx).IsValid() {
return nil
}
m := propagation.MapCarrier{}
otel.GetTextMapPropagator().Inject(ctx, m)
if len(m) == 0 {
return nil
}
return m
}
// Extract is ctx continuing the trace a message carried, as Inject wrote it.
// A message that carried none leaves ctx as it was.
func Extract(ctx context.Context, carried map[string]string) context.Context {
if len(carried) == 0 {
return ctx
}
return otel.GetTextMapPropagator().Extract(ctx, propagation.MapCarrier(carried))
}
+63
View File
@@ -2,7 +2,16 @@ package obs
import (
"context"
"errors"
"testing"
"go.opentelemetry.io/otel"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/propagation"
sdktrace "go.opentelemetry.io/otel/sdk/trace"
"go.opentelemetry.io/otel/sdk/trace/tracetest"
"go.opentelemetry.io/otel/trace"
"go.opentelemetry.io/otel/trace/noop"
)
func TestTracingIsOffUntilGivenSomewhereToSend(t *testing.T) {
@@ -17,3 +26,57 @@ func TestTracingIsOffUntilGivenSomewhereToSend(t *testing.T) {
}
}
}
func TestATraceCrossesAMessageAndContinuesOnTheOtherSide(t *testing.T) {
rec := tracetest.NewSpanRecorder()
prevTP, prevProp := otel.GetTracerProvider(), otel.GetTextMapPropagator()
otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)))
otel.SetTextMapPropagator(propagation.TraceContext{})
t.Cleanup(func() { otel.SetTracerProvider(prevTP); otel.SetTextMapPropagator(prevProp) })
if m := Inject(context.Background()); m != nil {
t.Errorf("a context with no span carried %v", m)
}
if ctx := Extract(context.Background(), nil); trace.SpanContextFromContext(ctx).IsValid() {
t.Error("nothing was carried, yet a trace continued")
}
ctx, sent := Tracer("t").Start(context.Background(), "send")
carried := Inject(ctx)
sent.End()
if carried["traceparent"] == "" {
t.Fatalf("no traceparent carried: %v", carried)
}
_, got := Tracer("t").Start(Extract(context.Background(), carried), "receive")
End(got, errors.New("it broke"))
spans := rec.Ended()
if len(spans) != 2 {
t.Fatalf("%d spans", len(spans))
}
recv := spans[1]
if recv.Parent().SpanID() != sent.SpanContext().SpanID() || recv.SpanContext().TraceID() != sent.SpanContext().TraceID() {
t.Error("the receiving span is not the sending one's child")
}
if recv.Status().Code != codes.Error || recv.Status().Description != "it broke" {
t.Errorf("a failure was not recorded: %+v", recv.Status())
}
}
func TestWithTracingOffAMessageCarriesNothing(t *testing.T) {
// What a process that never called SetupTracing has: spans that are not
// recorded and contexts with nothing to propagate.
prevTP, prevProp := otel.GetTracerProvider(), otel.GetTextMapPropagator()
otel.SetTracerProvider(noop.NewTracerProvider())
otel.SetTextMapPropagator(propagation.NewCompositeTextMapPropagator())
t.Cleanup(func() { otel.SetTracerProvider(prevTP); otel.SetTextMapPropagator(prevProp) })
ctx, span := Tracer("t").Start(context.Background(), "send")
defer span.End()
if span.IsRecording() {
t.Error("a span is recorded with tracing off")
}
if m := Inject(ctx); m != nil {
t.Errorf("a message carries %v with tracing off", m)
}
}