Ilker ManapandClaude Opus 5.5
e725adc2f0
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
99 lines
3.0 KiB
Go
99 lines
3.0 KiB
Go
// Package obstest records spans for tests that assert what was traced.
|
|
package obstest
|
|
|
|
import (
|
|
"testing"
|
|
"time"
|
|
|
|
"go.opentelemetry.io/otel"
|
|
"go.opentelemetry.io/otel/propagation"
|
|
sdktrace "go.opentelemetry.io/otel/sdk/trace"
|
|
"go.opentelemetry.io/otel/sdk/trace/tracetest"
|
|
"go.opentelemetry.io/otel/trace"
|
|
)
|
|
|
|
// Recorder is every span ended while a test runs.
|
|
type Recorder struct{ *tracetest.SpanRecorder }
|
|
|
|
// Record installs a provider that keeps every span, and the W3C propagator,
|
|
// for the length of the test, putting back what was there after.
|
|
//
|
|
// Global, as the code under test finds it, so no test that records may run
|
|
// in parallel with another that traces.
|
|
func Record(t *testing.T) *Recorder {
|
|
t.Helper()
|
|
rec := tracetest.NewSpanRecorder()
|
|
prevTP, prevProp := otel.GetTracerProvider(), otel.GetTextMapPropagator()
|
|
otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(rec)))
|
|
otel.SetTextMapPropagator(propagation.TraceContext{})
|
|
t.Cleanup(func() { otel.SetTracerProvider(prevTP); otel.SetTextMapPropagator(prevProp) })
|
|
return &Recorder{rec}
|
|
}
|
|
|
|
// Named is the ended spans called name, oldest first.
|
|
func (r *Recorder) Named(name string) []sdktrace.ReadOnlySpan {
|
|
var out []sdktrace.ReadOnlySpan
|
|
for _, s := range r.Ended() {
|
|
if s.Name() == name {
|
|
out = append(out, s)
|
|
}
|
|
}
|
|
return out
|
|
}
|
|
|
|
// One is the single ended span called name in trace tid, failing the test
|
|
// when there is not exactly one. A zero tid is any trace.
|
|
func (r *Recorder) One(t *testing.T, tid trace.TraceID, name string) sdktrace.ReadOnlySpan {
|
|
t.Helper()
|
|
var found []sdktrace.ReadOnlySpan
|
|
for _, s := range r.Named(name) {
|
|
if !tid.IsValid() || s.SpanContext().TraceID() == tid {
|
|
found = append(found, s)
|
|
}
|
|
}
|
|
if len(found) != 1 {
|
|
t.Fatalf("%d spans called %q in the trace; ended: %v", len(found), name, r.names())
|
|
}
|
|
return found[0]
|
|
}
|
|
|
|
// Await is One, waiting up to limit for the span to end: the code that
|
|
// ends it may run after whatever the test waited on.
|
|
func (r *Recorder) Await(t *testing.T, tid trace.TraceID, name string, limit time.Duration) sdktrace.ReadOnlySpan {
|
|
t.Helper()
|
|
deadline := time.Now().Add(limit)
|
|
for time.Now().Before(deadline) {
|
|
for _, s := range r.Named(name) {
|
|
if s.SpanContext().TraceID() == tid {
|
|
return r.One(t, tid, name)
|
|
}
|
|
}
|
|
time.Sleep(10 * time.Millisecond)
|
|
}
|
|
t.Fatalf("no span called %q ended within %v; ended: %v", name, limit, r.names())
|
|
return nil
|
|
}
|
|
|
|
func (r *Recorder) names() []string {
|
|
var out []string
|
|
for _, s := range r.Ended() {
|
|
out = append(out, s.Name())
|
|
}
|
|
return out
|
|
}
|
|
|
|
// Attrs is a span's attributes, as strings.
|
|
func Attrs(s sdktrace.ReadOnlySpan) map[string]string {
|
|
out := map[string]string{}
|
|
for _, a := range s.Attributes() {
|
|
out[string(a.Key)] = a.Value.Emit()
|
|
}
|
|
return out
|
|
}
|
|
|
|
// ChildOf reports whether child's parent is parent.
|
|
func ChildOf(child, parent sdktrace.ReadOnlySpan) bool {
|
|
return child.Parent().SpanID() == parent.SpanContext().SpanID() &&
|
|
child.SpanContext().TraceID() == parent.SpanContext().TraceID()
|
|
}
|