Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node

A reconcile pass that does something is a trace, with placement (where it
went, or every candidate turned down), scale-up/down, drain, stop, eviction,
preemption, reaping and restarts beneath it. A replica's start is a span open
from placement until it serves; the start command carries its context in a
new ControlMessage.trace field, and the agent continues the trace through
the weights fetch, engine launch, rendezvous and readiness wait. Stops,
drains, adoption and engine installs are traced on the node, model jobs are
a trace with a span per step linked to the replica's start, and agent.yaml
takes the same tracing block as server.yaml. Off, every span is the no-op one
and commands carry nothing.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:03:17 +02:00
1 parent e59ff52932
commit e725adc2f0
33 files changed
+1607 -143

No files matched your search

@@ -172,6 +172,9 @@ type Job struct {
// how fast it said it.
Reply string `json:"reply,omitempty"`
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
// trace is the job's spans, when tracing is on.
trace jobTrace
}
// Server runs jobs and remembers them.
@@ -318,6 +321,7 @@ func (s *Server) snapshot(j *Job) Job {
func (s *Server) step(j *Job, name, state, detail string) {
s.mu.Lock()
defer s.mu.Unlock()
j.traceStep(name, state, detail)
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
return
@@ -329,6 +333,7 @@ func (s *Server) fail(j *Job, name string, err error) {
s.step(j, name, Failed, err.Error())
s.mu.Lock()
j.State, j.Error = Failed, err.Error()
j.traceEnd(err)
s.mu.Unlock()
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
}
@@ -617,6 +622,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
func (s *Server) finish(j *Job) {
s.mu.Lock()
j.State = Done
j.traceEnd(nil)
s.mu.Unlock()
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
}
@@ -785,6 +791,7 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
continue
}
if r.State == reconciler.Ready {
s.linkReplica(j, r)
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
strings.Join(r.NodeIDs(), ","), r.Endpoint))
return r, nil
+99
View File
@@ -0,0 +1,99 @@
package modelserve
import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// A job is a trace of its own, and each of its steps a span beneath it —
// registry, mirror, check, catalog, placement, smoke test — opened when the
// step begins and closed when it is done, failed or skipped. The steps are
// what a job already records for `ctl model jobs`; the trace adds where the
// time went, and from the placement step a link to the replica's own start,
// whose trace goes on down to the node.
//
// Begun by the first step rather than where a job is made, so every kind of
// job is traced by recording steps and nothing else.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
// jobTrace is a job's spans. Read and written with Server.mu held.
type jobTrace struct {
span trace.Span
step trace.Span
stepName string
}
// traceStep records a step on the job's trace. Called with s.mu held.
func (j *Job) traceStep(name, state, detail string) {
if j.trace.span == nil {
kind := j.Kind
if kind == "" {
kind = "serve"
}
_, j.trace.span = obs.Tracer(tracerName).Start(context.Background(), kind+" model",
trace.WithAttributes(
attribute.String("llmgw.job", j.ID),
attribute.String("llmgw.repo", j.Request.Repo),
attribute.String("llmgw.model", j.Model),
))
}
if j.trace.step != nil && j.trace.stepName != name {
// A step left running when the next began: over, whatever it said.
j.trace.step.End()
j.trace.step = nil
}
if j.trace.step == nil {
ctx := trace.ContextWithSpan(context.Background(), j.trace.span)
_, j.trace.step = obs.Tracer(tracerName).Start(ctx, name)
j.trace.stepName = name
}
span := j.trace.step
span.SetAttributes(attribute.String("llmgw.state", state), attribute.String("llmgw.detail", detail))
switch state {
case Running:
return
case Failed:
span.SetStatus(codes.Error, detail)
}
span.End()
j.trace.step = nil
}
// traceEnd closes the job's trace, failed when err is set. Called with s.mu
// held.
func (j *Job) traceEnd(err error) {
if j.trace.step != nil {
j.trace.step.End()
j.trace.step = nil
}
if j.trace.span == nil {
return
}
if j.Model != "" {
j.trace.span.SetAttributes(attribute.String("llmgw.model", j.Model))
}
obs.End(j.trace.span, err)
j.trace.span = nil
}
// linkReplica links the step in progress to the start of a replica it waited
// for, which is in the reconciler's trace rather than this one.
func (s *Server) linkReplica(j *Job, r reconciler.Replica) {
sc := r.StartTrace()
if !sc.IsValid() {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if j.trace.step != nil {
j.trace.step.AddLink(trace.Link{SpanContext: sc,
Attributes: []attribute.KeyValue{attribute.String("llmgw.replica", r.ID)}})
}
}
+65
View File
@@ -0,0 +1,65 @@
package modelserve
import (
"strings"
"testing"
"go.opentelemetry.io/otel/codes"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
)
func TestAJobIsATraceWithASpanPerStep(t *testing.T) {
rec := obstest.Record(t)
files := []hf.File{{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}}
s, _, _, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
if err != nil {
t.Fatal(err)
}
if j := wait(t, s, job.ID); j.State != Done {
t.Fatalf("job %s: %s", j.State, j.Error)
}
root := rec.One(t, [16]byte{}, "serve model")
tid := root.SpanContext().TraceID()
if a := obstest.Attrs(root); a["llmgw.job"] != job.ID || a["llmgw.model"] != "qwen2.5-1.5b-instruct" {
t.Errorf("the job's span does not say which job: %v", a)
}
var steps []string
for _, sp := range rec.Ended() {
if sp.SpanContext().TraceID() == tid && obstest.ChildOf(sp, root) {
steps = append(steps, sp.Name())
}
}
if strings.Join(steps, ",") != "registry,mirror,check,catalog,placement,smoke test" {
t.Errorf("step spans %v", steps)
}
smoke := rec.One(t, tid, "smoke test")
if a := obstest.Attrs(smoke); a["llmgw.state"] != Done || !strings.Contains(a["llmgw.detail"], "Hello!") {
t.Errorf("the smoke test's span says %v", a)
}
}
func TestAFailedJobsTraceEndsAtTheStepThatFailed(t *testing.T) {
rec := obstest.Record(t)
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
s, _, _, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
if j := wait(t, s, job.ID); j.State != Failed {
t.Fatalf("job %s", j.State)
}
root := rec.One(t, [16]byte{}, "serve model")
if root.Status().Code != codes.Error || !strings.Contains(root.Status().Description, "registry published") {
t.Errorf("the failed job's span says %+v", root.Status())
}
mirror := rec.One(t, root.SpanContext().TraceID(), "mirror")
if mirror.Status().Code != codes.Error {
t.Errorf("the step that failed is %+v", mirror.Status())
}
if len(rec.Named("catalog")) != 0 {
t.Error("a step after the failure was traced")
}
}
+3
View File
@@ -609,6 +609,9 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
}
}
if left == 0 && len(ready) > 0 {
for _, r := range ready {
s.linkReplica(j, r)
}
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
reply, tps, err := s.smoke(ctx, ready[0], name)