Merge C20 tracing beyond the router

Conflicts: config imports and docgen sources (both kept); agent.pb.go
regenerated with buf from the merged proto (MIG devices and trace context).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:05:29 +02:00
commit 801e2d51c0
33 files changed
+1608 -144

No files matched your search

@@ -178,6 +178,9 @@ type Job struct {
// how fast it said it.
Reply string `json:"reply,omitempty"`
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
// trace is the job's spans, when tracing is on.
trace jobTrace
}
// Server runs jobs and remembers them.
@@ -324,6 +327,7 @@ func (s *Server) snapshot(j *Job) Job {
func (s *Server) step(j *Job, name, state, detail string) {
s.mu.Lock()
defer s.mu.Unlock()
j.traceStep(name, state, detail)
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
return
@@ -335,6 +339,7 @@ func (s *Server) fail(j *Job, name string, err error) {
s.step(j, name, Failed, err.Error())
s.mu.Lock()
j.State, j.Error = Failed, err.Error()
j.traceEnd(err)
s.mu.Unlock()
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
}
@@ -623,6 +628,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
func (s *Server) finish(j *Job) {
s.mu.Lock()
j.State = Done
j.traceEnd(nil)
s.mu.Unlock()
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
}
@@ -791,6 +797,7 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
continue
}
if r.State == reconciler.Ready {
s.linkReplica(j, r)
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
strings.Join(r.NodeIDs(), ","), r.Endpoint))
return r, nil
+99
View File
@@ -0,0 +1,99 @@
package modelserve
import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// A job is a trace of its own, and each of its steps a span beneath it —
// registry, mirror, check, catalog, placement, smoke test — opened when the
// step begins and closed when it is done, failed or skipped. The steps are
// what a job already records for `ctl model jobs`; the trace adds where the
// time went, and from the placement step a link to the replica's own start,
// whose trace goes on down to the node.
//
// Begun by the first step rather than where a job is made, so every kind of
// job is traced by recording steps and nothing else.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
// jobTrace is a job's spans. Read and written with Server.mu held.
type jobTrace struct {
span trace.Span
step trace.Span
stepName string
}
// traceStep records a step on the job's trace. Called with s.mu held.
func (j *Job) traceStep(name, state, detail string) {
if j.trace.span == nil {
kind := j.Kind
if kind == "" {
kind = "serve"
}
_, j.trace.span = obs.Tracer(tracerName).Start(context.Background(), kind+" model",
trace.WithAttributes(
attribute.String("llmgw.job", j.ID),
attribute.String("llmgw.repo", j.Request.Repo),
attribute.String("llmgw.model", j.Model),
))
}
if j.trace.step != nil && j.trace.stepName != name {
// A step left running when the next began: over, whatever it said.
j.trace.step.End()
j.trace.step = nil
}
if j.trace.step == nil {
ctx := trace.ContextWithSpan(context.Background(), j.trace.span)
_, j.trace.step = obs.Tracer(tracerName).Start(ctx, name)
j.trace.stepName = name
}
span := j.trace.step
span.SetAttributes(attribute.String("llmgw.state", state), attribute.String("llmgw.detail", detail))
switch state {
case Running:
return
case Failed:
span.SetStatus(codes.Error, detail)
}
span.End()
j.trace.step = nil
}
// traceEnd closes the job's trace, failed when err is set. Called with s.mu
// held.
func (j *Job) traceEnd(err error) {
if j.trace.step != nil {
j.trace.step.End()
j.trace.step = nil
}
if j.trace.span == nil {
return
}
if j.Model != "" {
j.trace.span.SetAttributes(attribute.String("llmgw.model", j.Model))
}
obs.End(j.trace.span, err)
j.trace.span = nil
}
// linkReplica links the step in progress to the start of a replica it waited
// for, which is in the reconciler's trace rather than this one.
func (s *Server) linkReplica(j *Job, r reconciler.Replica) {
sc := r.StartTrace()
if !sc.IsValid() {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if j.trace.step != nil {
j.trace.step.AddLink(trace.Link{SpanContext: sc,
Attributes: []attribute.KeyValue{attribute.String("llmgw.replica", r.ID)}})
}
}
+65
View File
@@ -0,0 +1,65 @@
package modelserve
import (
"strings"
"testing"
"go.opentelemetry.io/otel/codes"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
)
func TestAJobIsATraceWithASpanPerStep(t *testing.T) {
rec := obstest.Record(t)
files := []hf.File{{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}}
s, _, _, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
if err != nil {
t.Fatal(err)
}
if j := wait(t, s, job.ID); j.State != Done {
t.Fatalf("job %s: %s", j.State, j.Error)
}
root := rec.One(t, [16]byte{}, "serve model")
tid := root.SpanContext().TraceID()
if a := obstest.Attrs(root); a["llmgw.job"] != job.ID || a["llmgw.model"] != "qwen2.5-1.5b-instruct" {
t.Errorf("the job's span does not say which job: %v", a)
}
var steps []string
for _, sp := range rec.Ended() {
if sp.SpanContext().TraceID() == tid && obstest.ChildOf(sp, root) {
steps = append(steps, sp.Name())
}
}
if strings.Join(steps, ",") != "registry,mirror,check,catalog,placement,smoke test" {
t.Errorf("step spans %v", steps)
}
smoke := rec.One(t, tid, "smoke test")
if a := obstest.Attrs(smoke); a["llmgw.state"] != Done || !strings.Contains(a["llmgw.detail"], "Hello!") {
t.Errorf("the smoke test's span says %v", a)
}
}
func TestAFailedJobsTraceEndsAtTheStepThatFailed(t *testing.T) {
rec := obstest.Record(t)
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
s, _, _, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
if j := wait(t, s, job.ID); j.State != Failed {
t.Fatalf("job %s", j.State)
}
root := rec.One(t, [16]byte{}, "serve model")
if root.Status().Code != codes.Error || !strings.Contains(root.Status().Description, "registry published") {
t.Errorf("the failed job's span says %+v", root.Status())
}
mirror := rec.One(t, root.SpanContext().TraceID(), "mirror")
if mirror.Status().Code != codes.Error {
t.Errorf("the step that failed is %+v", mirror.Status())
}
if len(rec.Named("catalog")) != 0 {
t.Error("a step after the failure was traced")
}
}
+3
View File
@@ -636,6 +636,9 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
}
}
if left == 0 && len(ready) > 0 {
for _, r := range ready {
s.linkReplica(j, r)
}
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
reply, tps, err := s.smoke(ctx, ready[0], name)