Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
e725adc2f0
33 files changed
+1607
-143
No files matched your search
@@ -172,6 +172,9 @@ type Job struct {
|
||||
// how fast it said it.
|
||||
Reply string `json:"reply,omitempty"`
|
||||
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
|
||||
|
||||
// trace is the job's spans, when tracing is on.
|
||||
trace jobTrace
|
||||
}
|
||||
|
||||
// Server runs jobs and remembers them.
|
||||
@@ -318,6 +321,7 @@ func (s *Server) snapshot(j *Job) Job {
|
||||
func (s *Server) step(j *Job, name, state, detail string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
j.traceStep(name, state, detail)
|
||||
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
|
||||
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
|
||||
return
|
||||
@@ -329,6 +333,7 @@ func (s *Server) fail(j *Job, name string, err error) {
|
||||
s.step(j, name, Failed, err.Error())
|
||||
s.mu.Lock()
|
||||
j.State, j.Error = Failed, err.Error()
|
||||
j.traceEnd(err)
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
|
||||
}
|
||||
@@ -617,6 +622,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
func (s *Server) finish(j *Job) {
|
||||
s.mu.Lock()
|
||||
j.State = Done
|
||||
j.traceEnd(nil)
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
|
||||
}
|
||||
@@ -785,6 +791,7 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
|
||||
continue
|
||||
}
|
||||
if r.State == reconciler.Ready {
|
||||
s.linkReplica(j, r)
|
||||
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
|
||||
strings.Join(r.NodeIDs(), ","), r.Endpoint))
|
||||
return r, nil
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// A job is a trace of its own, and each of its steps a span beneath it —
|
||||
// registry, mirror, check, catalog, placement, smoke test — opened when the
|
||||
// step begins and closed when it is done, failed or skipped. The steps are
|
||||
// what a job already records for `ctl model jobs`; the trace adds where the
|
||||
// time went, and from the placement step a link to the replica's own start,
|
||||
// whose trace goes on down to the node.
|
||||
//
|
||||
// Begun by the first step rather than where a job is made, so every kind of
|
||||
// job is traced by recording steps and nothing else.
|
||||
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
|
||||
// jobTrace is a job's spans. Read and written with Server.mu held.
|
||||
type jobTrace struct {
|
||||
span trace.Span
|
||||
step trace.Span
|
||||
stepName string
|
||||
}
|
||||
|
||||
// traceStep records a step on the job's trace. Called with s.mu held.
|
||||
func (j *Job) traceStep(name, state, detail string) {
|
||||
if j.trace.span == nil {
|
||||
kind := j.Kind
|
||||
if kind == "" {
|
||||
kind = "serve"
|
||||
}
|
||||
_, j.trace.span = obs.Tracer(tracerName).Start(context.Background(), kind+" model",
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.job", j.ID),
|
||||
attribute.String("llmgw.repo", j.Request.Repo),
|
||||
attribute.String("llmgw.model", j.Model),
|
||||
))
|
||||
}
|
||||
if j.trace.step != nil && j.trace.stepName != name {
|
||||
// A step left running when the next began: over, whatever it said.
|
||||
j.trace.step.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
if j.trace.step == nil {
|
||||
ctx := trace.ContextWithSpan(context.Background(), j.trace.span)
|
||||
_, j.trace.step = obs.Tracer(tracerName).Start(ctx, name)
|
||||
j.trace.stepName = name
|
||||
}
|
||||
span := j.trace.step
|
||||
span.SetAttributes(attribute.String("llmgw.state", state), attribute.String("llmgw.detail", detail))
|
||||
switch state {
|
||||
case Running:
|
||||
return
|
||||
case Failed:
|
||||
span.SetStatus(codes.Error, detail)
|
||||
}
|
||||
span.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
|
||||
// traceEnd closes the job's trace, failed when err is set. Called with s.mu
|
||||
// held.
|
||||
func (j *Job) traceEnd(err error) {
|
||||
if j.trace.step != nil {
|
||||
j.trace.step.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
if j.trace.span == nil {
|
||||
return
|
||||
}
|
||||
if j.Model != "" {
|
||||
j.trace.span.SetAttributes(attribute.String("llmgw.model", j.Model))
|
||||
}
|
||||
obs.End(j.trace.span, err)
|
||||
j.trace.span = nil
|
||||
}
|
||||
|
||||
// linkReplica links the step in progress to the start of a replica it waited
|
||||
// for, which is in the reconciler's trace rather than this one.
|
||||
func (s *Server) linkReplica(j *Job, r reconciler.Replica) {
|
||||
sc := r.StartTrace()
|
||||
if !sc.IsValid() {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if j.trace.step != nil {
|
||||
j.trace.step.AddLink(trace.Link{SpanContext: sc,
|
||||
Attributes: []attribute.KeyValue{attribute.String("llmgw.replica", r.ID)}})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAJobIsATraceWithASpanPerStep(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
files := []hf.File{{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}}
|
||||
s, _, _, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
|
||||
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("job %s: %s", j.State, j.Error)
|
||||
}
|
||||
|
||||
root := rec.One(t, [16]byte{}, "serve model")
|
||||
tid := root.SpanContext().TraceID()
|
||||
if a := obstest.Attrs(root); a["llmgw.job"] != job.ID || a["llmgw.model"] != "qwen2.5-1.5b-instruct" {
|
||||
t.Errorf("the job's span does not say which job: %v", a)
|
||||
}
|
||||
var steps []string
|
||||
for _, sp := range rec.Ended() {
|
||||
if sp.SpanContext().TraceID() == tid && obstest.ChildOf(sp, root) {
|
||||
steps = append(steps, sp.Name())
|
||||
}
|
||||
}
|
||||
if strings.Join(steps, ",") != "registry,mirror,check,catalog,placement,smoke test" {
|
||||
t.Errorf("step spans %v", steps)
|
||||
}
|
||||
smoke := rec.One(t, tid, "smoke test")
|
||||
if a := obstest.Attrs(smoke); a["llmgw.state"] != Done || !strings.Contains(a["llmgw.detail"], "Hello!") {
|
||||
t.Errorf("the smoke test's span says %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedJobsTraceEndsAtTheStepThatFailed(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
|
||||
s, _, _, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
|
||||
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||
if j := wait(t, s, job.ID); j.State != Failed {
|
||||
t.Fatalf("job %s", j.State)
|
||||
}
|
||||
|
||||
root := rec.One(t, [16]byte{}, "serve model")
|
||||
if root.Status().Code != codes.Error || !strings.Contains(root.Status().Description, "registry published") {
|
||||
t.Errorf("the failed job's span says %+v", root.Status())
|
||||
}
|
||||
mirror := rec.One(t, root.SpanContext().TraceID(), "mirror")
|
||||
if mirror.Status().Code != codes.Error {
|
||||
t.Errorf("the step that failed is %+v", mirror.Status())
|
||||
}
|
||||
if len(rec.Named("catalog")) != 0 {
|
||||
t.Error("a step after the failure was traced")
|
||||
}
|
||||
}
|
||||
@@ -609,6 +609,9 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
|
||||
}
|
||||
}
|
||||
if left == 0 && len(ready) > 0 {
|
||||
for _, r := range ready {
|
||||
s.linkReplica(j, r)
|
||||
}
|
||||
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
|
||||
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
|
||||
reply, tps, err := s.smoke(ctx, ready[0], name)
|
||||
|
||||
Reference in new issue
Block a user