Merge C20 tracing beyond the router
Conflicts: config imports and docgen sources (both kept); agent.pb.go regenerated with buf from the merged proto (MIG devices and trace context). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
801e2d51c0
33 files changed
+1608
-144
No files matched your search
@@ -178,6 +178,9 @@ type Job struct {
|
||||
// how fast it said it.
|
||||
Reply string `json:"reply,omitempty"`
|
||||
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
|
||||
|
||||
// trace is the job's spans, when tracing is on.
|
||||
trace jobTrace
|
||||
}
|
||||
|
||||
// Server runs jobs and remembers them.
|
||||
@@ -324,6 +327,7 @@ func (s *Server) snapshot(j *Job) Job {
|
||||
func (s *Server) step(j *Job, name, state, detail string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
j.traceStep(name, state, detail)
|
||||
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
|
||||
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
|
||||
return
|
||||
@@ -335,6 +339,7 @@ func (s *Server) fail(j *Job, name string, err error) {
|
||||
s.step(j, name, Failed, err.Error())
|
||||
s.mu.Lock()
|
||||
j.State, j.Error = Failed, err.Error()
|
||||
j.traceEnd(err)
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
|
||||
}
|
||||
@@ -623,6 +628,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
func (s *Server) finish(j *Job) {
|
||||
s.mu.Lock()
|
||||
j.State = Done
|
||||
j.traceEnd(nil)
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
|
||||
}
|
||||
@@ -791,6 +797,7 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
|
||||
continue
|
||||
}
|
||||
if r.State == reconciler.Ready {
|
||||
s.linkReplica(j, r)
|
||||
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
|
||||
strings.Join(r.NodeIDs(), ","), r.Endpoint))
|
||||
return r, nil
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// A job is a trace of its own, and each of its steps a span beneath it —
|
||||
// registry, mirror, check, catalog, placement, smoke test — opened when the
|
||||
// step begins and closed when it is done, failed or skipped. The steps are
|
||||
// what a job already records for `ctl model jobs`; the trace adds where the
|
||||
// time went, and from the placement step a link to the replica's own start,
|
||||
// whose trace goes on down to the node.
|
||||
//
|
||||
// Begun by the first step rather than where a job is made, so every kind of
|
||||
// job is traced by recording steps and nothing else.
|
||||
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
|
||||
// jobTrace is a job's spans. Read and written with Server.mu held.
|
||||
type jobTrace struct {
|
||||
span trace.Span
|
||||
step trace.Span
|
||||
stepName string
|
||||
}
|
||||
|
||||
// traceStep records a step on the job's trace. Called with s.mu held.
|
||||
func (j *Job) traceStep(name, state, detail string) {
|
||||
if j.trace.span == nil {
|
||||
kind := j.Kind
|
||||
if kind == "" {
|
||||
kind = "serve"
|
||||
}
|
||||
_, j.trace.span = obs.Tracer(tracerName).Start(context.Background(), kind+" model",
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.job", j.ID),
|
||||
attribute.String("llmgw.repo", j.Request.Repo),
|
||||
attribute.String("llmgw.model", j.Model),
|
||||
))
|
||||
}
|
||||
if j.trace.step != nil && j.trace.stepName != name {
|
||||
// A step left running when the next began: over, whatever it said.
|
||||
j.trace.step.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
if j.trace.step == nil {
|
||||
ctx := trace.ContextWithSpan(context.Background(), j.trace.span)
|
||||
_, j.trace.step = obs.Tracer(tracerName).Start(ctx, name)
|
||||
j.trace.stepName = name
|
||||
}
|
||||
span := j.trace.step
|
||||
span.SetAttributes(attribute.String("llmgw.state", state), attribute.String("llmgw.detail", detail))
|
||||
switch state {
|
||||
case Running:
|
||||
return
|
||||
case Failed:
|
||||
span.SetStatus(codes.Error, detail)
|
||||
}
|
||||
span.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
|
||||
// traceEnd closes the job's trace, failed when err is set. Called with s.mu
|
||||
// held.
|
||||
func (j *Job) traceEnd(err error) {
|
||||
if j.trace.step != nil {
|
||||
j.trace.step.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
if j.trace.span == nil {
|
||||
return
|
||||
}
|
||||
if j.Model != "" {
|
||||
j.trace.span.SetAttributes(attribute.String("llmgw.model", j.Model))
|
||||
}
|
||||
obs.End(j.trace.span, err)
|
||||
j.trace.span = nil
|
||||
}
|
||||
|
||||
// linkReplica links the step in progress to the start of a replica it waited
|
||||
// for, which is in the reconciler's trace rather than this one.
|
||||
func (s *Server) linkReplica(j *Job, r reconciler.Replica) {
|
||||
sc := r.StartTrace()
|
||||
if !sc.IsValid() {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if j.trace.step != nil {
|
||||
j.trace.step.AddLink(trace.Link{SpanContext: sc,
|
||||
Attributes: []attribute.KeyValue{attribute.String("llmgw.replica", r.ID)}})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAJobIsATraceWithASpanPerStep(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
files := []hf.File{{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}}
|
||||
s, _, _, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
|
||||
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("job %s: %s", j.State, j.Error)
|
||||
}
|
||||
|
||||
root := rec.One(t, [16]byte{}, "serve model")
|
||||
tid := root.SpanContext().TraceID()
|
||||
if a := obstest.Attrs(root); a["llmgw.job"] != job.ID || a["llmgw.model"] != "qwen2.5-1.5b-instruct" {
|
||||
t.Errorf("the job's span does not say which job: %v", a)
|
||||
}
|
||||
var steps []string
|
||||
for _, sp := range rec.Ended() {
|
||||
if sp.SpanContext().TraceID() == tid && obstest.ChildOf(sp, root) {
|
||||
steps = append(steps, sp.Name())
|
||||
}
|
||||
}
|
||||
if strings.Join(steps, ",") != "registry,mirror,check,catalog,placement,smoke test" {
|
||||
t.Errorf("step spans %v", steps)
|
||||
}
|
||||
smoke := rec.One(t, tid, "smoke test")
|
||||
if a := obstest.Attrs(smoke); a["llmgw.state"] != Done || !strings.Contains(a["llmgw.detail"], "Hello!") {
|
||||
t.Errorf("the smoke test's span says %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedJobsTraceEndsAtTheStepThatFailed(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
|
||||
s, _, _, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
|
||||
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||
if j := wait(t, s, job.ID); j.State != Failed {
|
||||
t.Fatalf("job %s", j.State)
|
||||
}
|
||||
|
||||
root := rec.One(t, [16]byte{}, "serve model")
|
||||
if root.Status().Code != codes.Error || !strings.Contains(root.Status().Description, "registry published") {
|
||||
t.Errorf("the failed job's span says %+v", root.Status())
|
||||
}
|
||||
mirror := rec.One(t, root.SpanContext().TraceID(), "mirror")
|
||||
if mirror.Status().Code != codes.Error {
|
||||
t.Errorf("the step that failed is %+v", mirror.Status())
|
||||
}
|
||||
if len(rec.Named("catalog")) != 0 {
|
||||
t.Error("a step after the failure was traced")
|
||||
}
|
||||
}
|
||||
@@ -636,6 +636,9 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
|
||||
}
|
||||
}
|
||||
if left == 0 && len(ready) > 0 {
|
||||
for _, r := range ready {
|
||||
s.linkReplica(j, r)
|
||||
}
|
||||
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
|
||||
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
|
||||
reply, tps, err := s.smoke(ctx, ready[0], name)
|
||||
|
||||
Reference in new issue
Block a user