Merge C20 tracing beyond the router

Conflicts: config imports and docgen sources (both kept); agent.pb.go
regenerated with buf from the merged proto (MIG devices and trace context).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-05 00:05:29 +02:00
commit 801e2d51c0
33 files changed
+1608 -144

No files matched your search

+34 -7
View File
@@ -14,6 +14,8 @@ import (
"time"
"connectrpc.com/connect"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/trace"
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
@@ -22,6 +24,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// StatusSink receives shard transitions and recovers replica state from what
@@ -311,7 +314,20 @@ func (s *Service) Connect(
// unreachable in production while its unit tests, which call the reconciler
// directly, kept passing.
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
// A trace of its own, and only for a node that reported something: a
// node running nothing registering is not an event anyone looks for.
tctx, span := ctx, trace.SpanFromContext(context.Background()) // the no-op span
if len(shards) > 0 {
tctx, span = obs.Tracer(tracerName).Start(ctx, "adopt",
trace.WithNewRoot(),
trace.WithAttributes(
attribute.String("llmgw.node", nodeID),
attribute.Int("llmgw.shards", len(shards)),
))
}
defer span.End()
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
span.SetAttributes(attribute.Int("llmgw.unclaimed", len(unclaimed)))
for _, u := range unclaimed {
// The model or profile left the catalog while this was running. Nobody
// will ever route to it, and it is holding GPUs.
@@ -319,7 +335,8 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
select {
case c.send <- &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
Trace: obs.Inject(tctx),
}:
case <-ctx.Done():
return
@@ -715,27 +732,37 @@ type Leader interface{ IsLeader() bool }
// and what every deployment before leadership had.
func (s *Service) SetLeadership(l Leader) { s.leader = l }
// tracerName names the spans this package opens.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
// ErrNotConnected means no agent stream is open for that node.
var ErrNotConnected = errors.New("no agent connected")
// StartShard implements reconciler.Dispatcher.
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
//
// The command carries the trace of ctx, so the agent's spans for this start
// — the weights, the engine's launch, its readiness — are in the trace of
// the replica they are starting rather than in one of their own.
func (s *Service) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
Trace: obs.Inject(ctx),
})
}
// StopShard implements reconciler.Dispatcher.
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
func (s *Service) StopShard(ctx context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
Trace: obs.Inject(ctx),
})
}
// DrainShard implements reconciler.Dispatcher.
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
func (s *Service) DrainShard(ctx context.Context, nodeID, shardID string) error {
return s.send(nodeID, &llmgwv1.ControlMessage{
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
Trace: obs.Inject(ctx),
})
}
@@ -178,6 +178,9 @@ type Job struct {
// how fast it said it.
Reply string `json:"reply,omitempty"`
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
// trace is the job's spans, when tracing is on.
trace jobTrace
}
// Server runs jobs and remembers them.
@@ -324,6 +327,7 @@ func (s *Server) snapshot(j *Job) Job {
func (s *Server) step(j *Job, name, state, detail string) {
s.mu.Lock()
defer s.mu.Unlock()
j.traceStep(name, state, detail)
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
return
@@ -335,6 +339,7 @@ func (s *Server) fail(j *Job, name string, err error) {
s.step(j, name, Failed, err.Error())
s.mu.Lock()
j.State, j.Error = Failed, err.Error()
j.traceEnd(err)
s.mu.Unlock()
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
}
@@ -623,6 +628,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
func (s *Server) finish(j *Job) {
s.mu.Lock()
j.State = Done
j.traceEnd(nil)
s.mu.Unlock()
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
}
@@ -791,6 +797,7 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
continue
}
if r.State == reconciler.Ready {
s.linkReplica(j, r)
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
strings.Join(r.NodeIDs(), ","), r.Endpoint))
return r, nil
+99
View File
@@ -0,0 +1,99 @@
package modelserve
import (
"context"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// A job is a trace of its own, and each of its steps a span beneath it —
// registry, mirror, check, catalog, placement, smoke test — opened when the
// step begins and closed when it is done, failed or skipped. The steps are
// what a job already records for `ctl model jobs`; the trace adds where the
// time went, and from the placement step a link to the replica's own start,
// whose trace goes on down to the node.
//
// Begun by the first step rather than where a job is made, so every kind of
// job is traced by recording steps and nothing else.
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
// jobTrace is a job's spans. Read and written with Server.mu held.
type jobTrace struct {
span trace.Span
step trace.Span
stepName string
}
// traceStep records a step on the job's trace. Called with s.mu held.
func (j *Job) traceStep(name, state, detail string) {
if j.trace.span == nil {
kind := j.Kind
if kind == "" {
kind = "serve"
}
_, j.trace.span = obs.Tracer(tracerName).Start(context.Background(), kind+" model",
trace.WithAttributes(
attribute.String("llmgw.job", j.ID),
attribute.String("llmgw.repo", j.Request.Repo),
attribute.String("llmgw.model", j.Model),
))
}
if j.trace.step != nil && j.trace.stepName != name {
// A step left running when the next began: over, whatever it said.
j.trace.step.End()
j.trace.step = nil
}
if j.trace.step == nil {
ctx := trace.ContextWithSpan(context.Background(), j.trace.span)
_, j.trace.step = obs.Tracer(tracerName).Start(ctx, name)
j.trace.stepName = name
}
span := j.trace.step
span.SetAttributes(attribute.String("llmgw.state", state), attribute.String("llmgw.detail", detail))
switch state {
case Running:
return
case Failed:
span.SetStatus(codes.Error, detail)
}
span.End()
j.trace.step = nil
}
// traceEnd closes the job's trace, failed when err is set. Called with s.mu
// held.
func (j *Job) traceEnd(err error) {
if j.trace.step != nil {
j.trace.step.End()
j.trace.step = nil
}
if j.trace.span == nil {
return
}
if j.Model != "" {
j.trace.span.SetAttributes(attribute.String("llmgw.model", j.Model))
}
obs.End(j.trace.span, err)
j.trace.span = nil
}
// linkReplica links the step in progress to the start of a replica it waited
// for, which is in the reconciler's trace rather than this one.
func (s *Server) linkReplica(j *Job, r reconciler.Replica) {
sc := r.StartTrace()
if !sc.IsValid() {
return
}
s.mu.Lock()
defer s.mu.Unlock()
if j.trace.step != nil {
j.trace.step.AddLink(trace.Link{SpanContext: sc,
Attributes: []attribute.KeyValue{attribute.String("llmgw.replica", r.ID)}})
}
}
+65
View File
@@ -0,0 +1,65 @@
package modelserve
import (
"strings"
"testing"
"go.opentelemetry.io/otel/codes"
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
)
func TestAJobIsATraceWithASpanPerStep(t *testing.T) {
rec := obstest.Record(t)
files := []hf.File{{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}}
s, _, _, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
if err != nil {
t.Fatal(err)
}
if j := wait(t, s, job.ID); j.State != Done {
t.Fatalf("job %s: %s", j.State, j.Error)
}
root := rec.One(t, [16]byte{}, "serve model")
tid := root.SpanContext().TraceID()
if a := obstest.Attrs(root); a["llmgw.job"] != job.ID || a["llmgw.model"] != "qwen2.5-1.5b-instruct" {
t.Errorf("the job's span does not say which job: %v", a)
}
var steps []string
for _, sp := range rec.Ended() {
if sp.SpanContext().TraceID() == tid && obstest.ChildOf(sp, root) {
steps = append(steps, sp.Name())
}
}
if strings.Join(steps, ",") != "registry,mirror,check,catalog,placement,smoke test" {
t.Errorf("step spans %v", steps)
}
smoke := rec.One(t, tid, "smoke test")
if a := obstest.Attrs(smoke); a["llmgw.state"] != Done || !strings.Contains(a["llmgw.detail"], "Hello!") {
t.Errorf("the smoke test's span says %v", a)
}
}
func TestAFailedJobsTraceEndsAtTheStepThatFailed(t *testing.T) {
rec := obstest.Record(t)
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
s, _, _, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
if j := wait(t, s, job.ID); j.State != Failed {
t.Fatalf("job %s", j.State)
}
root := rec.One(t, [16]byte{}, "serve model")
if root.Status().Code != codes.Error || !strings.Contains(root.Status().Description, "registry published") {
t.Errorf("the failed job's span says %+v", root.Status())
}
mirror := rec.One(t, root.SpanContext().TraceID(), "mirror")
if mirror.Status().Code != codes.Error {
t.Errorf("the step that failed is %+v", mirror.Status())
}
if len(rec.Named("catalog")) != 0 {
t.Error("a step after the failure was traced")
}
}
+3
View File
@@ -636,6 +636,9 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
}
}
if left == 0 && len(ready) > 0 {
for _, r := range ready {
s.linkReplica(j, r)
}
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
reply, tps, err := s.smoke(ctx, ready[0], name)
+11 -1
View File
@@ -6,6 +6,8 @@ import (
"sort"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
@@ -53,7 +55,15 @@ func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
r.log.Info("evicting an idle replica to make room",
"stopping", best.ID, "idle_model", best.Model,
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
r.stopReplica(ctx, best.ID)
ctx, span := startSpan(ctx, "evict",
attribute.String("llmgw.for", m.Name),
attribute.String("llmgw.replica", best.ID),
attribute.String("llmgw.model", best.Model),
attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
attribute.Int("llmgw.gpus", best.GPUs),
attribute.Int("llmgw.candidates", len(candidates)))
defer span.End()
r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
return true
}
+16 -1
View File
@@ -2,10 +2,13 @@ package reconciler
import (
"context"
"fmt"
"slices"
"sort"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
@@ -113,12 +116,24 @@ func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []vict
r.preempting[m.Name] = ids
r.mu.Unlock()
victims := make([]string, 0, len(chosen))
for _, v := range chosen {
r.log.Warn("preempting a replica to make room",
"replica", v.ID, "model", v.Model, "class", v.Class, "above_min", v.AboveMin,
"for", m.Name, "for_class", m.EffectiveClass())
desc := fmt.Sprintf("%s (%s, %s)", v.ID, v.Model, v.Class)
if v.AboveMin {
desc = fmt.Sprintf("%s (%s, %s, above its minimum)", v.ID, v.Model, v.Class)
}
victims = append(victims, desc)
}
r.scaleDown(ctx, ids)
ctx, span := startSpan(ctx, "preempt",
attribute.String("llmgw.for", m.Name),
attribute.String("llmgw.for_class", m.EffectiveClass()),
attribute.StringSlice("llmgw.victims", victims),
attribute.String("llmgw.hold", r.cfg.PreemptHold.String()))
defer span.End()
r.scaleDown(ctx, ids, "preempted for "+m.Name)
}
// awaitingPreemption reports whether replicas stopped for a model are still
+184 -25
View File
@@ -11,6 +11,10 @@ import (
"sync/atomic"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
@@ -18,6 +22,7 @@ import (
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// ShardCommand is everything an agent needs to launch one shard.
@@ -405,7 +410,9 @@ func (r *Reconciler) Reconcile(ctx context.Context) {
if r.leader != nil && !r.leader.IsLeader() {
return
}
r.expireNodes()
ctx, p := beginPass(ctx)
defer p.end()
r.expireNodes(ctx)
r.expireStuckReplicas()
r.drainCordonedNodes(ctx)
r.reapFailed(ctx)
@@ -452,7 +459,7 @@ func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
for _, rep := range gone {
r.log.Info("stopping a replica whose model left the catalog",
"replica", rep.ID, "model", rep.Model)
r.stopReplica(ctx, rep.ID)
r.stopReplica(ctx, rep.ID, "its model left the catalog")
}
}
@@ -523,6 +530,7 @@ func (r *Reconciler) expireStuckReplicas() {
}
rep.State = Failed
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
rep.startDone(now)
stuck = append(stuck, rep)
}
r.mu.Unlock()
@@ -555,7 +563,7 @@ func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
// The two-stage delay is deliberate: a brief network blip must not tear down
// replicas that are still serving traffic perfectly well, and rebuilding a
// multi-node replica costs tens of minutes of model loading.
func (r *Reconciler) expireNodes() {
func (r *Reconciler) expireNodes(ctx context.Context) {
now := r.now()
for _, n := range r.inv.Nodes() {
silent := now.Sub(n.LastSeen)
@@ -563,14 +571,21 @@ func (r *Reconciler) expireNodes() {
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
r.inv.SetState(n.ID, inventory.Unreachable)
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
_, span := startSpan(ctx, "node unreachable", attribute.String("llmgw.node", n.ID),
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
span.End()
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
_, span := startSpan(ctx, "node lost", attribute.String("llmgw.node", n.ID),
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
r.inv.SetState(n.ID, inventory.Lost)
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
"node", n.ID, "silent_for", silent.Round(time.Second))
r.notify.Load().Notify(notify.NodeLost, n.ID,
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
span.SetAttributes(attribute.StringSlice("llmgw.replicas_failed",
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")))
span.End()
}
}
}
@@ -609,24 +624,26 @@ func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
for _, rep := range toDrain {
r.log.Info("draining replica off a node being taken out of service",
"replica", rep.ID, "nodes", rep.NodeIDs())
for _, s := range shards[rep.ID] {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
r.drainShards(ctx, rep.ID, shards[rep.ID], "its node is being drained")
}
}
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
// failReplicasOn fails every replica on a node, and says which.
func (r *Reconciler) failReplicasOn(nodeID, reason string) []string {
r.mu.Lock()
defer r.mu.Unlock()
var ids []string
for _, rep := range r.replicas {
if rep.State.Terminal() || !rep.OnNode(nodeID) {
continue
}
rep.State = Failed
rep.Err = reason
rep.startDone(r.now())
ids = append(ids, rep.ID)
}
slices.Sort(ids)
return ids
}
// reapFailed releases the resources of terminal replicas and tells the agents
@@ -639,8 +656,10 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
doomed = append(doomed, rep)
}
}
parkedNow := map[string]bool{}
for _, rep := range doomed {
delete(r.replicas, rep.ID)
rep.startDone(r.now())
if rep.State == Failed {
// Carry the failure count on the model, not the replica: the
// replica is about to disappear, and the backoff has to survive it.
@@ -652,6 +671,7 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
r.parked[rep.Model] = parkRecord{
Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m),
}
parkedNow[rep.ID] = true
r.log.Error("model parked: it failed every time it started, and is not retried "+
"until its definition changes or `llmgw ctl retry` is run",
"model", rep.Model, "failures", n, "last_error", rep.Err)
@@ -667,23 +687,52 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
for _, rep := range doomed {
// rep is out of the map now, so no agent report can reach it any more.
sctx, span := startSpan(ctx, "reap replica", replicaAttrs(rep)...)
span.SetAttributes(attribute.String("llmgw.state", rep.State.String()))
if rep.State == Failed {
r.mu.Lock()
retry := r.backoff(r.attempts[rep.Model])
attempts := r.attempts[rep.Model]
retry := r.backoff(attempts)
r.mu.Unlock()
r.log.Error("replica failed",
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
"retry_in", retry.Round(time.Second))
span.SetAttributes(
attribute.String("llmgw.error", rep.Err),
attribute.Int("llmgw.failures_in_a_row", attempts),
attribute.Float64("llmgw.retry_in_seconds", retry.Seconds()),
attribute.Bool("llmgw.parked", parkedNow[rep.ID]),
)
}
// Stop every shard, including the ones that looked healthy: a partially
// alive replica holds GPUs while serving nothing.
for _, s := range rep.Shards {
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
if err := r.disp.StopShard(sctx, s.NodeID, s.ID); err != nil {
r.log.Warn("stopping a shard of a dead replica failed",
"replica", rep.ID, "shard", s.ID, "err", err)
}
}
r.inv.ReleaseReplica(rep.ID)
span.End()
}
}
// drainShards tells a replica's nodes to stop routing to its shards, as one
// "drain replica" span saying why.
func (r *Reconciler) drainShards(ctx context.Context, id string, shards []Shard, why string) {
nodes := make([]string, 0, len(shards))
for _, s := range shards {
nodes = append(nodes, s.NodeID)
}
ctx, span := startSpan(ctx, "drain replica",
attribute.String("llmgw.replica", id),
attribute.StringSlice("llmgw.nodes", nodes),
attribute.String("llmgw.reason", why))
defer span.End()
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
}
@@ -727,12 +776,14 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
}
}
// The next restart an operator queued, once the last one is done.
restarting := ""
if len(retiringIDs) == 0 {
if id, ok := r.nextRestartLocked(m.Name); ok {
r.replicas[id].RetiringAt = r.now()
retiringIDs = append(retiringIDs, id)
liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id })
readyLive--
restarting = id
r.log.Info("restarting a replica: placing its replacement first",
"replica", id, "model", m.Name)
}
@@ -749,6 +800,11 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
r.log.Info("model unparked: its definition changed", "model", m.Name)
}
r.mu.Unlock()
if restarting != "" {
_, span := startSpan(ctx, "restart replica",
attribute.String("llmgw.replica", restarting), attribute.String("llmgw.model", m.Name))
span.End()
}
desired := r.desiredFor(m)
outcome := notTried
@@ -765,9 +821,20 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// the real error in noise.
return
}
outcome = r.scaleUp(ctx, m, avoid)
sctx, span := startSpan(ctx, "scale up",
attribute.String("llmgw.model", m.Name),
attribute.Int("llmgw.replicas", len(liveIDs)),
attribute.Int("llmgw.desired", desired))
outcome = r.scaleUp(sctx, m, avoid)
span.SetAttributes(attribute.String("llmgw.outcome", outcome.String()))
span.End()
case len(liveIDs) > desired:
r.scaleDown(ctx, liveIDs[desired:])
sctx, span := startSpan(ctx, "scale down",
attribute.String("llmgw.model", m.Name),
attribute.Int("llmgw.replicas", len(liveIDs)),
attribute.Int("llmgw.desired", desired))
r.scaleDown(sctx, liveIDs[desired:], "more replicas than desired")
span.End()
}
// A drained replica is stopped once its replacement is serving, or once the
@@ -790,17 +857,17 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
switch {
case !ok:
case replaced:
r.stopReplica(ctx, id)
r.stopReplica(ctx, id, "drained, and its replacement serves")
case expired && preemptedFor != "":
r.log.Info("stopping a preempted replica", "replica", id,
"for", preemptedFor, "after", grace)
r.stopReplica(ctx, id)
r.stopReplica(ctx, id, "preempted for "+preemptedFor+", and its grace is over")
case expired:
// The replacement never arrived — often because nowhere else can
// host it. Draining forever would leave the node undrainable.
r.log.Warn("stopping a drained replica without a replacement",
"replica", id, "after", grace)
r.stopReplica(ctx, id)
r.stopReplica(ctx, id, "drained, and no replacement arrived within "+grace.String())
}
}
@@ -813,6 +880,9 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
// allocated, and starting several at once from the same inventory snapshot
// would hand them the same GPUs.
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome {
_, pspan := startSpan(ctx, "place",
attribute.String("llmgw.model", m.Name),
attribute.StringSlice("llmgw.avoid", avoid))
profiles := r.orderedProfiles(m)
res := r.sched.Place(profiles, avoid)
if res.Placement == nil && len(avoid) > 0 {
@@ -820,7 +890,10 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
// is the only thing standing in the way, take the placement anyway:
// a second replica on one node still survives an engine crash.
res = r.sched.Place(profiles, nil)
pspan.SetAttributes(attribute.Bool("llmgw.spread_given_up", res.Placement != nil))
}
placementAttrs(pspan, res)
pspan.End()
r.mu.Lock()
r.lastExplain[m.Name] = res
@@ -852,10 +925,12 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
{Profile: res.Placement.Profile.Name, Reason: why}}}
r.mu.Unlock()
r.notePlacementFailure(m.Name, why)
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.over_limit", why))
return overLimit
}
if err := r.startReplica(ctx, m, res.Placement); err != nil {
r.log.Error("starting replica failed", "model", m.Name, "err", err)
trace.SpanFromContext(ctx).SetStatus(codes.Error, err.Error())
return noRoom
}
return placed
@@ -872,6 +947,51 @@ const (
overLimit // there is room, but the owner's quota does not allow it
)
// String names an outcome for a trace.
func (o scaleOutcome) String() string {
switch o {
case placed:
return "placed"
case makingRoom:
return "making room"
case noRoom:
return "no room"
case overLimit:
return "over its quota"
default:
return "not tried"
}
}
// placementAttrs puts a placement's answer on its span: where it went, or
// every candidate it turned down and why, which is what `ctl
// explain-placement` would have printed at that moment.
func placementAttrs(span trace.Span, res placement.Result) {
span.SetAttributes(attribute.Int("llmgw.rejections", len(res.Rejections)))
pl := res.Placement
if pl == nil {
explain := res.Explain()
if len(explain) > 4096 {
explain = explain[:4096] + "…"
}
span.SetAttributes(attribute.String("llmgw.explain", explain))
span.SetStatus(codes.Error, "fits nowhere")
return
}
nodes := make([]string, 0, len(pl.Nodes))
for _, a := range pl.Nodes {
nodes = append(nodes, a.NodeID)
}
span.SetAttributes(
attribute.String("llmgw.profile", pl.Profile.Name),
attribute.String("llmgw.engine", pl.Profile.Engine),
attribute.String("llmgw.gpu_class", pl.Profile.GPUClass),
attribute.String("llmgw.pool", pl.Pool),
attribute.String("llmgw.fabric", pl.Fabric),
attribute.StringSlice("llmgw.nodes", nodes),
)
}
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
prof, ok := m.ProfileByName(pl.Profile.Name)
if !ok {
@@ -936,6 +1056,19 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
nodeIDs := rep.NodeIDs()
// Open until the replica serves, fails or is stopped, and the parent of
// every node's part of the start: the agents' spans arrive under it.
rctx, rspan := startSpan(ctx, "replica start", append(replicaAttrs(rep),
attribute.String("llmgw.engine", prof.Engine),
attribute.String("llmgw.gpu_class", prof.GPUClass),
attribute.String("llmgw.pool", pl.Pool),
attribute.String("llmgw.fabric", pl.Fabric),
attribute.String("llmgw.endpoint", rep.Endpoint),
attribute.Int("llmgw.attempt", attempt),
attribute.String("llmgw.startup_timeout", r.startupTimeout(rep).String()),
)...)
rep.startSpan, rep.startTrace = rspan, rspan.SpanContext()
r.mu.Lock()
r.replicas[id] = rep
r.mu.Unlock()
@@ -965,7 +1098,19 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
Fabric: pl.Fabric,
ProfileFP: prof.Fingerprint(),
}
if err := r.disp.StartShard(ctx, cmd); err != nil {
// The command, sent: its context travels with it to the agent.
sctx, sspan := tracer().Start(rctx, "send start",
trace.WithSpanKind(trace.SpanKindProducer),
trace.WithAttributes(
attribute.String("llmgw.shard", s.ID),
attribute.String("llmgw.node", s.NodeID),
attribute.Int("llmgw.rank", s.Rank),
attribute.IntSlice("llmgw.gpus", s.GPUIndices),
attribute.Int("llmgw.port", s.HTTPPort),
))
err := r.disp.StartShard(sctx, cmd)
obs.End(sspan, err)
if err != nil {
r.markShard(id, s.ID, supervisor.Failed, err.Error())
return fmt.Errorf("starting shard %s: %w", s.ID, err)
}
@@ -1031,7 +1176,8 @@ func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
return r.cfg.PortRange[0]
}
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
// scaleDown drains replicas; why is for the trace.
func (r *Reconciler) scaleDown(ctx context.Context, excess []string, why string) {
for _, id := range excess {
r.mu.Lock()
rep, ok := r.replicas[id]
@@ -1045,15 +1191,12 @@ func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
r.mu.Unlock()
r.log.Info("draining replica", "replica", id)
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
}
r.drainShards(ctx, id, shards, why)
}
}
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
// stopReplica stops a replica's shards and forgets it; why is for the trace.
func (r *Reconciler) stopReplica(ctx context.Context, id, why string) {
r.mu.Lock()
rep, ok := r.replicas[id]
if !ok {
@@ -1062,6 +1205,10 @@ func (r *Reconciler) stopReplica(ctx context.Context, id string) {
}
shards := slices.Clone(rep.Shards)
rep.State = Stopped
rep.startDone(r.now())
ctx, span := startSpan(ctx, "stop replica", append(replicaAttrs(rep),
attribute.String("llmgw.reason", why))...)
defer span.End()
r.mu.Unlock()
for _, s := range shards {
@@ -1130,6 +1277,18 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e
prev := rep.State
rep.deriveState(r.now())
if rep.startSpan != nil {
ev := []attribute.KeyValue{
attribute.String("llmgw.shard", shardID), attribute.String("llmgw.state", st.String()),
}
if errMsg != "" {
ev = append(ev, attribute.String("llmgw.error", errMsg))
}
rep.startSpan.AddEvent("shard "+st.String(), trace.WithAttributes(ev...))
}
if rep.State == Ready || rep.State.Terminal() {
rep.startDone(r.now())
}
if rep.State == Ready && prev != Ready {
// A model that comes up clears its backoff; the next failure starts
// counting from scratch rather than from an old streak.
@@ -11,6 +11,8 @@ import (
"testing"
"time"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
@@ -24,9 +26,12 @@ type fakeDispatcher struct {
stopped []string
drained []string
failNext error
// traces are the span each start was sent under, which is what the
// real dispatcher puts on the command for the agent.
traces []trace.SpanContext
}
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
func (d *fakeDispatcher) StartShard(ctx context.Context, cmd ShardCommand) error {
d.mu.Lock()
defer d.mu.Unlock()
if d.failNext != nil {
@@ -35,6 +40,7 @@ func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
return err
}
d.started = append(d.started, cmd)
d.traces = append(d.traces, trace.SpanContextFromContext(ctx))
return nil
}
+13
View File
@@ -7,6 +7,8 @@ import (
"slices"
"time"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
)
@@ -166,8 +168,19 @@ type Replica struct {
// carried across rebuilds so the backoff does not reset on every attempt.
Attempt int
Err string
// startSpan is the "replica start" span, open from placement until the
// replica serves, fails or is stopped; nil once closed, and for a replica
// adopted rather than started. startTrace names it for good.
startSpan trace.Span
startTrace trace.SpanContext
}
// StartTrace names the span of this replica's start, for something that
// waited on it — a job serving a model — to link to. Invalid for a replica
// adopted rather than started, and whenever tracing is off.
func (r Replica) StartTrace() trace.SpanContext { return r.startTrace }
// NodeIDs lists the nodes this replica occupies.
func (r Replica) NodeIDs() []string {
out := make([]string, 0, len(r.Shards))
+18 -15
View File
@@ -7,6 +7,8 @@ import (
"slices"
"time"
"go.opentelemetry.io/otel/attribute"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
)
@@ -48,39 +50,39 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string,
}
switch {
case readyLive >= desired:
r.drainForRestart(ctx, id)
r.drainForRestart(ctx, id, "restarted: its replacement serves")
case inPlace:
r.log.Warn("restarting a replica in place, as asked: stopped before its replacement starts", "replica", id)
r.stopForRestart(ctx, id)
r.stopForRestart(ctx, id, "restarted in place, as asked")
case live >= desired:
// A replacement is placed and loading; it is waited for.
case parked:
r.refuseRestart(id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
r.refuseRestart(ctx, id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
case outcome == noRoom && r.fitsWithout(m, nil, []victim{{ID: id, Model: m.Name}}):
r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way",
"replica", id, "model", m.Name)
r.stopForRestart(ctx, id)
r.stopForRestart(ctx, id, "restarted in place: it was the only thing in its replacement's way")
case outcome == noRoom || outcome == overLimit:
r.refuseRestart(id, "its replacement fits on no node even with this replica stopped "+
r.refuseRestart(ctx, id, "its replacement fits on no node even with this replica stopped "+
"(llmgw ctl explain-placement "+m.Name+")")
case outcome == notTried && r.now().Sub(since) > r.cfg.DrainGrace:
r.log.Warn("restarting a replica in place: no replacement beside it",
"replica", id, "after", r.cfg.DrainGrace)
r.stopForRestart(ctx, id)
r.stopForRestart(ctx, id, "restarted in place: no replacement beside it within "+r.cfg.DrainGrace.String())
}
}
}
// stopForRestart drains a replica and stops it at once, for its replacement
// to take its place.
func (r *Reconciler) stopForRestart(ctx context.Context, id string) {
r.drainForRestart(ctx, id)
r.stopReplica(ctx, id)
func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) {
r.drainForRestart(ctx, id, why)
r.stopReplica(ctx, id, why)
}
// refuseRestart gives up a replica's restart, leaving it serving, and keeps
// why for whoever looks at the replica.
func (r *Reconciler) refuseRestart(id, why string) {
func (r *Reconciler) refuseRestart(ctx context.Context, id, why string) {
r.mu.Lock()
if rep, ok := r.replicas[id]; ok {
rep.RetiringAt = time.Time{}
@@ -89,9 +91,12 @@ func (r *Reconciler) refuseRestart(id, why string) {
r.restartRefused[id] = why
r.mu.Unlock()
r.log.Warn("not restarting a replica: it keeps serving", "replica", id, "because", why)
_, span := startSpan(ctx, "restart refused",
attribute.String("llmgw.replica", id), attribute.String("llmgw.reason", why))
span.End()
}
func (r *Reconciler) drainForRestart(ctx context.Context, id string) {
func (r *Reconciler) drainForRestart(ctx context.Context, id, why string) {
r.mu.Lock()
rep, ok := r.replicas[id]
var shards []Shard
@@ -101,10 +106,8 @@ func (r *Reconciler) drainForRestart(ctx context.Context, id string) {
}
delete(r.restartInPlace, id)
r.mu.Unlock()
for _, s := range shards {
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
}
if ok {
r.drainShards(ctx, id, shards, why)
}
}
+122
View File
@@ -0,0 +1,122 @@
package reconciler
import (
"context"
"errors"
"sync"
"time"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
"go.opentelemetry.io/otel/trace"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
)
// What is traced here, and why it is shaped this way.
//
// A pass runs every few seconds and nearly always finds nothing to do. A
// trace per pass would bury the few that placed or stopped something under
// seventeen thousand empty ones a day, so a pass's span is begun only when
// the pass first does something, and backdated to when the pass began.
//
// A replica's start outlives the pass that placed it by minutes. Its span,
// "replica start", is opened beneath the placement and closed when the
// replica serves, fails, or is stopped, and the command to each node carries
// it, so the agent's spans for that shard — weights, launch, readiness — are
// its descendants. That one trace is the answer to "why did this model take
// four minutes to start".
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
func tracer() trace.Tracer { return obs.Tracer(tracerName) }
// pass is one reconcile pass's span, begun on first use.
type pass struct {
start time.Time
base context.Context
once sync.Once
ctx context.Context
span trace.Span
}
type passKey struct{}
// beginPass marks ctx as a pass's, for startSpan to hang its spans on.
func beginPass(ctx context.Context) (context.Context, *pass) {
p := &pass{start: time.Now(), base: ctx}
return context.WithValue(ctx, passKey{}, p), p
}
// context is the pass's span's context, beginning the span if it has not
// been.
func (p *pass) context() context.Context {
p.once.Do(func() {
p.ctx, p.span = tracer().Start(p.base, "reconcile", trace.WithTimestamp(p.start))
})
return p.ctx
}
// end closes the pass's span, if anything began it.
func (p *pass) end() {
p.once.Do(func() {}) // nothing may begin it after this
if p.span != nil {
p.span.End()
}
}
// startSpan opens a span for something the reconciler did: beneath the span
// already in ctx, or beneath the pass's — begun now — when ctx holds the
// pass and nothing else.
func startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) {
if !trace.SpanContextFromContext(ctx).IsValid() {
if p, ok := ctx.Value(passKey{}).(*pass); ok {
if pctx := p.context(); pctx != nil {
ctx = pctx
}
}
}
return tracer().Start(ctx, name, trace.WithAttributes(attrs...))
}
// endSpan closes a span, marking it failed when why is not empty.
func endSpan(span trace.Span, why string) {
if why != "" {
obs.End(span, errors.New(why))
return
}
span.End()
}
// replicaAttrs are what every span about one replica says.
func replicaAttrs(rep *Replica) []attribute.KeyValue {
return []attribute.KeyValue{
attribute.String("llmgw.replica", rep.ID),
attribute.String("llmgw.model", rep.Model),
attribute.String("llmgw.profile", rep.Profile),
attribute.StringSlice("llmgw.nodes", rep.NodeIDs()),
}
}
// startDone closes a replica's "replica start" span, once: when it serves,
// with how long that took, or when it fails or is stopped first, with why.
// Called with r.mu held, or before the replica is in the map.
func (rep *Replica) startDone(now time.Time) {
if rep.startSpan == nil {
return
}
span := rep.startSpan
rep.startSpan = nil
span.SetAttributes(attribute.String("llmgw.state", rep.State.String()))
switch rep.State {
case Ready:
span.SetAttributes(attribute.Float64("llmgw.ready_after_seconds", now.Sub(rep.CreatedAt).Seconds()))
span.SetStatus(codes.Ok, "")
case Failed:
span.SetStatus(codes.Error, rep.Err)
default:
span.SetStatus(codes.Error, "stopped before it was ready")
}
span.End()
}
+174
View File
@@ -0,0 +1,174 @@
package reconciler
import (
"errors"
"strings"
"testing"
"go.opentelemetry.io/otel/codes"
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
)
func TestAPlacementIsTracedFromThePassToTheCommandAndTheReplicaServing(t *testing.T) {
rec := obstest.Record(t)
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
pass := rec.One(t, [16]byte{}, "reconcile")
tid := pass.SpanContext().TraceID()
up := rec.One(t, tid, "scale up")
place := rec.One(t, tid, "place")
if !obstest.ChildOf(up, pass) || !obstest.ChildOf(place, up) {
t.Fatal("the placement is not beneath the scale-up, beneath the pass")
}
if a := obstest.Attrs(place); a["llmgw.model"] != "llama" || a["llmgw.profile"] != "tp4" ||
a["llmgw.gpu_class"] != "h100-80g" || a["llmgw.nodes"] != `["1"]` {
t.Errorf("the placement does not say where it went: %v", a)
}
if a := obstest.Attrs(up); a["llmgw.outcome"] != "placed" || a["llmgw.desired"] != "1" {
t.Errorf("the scale-up does not say what it did: %v", a)
}
// The command went under a span of its own, beneath the replica's start,
// which is still open: the replica is loading.
send := rec.One(t, tid, "send start")
if got := h.disp.traces; len(got) != 1 || got[0].SpanID() != send.SpanContext().SpanID() {
t.Fatalf("the command was not sent under its span: %v", got)
}
if len(rec.Named("replica start")) != 0 {
t.Fatal("the replica's start ended before the replica served")
}
h.markReady()
start := rec.One(t, tid, "replica start")
if !obstest.ChildOf(start, up) || !obstest.ChildOf(send, start) {
t.Error("the replica's start is not beneath the scale-up, or the command beneath it")
}
a := obstest.Attrs(start)
if a["llmgw.state"] != "ready" || a["llmgw.profile"] != "tp4" || a["llmgw.ready_after_seconds"] == "" {
t.Errorf("the replica's start does not say how it ended: %v", a)
}
if start.Status().Code != codes.Ok {
t.Errorf("a replica that serves is %v", start.Status())
}
events := start.Events()
if len(events) == 0 || events[len(events)-1].Name != "shard ready" {
t.Errorf("the shard's progress is not on the replica's start: %v", events)
}
if sc := h.r.Replicas()[0].StartTrace(); sc.SpanID() != start.SpanContext().SpanID() {
t.Error("the replica does not name its start's span for a job to link to")
}
}
func TestAPassThatDoesNothingLeavesNoTrace(t *testing.T) {
rec := obstest.Record(t)
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
h.markReady()
before := len(rec.Ended())
// Every pass from here finds the model as it should be.
for range 3 {
h.r.Reconcile(t.Context())
}
if got := rec.Ended()[before:]; len(got) != 0 {
var names []string
for _, s := range got {
names = append(names, s.Name())
}
t.Errorf("passes with nothing to do were traced: %v", names)
}
}
func TestAPlacementThatFitsNowhereSaysWhy(t *testing.T) {
rec := obstest.Record(t)
m := testModel(1, catalog.Profile{Name: "tp16", GPUClass: "h100-80g", TP: 16})
h := newHarness(t, m, gpuNode("1", 8))
h.r.Reconcile(t.Context())
place := rec.One(t, [16]byte{}, "place")
if place.Status().Code != codes.Error {
t.Errorf("a placement that found nowhere is %v", place.Status())
}
if a := obstest.Attrs(place); a["llmgw.rejections"] == "0" || !strings.Contains(a["llmgw.explain"], "tp16") {
t.Errorf("the placement does not say what it turned down: %v", a)
}
if a := obstest.Attrs(rec.One(t, place.SpanContext().TraceID(), "scale up")); a["llmgw.outcome"] != "no room" {
t.Errorf("the scale-up says %v", a)
}
}
func TestAFailedStartEndsItsTraceWithTheEnginesWords(t *testing.T) {
rec := obstest.Record(t)
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
cmd := h.disp.startedShards()[0]
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Failed, "CUDA out of memory")
start := rec.One(t, [16]byte{}, "replica start")
if start.Status().Code != codes.Error || !strings.Contains(start.Status().Description, "CUDA out of memory") {
t.Errorf("the failed start says %+v", start.Status())
}
// The next pass reaps it, in a trace of its own, saying when it is tried
// again.
h.r.Reconcile(t.Context())
reap := rec.One(t, [16]byte{}, "reap replica")
a := obstest.Attrs(reap)
if a["llmgw.replica"] != cmd.ReplicaID || a["llmgw.failures_in_a_row"] != "1" || a["llmgw.retry_in_seconds"] == "" {
t.Errorf("the reaping does not say what happens next: %v", a)
}
}
func TestADispatchThatFailsIsTracedAsFailed(t *testing.T) {
rec := obstest.Record(t)
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.disp.failNext = errors.New("no agent connected")
h.r.Reconcile(t.Context())
send := rec.One(t, [16]byte{}, "send start")
if send.Status().Code != codes.Error {
t.Errorf("a command that was not sent is %v", send.Status())
}
if start := rec.One(t, [16]byte{}, "replica start"); start.Status().Code != codes.Error {
t.Errorf("a replica none of whose shards was sent is %v", start.Status())
}
}
func TestAPreemptionIsTracedWithItsVictims(t *testing.T) {
h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8))
rec := obstest.Record(t)
h.r.Reconcile(t.Context())
pre := rec.One(t, [16]byte{}, "preempt")
a := obstest.Attrs(pre)
if a["llmgw.for"] != "vital" || !strings.Contains(a["llmgw.victims"], "chat") {
t.Errorf("the preemption does not say who gave way to whom: %v", a)
}
drain := rec.One(t, pre.SpanContext().TraceID(), "drain replica")
if !obstest.ChildOf(drain, pre) || obstest.Attrs(drain)["llmgw.reason"] != "preempted for vital" {
t.Errorf("the victim's drain is not beneath the preemption: %v", obstest.Attrs(drain))
}
}
func TestAStopSaysWhy(t *testing.T) {
h := newHarness(t, testModel(1), gpuNode("1", 8))
h.r.Reconcile(t.Context())
h.markReady()
rec := obstest.Record(t)
h.setModels(t) // the model leaves the catalog
h.r.Reconcile(t.Context())
stop := rec.One(t, [16]byte{}, "stop replica")
if a := obstest.Attrs(stop); a["llmgw.model"] != "llama" || a["llmgw.reason"] != "its model left the catalog" {
t.Errorf("the stop does not say why: %v", a)
}
}