Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
e725adc2f0
33 files changed
+1607
-143
No files matched your search
@@ -14,6 +14,8 @@ import (
|
||||
"time"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
|
||||
@@ -22,6 +24,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// StatusSink receives shard transitions and recovers replica state from what
|
||||
@@ -311,7 +314,20 @@ func (s *Service) Connect(
|
||||
// unreachable in production while its unit tests, which call the reconciler
|
||||
// directly, kept passing.
|
||||
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
|
||||
// A trace of its own, and only for a node that reported something: a
|
||||
// node running nothing registering is not an event anyone looks for.
|
||||
tctx, span := ctx, trace.SpanFromContext(context.Background()) // the no-op span
|
||||
if len(shards) > 0 {
|
||||
tctx, span = obs.Tracer(tracerName).Start(ctx, "adopt",
|
||||
trace.WithNewRoot(),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", nodeID),
|
||||
attribute.Int("llmgw.shards", len(shards)),
|
||||
))
|
||||
}
|
||||
defer span.End()
|
||||
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
|
||||
span.SetAttributes(attribute.Int("llmgw.unclaimed", len(unclaimed)))
|
||||
for _, u := range unclaimed {
|
||||
// The model or profile left the catalog while this was running. Nobody
|
||||
// will ever route to it, and it is holding GPUs.
|
||||
@@ -319,7 +335,8 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
|
||||
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
|
||||
select {
|
||||
case c.send <- &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
|
||||
Trace: obs.Inject(tctx),
|
||||
}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -715,27 +732,37 @@ type Leader interface{ IsLeader() bool }
|
||||
// and what every deployment before leadership had.
|
||||
func (s *Service) SetLeadership(l Leader) { s.leader = l }
|
||||
|
||||
// tracerName names the spans this package opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
|
||||
// ErrNotConnected means no agent stream is open for that node.
|
||||
var ErrNotConnected = errors.New("no agent connected")
|
||||
|
||||
// StartShard implements reconciler.Dispatcher.
|
||||
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
|
||||
//
|
||||
// The command carries the trace of ctx, so the agent's spans for this start
|
||||
// — the weights, the engine's launch, its readiness — are in the trace of
|
||||
// the replica they are starting rather than in one of their own.
|
||||
func (s *Service) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
|
||||
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
// StopShard implements reconciler.Dispatcher.
|
||||
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
|
||||
func (s *Service) StopShard(ctx context.Context, nodeID, shardID string) error {
|
||||
return s.send(nodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
// DrainShard implements reconciler.Dispatcher.
|
||||
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
|
||||
func (s *Service) DrainShard(ctx context.Context, nodeID, shardID string) error {
|
||||
return s.send(nodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -172,6 +172,9 @@ type Job struct {
|
||||
// how fast it said it.
|
||||
Reply string `json:"reply,omitempty"`
|
||||
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
|
||||
|
||||
// trace is the job's spans, when tracing is on.
|
||||
trace jobTrace
|
||||
}
|
||||
|
||||
// Server runs jobs and remembers them.
|
||||
@@ -318,6 +321,7 @@ func (s *Server) snapshot(j *Job) Job {
|
||||
func (s *Server) step(j *Job, name, state, detail string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
j.traceStep(name, state, detail)
|
||||
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
|
||||
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
|
||||
return
|
||||
@@ -329,6 +333,7 @@ func (s *Server) fail(j *Job, name string, err error) {
|
||||
s.step(j, name, Failed, err.Error())
|
||||
s.mu.Lock()
|
||||
j.State, j.Error = Failed, err.Error()
|
||||
j.traceEnd(err)
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
|
||||
}
|
||||
@@ -617,6 +622,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
func (s *Server) finish(j *Job) {
|
||||
s.mu.Lock()
|
||||
j.State = Done
|
||||
j.traceEnd(nil)
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
|
||||
}
|
||||
@@ -785,6 +791,7 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
|
||||
continue
|
||||
}
|
||||
if r.State == reconciler.Ready {
|
||||
s.linkReplica(j, r)
|
||||
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
|
||||
strings.Join(r.NodeIDs(), ","), r.Endpoint))
|
||||
return r, nil
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// A job is a trace of its own, and each of its steps a span beneath it —
|
||||
// registry, mirror, check, catalog, placement, smoke test — opened when the
|
||||
// step begins and closed when it is done, failed or skipped. The steps are
|
||||
// what a job already records for `ctl model jobs`; the trace adds where the
|
||||
// time went, and from the placement step a link to the replica's own start,
|
||||
// whose trace goes on down to the node.
|
||||
//
|
||||
// Begun by the first step rather than where a job is made, so every kind of
|
||||
// job is traced by recording steps and nothing else.
|
||||
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
|
||||
// jobTrace is a job's spans. Read and written with Server.mu held.
|
||||
type jobTrace struct {
|
||||
span trace.Span
|
||||
step trace.Span
|
||||
stepName string
|
||||
}
|
||||
|
||||
// traceStep records a step on the job's trace. Called with s.mu held.
|
||||
func (j *Job) traceStep(name, state, detail string) {
|
||||
if j.trace.span == nil {
|
||||
kind := j.Kind
|
||||
if kind == "" {
|
||||
kind = "serve"
|
||||
}
|
||||
_, j.trace.span = obs.Tracer(tracerName).Start(context.Background(), kind+" model",
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.job", j.ID),
|
||||
attribute.String("llmgw.repo", j.Request.Repo),
|
||||
attribute.String("llmgw.model", j.Model),
|
||||
))
|
||||
}
|
||||
if j.trace.step != nil && j.trace.stepName != name {
|
||||
// A step left running when the next began: over, whatever it said.
|
||||
j.trace.step.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
if j.trace.step == nil {
|
||||
ctx := trace.ContextWithSpan(context.Background(), j.trace.span)
|
||||
_, j.trace.step = obs.Tracer(tracerName).Start(ctx, name)
|
||||
j.trace.stepName = name
|
||||
}
|
||||
span := j.trace.step
|
||||
span.SetAttributes(attribute.String("llmgw.state", state), attribute.String("llmgw.detail", detail))
|
||||
switch state {
|
||||
case Running:
|
||||
return
|
||||
case Failed:
|
||||
span.SetStatus(codes.Error, detail)
|
||||
}
|
||||
span.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
|
||||
// traceEnd closes the job's trace, failed when err is set. Called with s.mu
|
||||
// held.
|
||||
func (j *Job) traceEnd(err error) {
|
||||
if j.trace.step != nil {
|
||||
j.trace.step.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
if j.trace.span == nil {
|
||||
return
|
||||
}
|
||||
if j.Model != "" {
|
||||
j.trace.span.SetAttributes(attribute.String("llmgw.model", j.Model))
|
||||
}
|
||||
obs.End(j.trace.span, err)
|
||||
j.trace.span = nil
|
||||
}
|
||||
|
||||
// linkReplica links the step in progress to the start of a replica it waited
|
||||
// for, which is in the reconciler's trace rather than this one.
|
||||
func (s *Server) linkReplica(j *Job, r reconciler.Replica) {
|
||||
sc := r.StartTrace()
|
||||
if !sc.IsValid() {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if j.trace.step != nil {
|
||||
j.trace.step.AddLink(trace.Link{SpanContext: sc,
|
||||
Attributes: []attribute.KeyValue{attribute.String("llmgw.replica", r.ID)}})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAJobIsATraceWithASpanPerStep(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
files := []hf.File{{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}}
|
||||
s, _, _, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
|
||||
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("job %s: %s", j.State, j.Error)
|
||||
}
|
||||
|
||||
root := rec.One(t, [16]byte{}, "serve model")
|
||||
tid := root.SpanContext().TraceID()
|
||||
if a := obstest.Attrs(root); a["llmgw.job"] != job.ID || a["llmgw.model"] != "qwen2.5-1.5b-instruct" {
|
||||
t.Errorf("the job's span does not say which job: %v", a)
|
||||
}
|
||||
var steps []string
|
||||
for _, sp := range rec.Ended() {
|
||||
if sp.SpanContext().TraceID() == tid && obstest.ChildOf(sp, root) {
|
||||
steps = append(steps, sp.Name())
|
||||
}
|
||||
}
|
||||
if strings.Join(steps, ",") != "registry,mirror,check,catalog,placement,smoke test" {
|
||||
t.Errorf("step spans %v", steps)
|
||||
}
|
||||
smoke := rec.One(t, tid, "smoke test")
|
||||
if a := obstest.Attrs(smoke); a["llmgw.state"] != Done || !strings.Contains(a["llmgw.detail"], "Hello!") {
|
||||
t.Errorf("the smoke test's span says %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedJobsTraceEndsAtTheStepThatFailed(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
|
||||
s, _, _, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
|
||||
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||
if j := wait(t, s, job.ID); j.State != Failed {
|
||||
t.Fatalf("job %s", j.State)
|
||||
}
|
||||
|
||||
root := rec.One(t, [16]byte{}, "serve model")
|
||||
if root.Status().Code != codes.Error || !strings.Contains(root.Status().Description, "registry published") {
|
||||
t.Errorf("the failed job's span says %+v", root.Status())
|
||||
}
|
||||
mirror := rec.One(t, root.SpanContext().TraceID(), "mirror")
|
||||
if mirror.Status().Code != codes.Error {
|
||||
t.Errorf("the step that failed is %+v", mirror.Status())
|
||||
}
|
||||
if len(rec.Named("catalog")) != 0 {
|
||||
t.Error("a step after the failure was traced")
|
||||
}
|
||||
}
|
||||
@@ -609,6 +609,9 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
|
||||
}
|
||||
}
|
||||
if left == 0 && len(ready) > 0 {
|
||||
for _, r := range ready {
|
||||
s.linkReplica(j, r)
|
||||
}
|
||||
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
|
||||
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
|
||||
reply, tps, err := s.smoke(ctx, ready[0], name)
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
@@ -53,7 +55,15 @@ func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
|
||||
r.log.Info("evicting an idle replica to make room",
|
||||
"stopping", best.ID, "idle_model", best.Model,
|
||||
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
|
||||
r.stopReplica(ctx, best.ID)
|
||||
ctx, span := startSpan(ctx, "evict",
|
||||
attribute.String("llmgw.for", m.Name),
|
||||
attribute.String("llmgw.replica", best.ID),
|
||||
attribute.String("llmgw.model", best.Model),
|
||||
attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
|
||||
attribute.Int("llmgw.gpus", best.GPUs),
|
||||
attribute.Int("llmgw.candidates", len(candidates)))
|
||||
defer span.End()
|
||||
r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,13 @@ package reconciler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||
@@ -113,12 +116,24 @@ func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []vict
|
||||
r.preempting[m.Name] = ids
|
||||
r.mu.Unlock()
|
||||
|
||||
victims := make([]string, 0, len(chosen))
|
||||
for _, v := range chosen {
|
||||
r.log.Warn("preempting a replica to make room",
|
||||
"replica", v.ID, "model", v.Model, "class", v.Class, "above_min", v.AboveMin,
|
||||
"for", m.Name, "for_class", m.EffectiveClass())
|
||||
desc := fmt.Sprintf("%s (%s, %s)", v.ID, v.Model, v.Class)
|
||||
if v.AboveMin {
|
||||
desc = fmt.Sprintf("%s (%s, %s, above its minimum)", v.ID, v.Model, v.Class)
|
||||
}
|
||||
victims = append(victims, desc)
|
||||
}
|
||||
r.scaleDown(ctx, ids)
|
||||
ctx, span := startSpan(ctx, "preempt",
|
||||
attribute.String("llmgw.for", m.Name),
|
||||
attribute.String("llmgw.for_class", m.EffectiveClass()),
|
||||
attribute.StringSlice("llmgw.victims", victims),
|
||||
attribute.String("llmgw.hold", r.cfg.PreemptHold.String()))
|
||||
defer span.End()
|
||||
r.scaleDown(ctx, ids, "preempted for "+m.Name)
|
||||
}
|
||||
|
||||
// awaitingPreemption reports whether replicas stopped for a model are still
|
||||
|
||||
@@ -11,6 +11,10 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
@@ -18,6 +22,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// ShardCommand is everything an agent needs to launch one shard.
|
||||
@@ -402,7 +407,9 @@ func (r *Reconciler) Reconcile(ctx context.Context) {
|
||||
if r.leader != nil && !r.leader.IsLeader() {
|
||||
return
|
||||
}
|
||||
r.expireNodes()
|
||||
ctx, p := beginPass(ctx)
|
||||
defer p.end()
|
||||
r.expireNodes(ctx)
|
||||
r.expireStuckReplicas()
|
||||
r.drainCordonedNodes(ctx)
|
||||
r.reapFailed(ctx)
|
||||
@@ -449,7 +456,7 @@ func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
|
||||
for _, rep := range gone {
|
||||
r.log.Info("stopping a replica whose model left the catalog",
|
||||
"replica", rep.ID, "model", rep.Model)
|
||||
r.stopReplica(ctx, rep.ID)
|
||||
r.stopReplica(ctx, rep.ID, "its model left the catalog")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -520,6 +527,7 @@ func (r *Reconciler) expireStuckReplicas() {
|
||||
}
|
||||
rep.State = Failed
|
||||
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
|
||||
rep.startDone(now)
|
||||
stuck = append(stuck, rep)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
@@ -552,7 +560,7 @@ func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
|
||||
// The two-stage delay is deliberate: a brief network blip must not tear down
|
||||
// replicas that are still serving traffic perfectly well, and rebuilding a
|
||||
// multi-node replica costs tens of minutes of model loading.
|
||||
func (r *Reconciler) expireNodes() {
|
||||
func (r *Reconciler) expireNodes(ctx context.Context) {
|
||||
now := r.now()
|
||||
for _, n := range r.inv.Nodes() {
|
||||
silent := now.Sub(n.LastSeen)
|
||||
@@ -560,14 +568,21 @@ func (r *Reconciler) expireNodes() {
|
||||
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
|
||||
r.inv.SetState(n.ID, inventory.Unreachable)
|
||||
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
|
||||
_, span := startSpan(ctx, "node unreachable", attribute.String("llmgw.node", n.ID),
|
||||
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
|
||||
span.End()
|
||||
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
|
||||
_, span := startSpan(ctx, "node lost", attribute.String("llmgw.node", n.ID),
|
||||
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
|
||||
r.inv.SetState(n.ID, inventory.Lost)
|
||||
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
|
||||
"node", n.ID, "silent_for", silent.Round(time.Second))
|
||||
r.notify.Load().Notify(notify.NodeLost, n.ID,
|
||||
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
|
||||
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
|
||||
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
|
||||
span.SetAttributes(attribute.StringSlice("llmgw.replicas_failed",
|
||||
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")))
|
||||
span.End()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -606,24 +621,26 @@ func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
|
||||
for _, rep := range toDrain {
|
||||
r.log.Info("draining replica off a node being taken out of service",
|
||||
"replica", rep.ID, "nodes", rep.NodeIDs())
|
||||
for _, s := range shards[rep.ID] {
|
||||
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
||||
}
|
||||
}
|
||||
r.drainShards(ctx, rep.ID, shards[rep.ID], "its node is being drained")
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
|
||||
// failReplicasOn fails every replica on a node, and says which.
|
||||
func (r *Reconciler) failReplicasOn(nodeID, reason string) []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var ids []string
|
||||
for _, rep := range r.replicas {
|
||||
if rep.State.Terminal() || !rep.OnNode(nodeID) {
|
||||
continue
|
||||
}
|
||||
rep.State = Failed
|
||||
rep.Err = reason
|
||||
rep.startDone(r.now())
|
||||
ids = append(ids, rep.ID)
|
||||
}
|
||||
slices.Sort(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
// reapFailed releases the resources of terminal replicas and tells the agents
|
||||
@@ -636,8 +653,10 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
doomed = append(doomed, rep)
|
||||
}
|
||||
}
|
||||
parkedNow := map[string]bool{}
|
||||
for _, rep := range doomed {
|
||||
delete(r.replicas, rep.ID)
|
||||
rep.startDone(r.now())
|
||||
if rep.State == Failed {
|
||||
// Carry the failure count on the model, not the replica: the
|
||||
// replica is about to disappear, and the backoff has to survive it.
|
||||
@@ -649,6 +668,7 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
r.parked[rep.Model] = parkRecord{
|
||||
Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m),
|
||||
}
|
||||
parkedNow[rep.ID] = true
|
||||
r.log.Error("model parked: it failed every time it started, and is not retried "+
|
||||
"until its definition changes or `llmgw ctl retry` is run",
|
||||
"model", rep.Model, "failures", n, "last_error", rep.Err)
|
||||
@@ -664,23 +684,52 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
|
||||
for _, rep := range doomed {
|
||||
// rep is out of the map now, so no agent report can reach it any more.
|
||||
sctx, span := startSpan(ctx, "reap replica", replicaAttrs(rep)...)
|
||||
span.SetAttributes(attribute.String("llmgw.state", rep.State.String()))
|
||||
if rep.State == Failed {
|
||||
r.mu.Lock()
|
||||
retry := r.backoff(r.attempts[rep.Model])
|
||||
attempts := r.attempts[rep.Model]
|
||||
retry := r.backoff(attempts)
|
||||
r.mu.Unlock()
|
||||
r.log.Error("replica failed",
|
||||
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
|
||||
"retry_in", retry.Round(time.Second))
|
||||
span.SetAttributes(
|
||||
attribute.String("llmgw.error", rep.Err),
|
||||
attribute.Int("llmgw.failures_in_a_row", attempts),
|
||||
attribute.Float64("llmgw.retry_in_seconds", retry.Seconds()),
|
||||
attribute.Bool("llmgw.parked", parkedNow[rep.ID]),
|
||||
)
|
||||
}
|
||||
// Stop every shard, including the ones that looked healthy: a partially
|
||||
// alive replica holds GPUs while serving nothing.
|
||||
for _, s := range rep.Shards {
|
||||
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
if err := r.disp.StopShard(sctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("stopping a shard of a dead replica failed",
|
||||
"replica", rep.ID, "shard", s.ID, "err", err)
|
||||
}
|
||||
}
|
||||
r.inv.ReleaseReplica(rep.ID)
|
||||
span.End()
|
||||
}
|
||||
}
|
||||
|
||||
// drainShards tells a replica's nodes to stop routing to its shards, as one
|
||||
// "drain replica" span saying why.
|
||||
func (r *Reconciler) drainShards(ctx context.Context, id string, shards []Shard, why string) {
|
||||
nodes := make([]string, 0, len(shards))
|
||||
for _, s := range shards {
|
||||
nodes = append(nodes, s.NodeID)
|
||||
}
|
||||
ctx, span := startSpan(ctx, "drain replica",
|
||||
attribute.String("llmgw.replica", id),
|
||||
attribute.StringSlice("llmgw.nodes", nodes),
|
||||
attribute.String("llmgw.reason", why))
|
||||
defer span.End()
|
||||
for _, s := range shards {
|
||||
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -724,12 +773,14 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
}
|
||||
}
|
||||
// The next restart an operator queued, once the last one is done.
|
||||
restarting := ""
|
||||
if len(retiringIDs) == 0 {
|
||||
if id, ok := r.nextRestartLocked(m.Name); ok {
|
||||
r.replicas[id].RetiringAt = r.now()
|
||||
retiringIDs = append(retiringIDs, id)
|
||||
liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id })
|
||||
readyLive--
|
||||
restarting = id
|
||||
r.log.Info("restarting a replica: placing its replacement first",
|
||||
"replica", id, "model", m.Name)
|
||||
}
|
||||
@@ -746,6 +797,11 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
r.log.Info("model unparked: its definition changed", "model", m.Name)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if restarting != "" {
|
||||
_, span := startSpan(ctx, "restart replica",
|
||||
attribute.String("llmgw.replica", restarting), attribute.String("llmgw.model", m.Name))
|
||||
span.End()
|
||||
}
|
||||
|
||||
desired := r.desiredFor(m)
|
||||
outcome := notTried
|
||||
@@ -762,9 +818,20 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
// the real error in noise.
|
||||
return
|
||||
}
|
||||
outcome = r.scaleUp(ctx, m, avoid)
|
||||
sctx, span := startSpan(ctx, "scale up",
|
||||
attribute.String("llmgw.model", m.Name),
|
||||
attribute.Int("llmgw.replicas", len(liveIDs)),
|
||||
attribute.Int("llmgw.desired", desired))
|
||||
outcome = r.scaleUp(sctx, m, avoid)
|
||||
span.SetAttributes(attribute.String("llmgw.outcome", outcome.String()))
|
||||
span.End()
|
||||
case len(liveIDs) > desired:
|
||||
r.scaleDown(ctx, liveIDs[desired:])
|
||||
sctx, span := startSpan(ctx, "scale down",
|
||||
attribute.String("llmgw.model", m.Name),
|
||||
attribute.Int("llmgw.replicas", len(liveIDs)),
|
||||
attribute.Int("llmgw.desired", desired))
|
||||
r.scaleDown(sctx, liveIDs[desired:], "more replicas than desired")
|
||||
span.End()
|
||||
}
|
||||
|
||||
// A drained replica is stopped once its replacement is serving, or once the
|
||||
@@ -787,17 +854,17 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
switch {
|
||||
case !ok:
|
||||
case replaced:
|
||||
r.stopReplica(ctx, id)
|
||||
r.stopReplica(ctx, id, "drained, and its replacement serves")
|
||||
case expired && preemptedFor != "":
|
||||
r.log.Info("stopping a preempted replica", "replica", id,
|
||||
"for", preemptedFor, "after", grace)
|
||||
r.stopReplica(ctx, id)
|
||||
r.stopReplica(ctx, id, "preempted for "+preemptedFor+", and its grace is over")
|
||||
case expired:
|
||||
// The replacement never arrived — often because nowhere else can
|
||||
// host it. Draining forever would leave the node undrainable.
|
||||
r.log.Warn("stopping a drained replica without a replacement",
|
||||
"replica", id, "after", grace)
|
||||
r.stopReplica(ctx, id)
|
||||
r.stopReplica(ctx, id, "drained, and no replacement arrived within "+grace.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -810,6 +877,9 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
// allocated, and starting several at once from the same inventory snapshot
|
||||
// would hand them the same GPUs.
|
||||
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome {
|
||||
_, pspan := startSpan(ctx, "place",
|
||||
attribute.String("llmgw.model", m.Name),
|
||||
attribute.StringSlice("llmgw.avoid", avoid))
|
||||
profiles := r.orderedProfiles(m)
|
||||
res := r.sched.Place(profiles, avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
@@ -817,7 +887,10 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
|
||||
// is the only thing standing in the way, take the placement anyway:
|
||||
// a second replica on one node still survives an engine crash.
|
||||
res = r.sched.Place(profiles, nil)
|
||||
pspan.SetAttributes(attribute.Bool("llmgw.spread_given_up", res.Placement != nil))
|
||||
}
|
||||
placementAttrs(pspan, res)
|
||||
pspan.End()
|
||||
|
||||
r.mu.Lock()
|
||||
r.lastExplain[m.Name] = res
|
||||
@@ -849,10 +922,12 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
|
||||
{Profile: res.Placement.Profile.Name, Reason: why}}}
|
||||
r.mu.Unlock()
|
||||
r.notePlacementFailure(m.Name, why)
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.over_limit", why))
|
||||
return overLimit
|
||||
}
|
||||
if err := r.startReplica(ctx, m, res.Placement); err != nil {
|
||||
r.log.Error("starting replica failed", "model", m.Name, "err", err)
|
||||
trace.SpanFromContext(ctx).SetStatus(codes.Error, err.Error())
|
||||
return noRoom
|
||||
}
|
||||
return placed
|
||||
@@ -869,6 +944,51 @@ const (
|
||||
overLimit // there is room, but the owner's quota does not allow it
|
||||
)
|
||||
|
||||
// String names an outcome for a trace.
|
||||
func (o scaleOutcome) String() string {
|
||||
switch o {
|
||||
case placed:
|
||||
return "placed"
|
||||
case makingRoom:
|
||||
return "making room"
|
||||
case noRoom:
|
||||
return "no room"
|
||||
case overLimit:
|
||||
return "over its quota"
|
||||
default:
|
||||
return "not tried"
|
||||
}
|
||||
}
|
||||
|
||||
// placementAttrs puts a placement's answer on its span: where it went, or
|
||||
// every candidate it turned down and why, which is what `ctl
|
||||
// explain-placement` would have printed at that moment.
|
||||
func placementAttrs(span trace.Span, res placement.Result) {
|
||||
span.SetAttributes(attribute.Int("llmgw.rejections", len(res.Rejections)))
|
||||
pl := res.Placement
|
||||
if pl == nil {
|
||||
explain := res.Explain()
|
||||
if len(explain) > 4096 {
|
||||
explain = explain[:4096] + "…"
|
||||
}
|
||||
span.SetAttributes(attribute.String("llmgw.explain", explain))
|
||||
span.SetStatus(codes.Error, "fits nowhere")
|
||||
return
|
||||
}
|
||||
nodes := make([]string, 0, len(pl.Nodes))
|
||||
for _, a := range pl.Nodes {
|
||||
nodes = append(nodes, a.NodeID)
|
||||
}
|
||||
span.SetAttributes(
|
||||
attribute.String("llmgw.profile", pl.Profile.Name),
|
||||
attribute.String("llmgw.engine", pl.Profile.Engine),
|
||||
attribute.String("llmgw.gpu_class", pl.Profile.GPUClass),
|
||||
attribute.String("llmgw.pool", pl.Pool),
|
||||
attribute.String("llmgw.fabric", pl.Fabric),
|
||||
attribute.StringSlice("llmgw.nodes", nodes),
|
||||
)
|
||||
}
|
||||
|
||||
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
|
||||
prof, ok := m.ProfileByName(pl.Profile.Name)
|
||||
if !ok {
|
||||
@@ -933,6 +1053,19 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
|
||||
nodeIDs := rep.NodeIDs()
|
||||
|
||||
// Open until the replica serves, fails or is stopped, and the parent of
|
||||
// every node's part of the start: the agents' spans arrive under it.
|
||||
rctx, rspan := startSpan(ctx, "replica start", append(replicaAttrs(rep),
|
||||
attribute.String("llmgw.engine", prof.Engine),
|
||||
attribute.String("llmgw.gpu_class", prof.GPUClass),
|
||||
attribute.String("llmgw.pool", pl.Pool),
|
||||
attribute.String("llmgw.fabric", pl.Fabric),
|
||||
attribute.String("llmgw.endpoint", rep.Endpoint),
|
||||
attribute.Int("llmgw.attempt", attempt),
|
||||
attribute.String("llmgw.startup_timeout", r.startupTimeout(rep).String()),
|
||||
)...)
|
||||
rep.startSpan, rep.startTrace = rspan, rspan.SpanContext()
|
||||
|
||||
r.mu.Lock()
|
||||
r.replicas[id] = rep
|
||||
r.mu.Unlock()
|
||||
@@ -962,7 +1095,19 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
Fabric: pl.Fabric,
|
||||
ProfileFP: prof.Fingerprint(),
|
||||
}
|
||||
if err := r.disp.StartShard(ctx, cmd); err != nil {
|
||||
// The command, sent: its context travels with it to the agent.
|
||||
sctx, sspan := tracer().Start(rctx, "send start",
|
||||
trace.WithSpanKind(trace.SpanKindProducer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", s.ID),
|
||||
attribute.String("llmgw.node", s.NodeID),
|
||||
attribute.Int("llmgw.rank", s.Rank),
|
||||
attribute.IntSlice("llmgw.gpus", s.GPUIndices),
|
||||
attribute.Int("llmgw.port", s.HTTPPort),
|
||||
))
|
||||
err := r.disp.StartShard(sctx, cmd)
|
||||
obs.End(sspan, err)
|
||||
if err != nil {
|
||||
r.markShard(id, s.ID, supervisor.Failed, err.Error())
|
||||
return fmt.Errorf("starting shard %s: %w", s.ID, err)
|
||||
}
|
||||
@@ -1028,7 +1173,8 @@ func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
|
||||
return r.cfg.PortRange[0]
|
||||
}
|
||||
|
||||
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
|
||||
// scaleDown drains replicas; why is for the trace.
|
||||
func (r *Reconciler) scaleDown(ctx context.Context, excess []string, why string) {
|
||||
for _, id := range excess {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
@@ -1042,15 +1188,12 @@ func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
|
||||
r.mu.Unlock()
|
||||
|
||||
r.log.Info("draining replica", "replica", id)
|
||||
for _, s := range shards {
|
||||
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
||||
}
|
||||
}
|
||||
r.drainShards(ctx, id, shards, why)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
|
||||
// stopReplica stops a replica's shards and forgets it; why is for the trace.
|
||||
func (r *Reconciler) stopReplica(ctx context.Context, id, why string) {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
if !ok {
|
||||
@@ -1059,6 +1202,10 @@ func (r *Reconciler) stopReplica(ctx context.Context, id string) {
|
||||
}
|
||||
shards := slices.Clone(rep.Shards)
|
||||
rep.State = Stopped
|
||||
rep.startDone(r.now())
|
||||
ctx, span := startSpan(ctx, "stop replica", append(replicaAttrs(rep),
|
||||
attribute.String("llmgw.reason", why))...)
|
||||
defer span.End()
|
||||
r.mu.Unlock()
|
||||
|
||||
for _, s := range shards {
|
||||
@@ -1127,6 +1274,18 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e
|
||||
|
||||
prev := rep.State
|
||||
rep.deriveState(r.now())
|
||||
if rep.startSpan != nil {
|
||||
ev := []attribute.KeyValue{
|
||||
attribute.String("llmgw.shard", shardID), attribute.String("llmgw.state", st.String()),
|
||||
}
|
||||
if errMsg != "" {
|
||||
ev = append(ev, attribute.String("llmgw.error", errMsg))
|
||||
}
|
||||
rep.startSpan.AddEvent("shard "+st.String(), trace.WithAttributes(ev...))
|
||||
}
|
||||
if rep.State == Ready || rep.State.Terminal() {
|
||||
rep.startDone(r.now())
|
||||
}
|
||||
if rep.State == Ready && prev != Ready {
|
||||
// A model that comes up clears its backoff; the next failure starts
|
||||
// counting from scratch rather than from an old streak.
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
@@ -24,9 +26,12 @@ type fakeDispatcher struct {
|
||||
stopped []string
|
||||
drained []string
|
||||
failNext error
|
||||
// traces are the span each start was sent under, which is what the
|
||||
// real dispatcher puts on the command for the agent.
|
||||
traces []trace.SpanContext
|
||||
}
|
||||
|
||||
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
|
||||
func (d *fakeDispatcher) StartShard(ctx context.Context, cmd ShardCommand) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.failNext != nil {
|
||||
@@ -35,6 +40,7 @@ func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
|
||||
return err
|
||||
}
|
||||
d.started = append(d.started, cmd)
|
||||
d.traces = append(d.traces, trace.SpanContextFromContext(ctx))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
)
|
||||
|
||||
@@ -166,8 +168,19 @@ type Replica struct {
|
||||
// carried across rebuilds so the backoff does not reset on every attempt.
|
||||
Attempt int
|
||||
Err string
|
||||
|
||||
// startSpan is the "replica start" span, open from placement until the
|
||||
// replica serves, fails or is stopped; nil once closed, and for a replica
|
||||
// adopted rather than started. startTrace names it for good.
|
||||
startSpan trace.Span
|
||||
startTrace trace.SpanContext
|
||||
}
|
||||
|
||||
// StartTrace names the span of this replica's start, for something that
|
||||
// waited on it — a job serving a model — to link to. Invalid for a replica
|
||||
// adopted rather than started, and whenever tracing is off.
|
||||
func (r Replica) StartTrace() trace.SpanContext { return r.startTrace }
|
||||
|
||||
// NodeIDs lists the nodes this replica occupies.
|
||||
func (r Replica) NodeIDs() []string {
|
||||
out := make([]string, 0, len(r.Shards))
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
@@ -48,39 +50,39 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string,
|
||||
}
|
||||
switch {
|
||||
case readyLive >= desired:
|
||||
r.drainForRestart(ctx, id)
|
||||
r.drainForRestart(ctx, id, "restarted: its replacement serves")
|
||||
case inPlace:
|
||||
r.log.Warn("restarting a replica in place, as asked: stopped before its replacement starts", "replica", id)
|
||||
r.stopForRestart(ctx, id)
|
||||
r.stopForRestart(ctx, id, "restarted in place, as asked")
|
||||
case live >= desired:
|
||||
// A replacement is placed and loading; it is waited for.
|
||||
case parked:
|
||||
r.refuseRestart(id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
|
||||
r.refuseRestart(ctx, id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
|
||||
case outcome == noRoom && r.fitsWithout(m, nil, []victim{{ID: id, Model: m.Name}}):
|
||||
r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way",
|
||||
"replica", id, "model", m.Name)
|
||||
r.stopForRestart(ctx, id)
|
||||
r.stopForRestart(ctx, id, "restarted in place: it was the only thing in its replacement's way")
|
||||
case outcome == noRoom || outcome == overLimit:
|
||||
r.refuseRestart(id, "its replacement fits on no node even with this replica stopped "+
|
||||
r.refuseRestart(ctx, id, "its replacement fits on no node even with this replica stopped "+
|
||||
"(llmgw ctl explain-placement "+m.Name+")")
|
||||
case outcome == notTried && r.now().Sub(since) > r.cfg.DrainGrace:
|
||||
r.log.Warn("restarting a replica in place: no replacement beside it",
|
||||
"replica", id, "after", r.cfg.DrainGrace)
|
||||
r.stopForRestart(ctx, id)
|
||||
r.stopForRestart(ctx, id, "restarted in place: no replacement beside it within "+r.cfg.DrainGrace.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stopForRestart drains a replica and stops it at once, for its replacement
|
||||
// to take its place.
|
||||
func (r *Reconciler) stopForRestart(ctx context.Context, id string) {
|
||||
r.drainForRestart(ctx, id)
|
||||
r.stopReplica(ctx, id)
|
||||
func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) {
|
||||
r.drainForRestart(ctx, id, why)
|
||||
r.stopReplica(ctx, id, why)
|
||||
}
|
||||
|
||||
// refuseRestart gives up a replica's restart, leaving it serving, and keeps
|
||||
// why for whoever looks at the replica.
|
||||
func (r *Reconciler) refuseRestart(id, why string) {
|
||||
func (r *Reconciler) refuseRestart(ctx context.Context, id, why string) {
|
||||
r.mu.Lock()
|
||||
if rep, ok := r.replicas[id]; ok {
|
||||
rep.RetiringAt = time.Time{}
|
||||
@@ -89,9 +91,12 @@ func (r *Reconciler) refuseRestart(id, why string) {
|
||||
r.restartRefused[id] = why
|
||||
r.mu.Unlock()
|
||||
r.log.Warn("not restarting a replica: it keeps serving", "replica", id, "because", why)
|
||||
_, span := startSpan(ctx, "restart refused",
|
||||
attribute.String("llmgw.replica", id), attribute.String("llmgw.reason", why))
|
||||
span.End()
|
||||
}
|
||||
|
||||
func (r *Reconciler) drainForRestart(ctx context.Context, id string) {
|
||||
func (r *Reconciler) drainForRestart(ctx context.Context, id, why string) {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
var shards []Shard
|
||||
@@ -101,10 +106,8 @@ func (r *Reconciler) drainForRestart(ctx context.Context, id string) {
|
||||
}
|
||||
delete(r.restartInPlace, id)
|
||||
r.mu.Unlock()
|
||||
for _, s := range shards {
|
||||
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
||||
}
|
||||
if ok {
|
||||
r.drainShards(ctx, id, shards, why)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// What is traced here, and why it is shaped this way.
|
||||
//
|
||||
// A pass runs every few seconds and nearly always finds nothing to do. A
|
||||
// trace per pass would bury the few that placed or stopped something under
|
||||
// seventeen thousand empty ones a day, so a pass's span is begun only when
|
||||
// the pass first does something, and backdated to when the pass began.
|
||||
//
|
||||
// A replica's start outlives the pass that placed it by minutes. Its span,
|
||||
// "replica start", is opened beneath the placement and closed when the
|
||||
// replica serves, fails, or is stopped, and the command to each node carries
|
||||
// it, so the agent's spans for that shard — weights, launch, readiness — are
|
||||
// its descendants. That one trace is the answer to "why did this model take
|
||||
// four minutes to start".
|
||||
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
|
||||
func tracer() trace.Tracer { return obs.Tracer(tracerName) }
|
||||
|
||||
// pass is one reconcile pass's span, begun on first use.
|
||||
type pass struct {
|
||||
start time.Time
|
||||
base context.Context
|
||||
|
||||
once sync.Once
|
||||
ctx context.Context
|
||||
span trace.Span
|
||||
}
|
||||
|
||||
type passKey struct{}
|
||||
|
||||
// beginPass marks ctx as a pass's, for startSpan to hang its spans on.
|
||||
func beginPass(ctx context.Context) (context.Context, *pass) {
|
||||
p := &pass{start: time.Now(), base: ctx}
|
||||
return context.WithValue(ctx, passKey{}, p), p
|
||||
}
|
||||
|
||||
// context is the pass's span's context, beginning the span if it has not
|
||||
// been.
|
||||
func (p *pass) context() context.Context {
|
||||
p.once.Do(func() {
|
||||
p.ctx, p.span = tracer().Start(p.base, "reconcile", trace.WithTimestamp(p.start))
|
||||
})
|
||||
return p.ctx
|
||||
}
|
||||
|
||||
// end closes the pass's span, if anything began it.
|
||||
func (p *pass) end() {
|
||||
p.once.Do(func() {}) // nothing may begin it after this
|
||||
if p.span != nil {
|
||||
p.span.End()
|
||||
}
|
||||
}
|
||||
|
||||
// startSpan opens a span for something the reconciler did: beneath the span
|
||||
// already in ctx, or beneath the pass's — begun now — when ctx holds the
|
||||
// pass and nothing else.
|
||||
func startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) {
|
||||
if !trace.SpanContextFromContext(ctx).IsValid() {
|
||||
if p, ok := ctx.Value(passKey{}).(*pass); ok {
|
||||
if pctx := p.context(); pctx != nil {
|
||||
ctx = pctx
|
||||
}
|
||||
}
|
||||
}
|
||||
return tracer().Start(ctx, name, trace.WithAttributes(attrs...))
|
||||
}
|
||||
|
||||
// endSpan closes a span, marking it failed when why is not empty.
|
||||
func endSpan(span trace.Span, why string) {
|
||||
if why != "" {
|
||||
obs.End(span, errors.New(why))
|
||||
return
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
|
||||
// replicaAttrs are what every span about one replica says.
|
||||
func replicaAttrs(rep *Replica) []attribute.KeyValue {
|
||||
return []attribute.KeyValue{
|
||||
attribute.String("llmgw.replica", rep.ID),
|
||||
attribute.String("llmgw.model", rep.Model),
|
||||
attribute.String("llmgw.profile", rep.Profile),
|
||||
attribute.StringSlice("llmgw.nodes", rep.NodeIDs()),
|
||||
}
|
||||
}
|
||||
|
||||
// startDone closes a replica's "replica start" span, once: when it serves,
|
||||
// with how long that took, or when it fails or is stopped first, with why.
|
||||
// Called with r.mu held, or before the replica is in the map.
|
||||
func (rep *Replica) startDone(now time.Time) {
|
||||
if rep.startSpan == nil {
|
||||
return
|
||||
}
|
||||
span := rep.startSpan
|
||||
rep.startSpan = nil
|
||||
span.SetAttributes(attribute.String("llmgw.state", rep.State.String()))
|
||||
switch rep.State {
|
||||
case Ready:
|
||||
span.SetAttributes(attribute.Float64("llmgw.ready_after_seconds", now.Sub(rep.CreatedAt).Seconds()))
|
||||
span.SetStatus(codes.Ok, "")
|
||||
case Failed:
|
||||
span.SetStatus(codes.Error, rep.Err)
|
||||
default:
|
||||
span.SetStatus(codes.Error, "stopped before it was ready")
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAPlacementIsTracedFromThePassToTheCommandAndTheReplicaServing(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
pass := rec.One(t, [16]byte{}, "reconcile")
|
||||
tid := pass.SpanContext().TraceID()
|
||||
up := rec.One(t, tid, "scale up")
|
||||
place := rec.One(t, tid, "place")
|
||||
if !obstest.ChildOf(up, pass) || !obstest.ChildOf(place, up) {
|
||||
t.Fatal("the placement is not beneath the scale-up, beneath the pass")
|
||||
}
|
||||
if a := obstest.Attrs(place); a["llmgw.model"] != "llama" || a["llmgw.profile"] != "tp4" ||
|
||||
a["llmgw.gpu_class"] != "h100-80g" || a["llmgw.nodes"] != `["1"]` {
|
||||
t.Errorf("the placement does not say where it went: %v", a)
|
||||
}
|
||||
if a := obstest.Attrs(up); a["llmgw.outcome"] != "placed" || a["llmgw.desired"] != "1" {
|
||||
t.Errorf("the scale-up does not say what it did: %v", a)
|
||||
}
|
||||
|
||||
// The command went under a span of its own, beneath the replica's start,
|
||||
// which is still open: the replica is loading.
|
||||
send := rec.One(t, tid, "send start")
|
||||
if got := h.disp.traces; len(got) != 1 || got[0].SpanID() != send.SpanContext().SpanID() {
|
||||
t.Fatalf("the command was not sent under its span: %v", got)
|
||||
}
|
||||
if len(rec.Named("replica start")) != 0 {
|
||||
t.Fatal("the replica's start ended before the replica served")
|
||||
}
|
||||
|
||||
h.markReady()
|
||||
start := rec.One(t, tid, "replica start")
|
||||
if !obstest.ChildOf(start, up) || !obstest.ChildOf(send, start) {
|
||||
t.Error("the replica's start is not beneath the scale-up, or the command beneath it")
|
||||
}
|
||||
a := obstest.Attrs(start)
|
||||
if a["llmgw.state"] != "ready" || a["llmgw.profile"] != "tp4" || a["llmgw.ready_after_seconds"] == "" {
|
||||
t.Errorf("the replica's start does not say how it ended: %v", a)
|
||||
}
|
||||
if start.Status().Code != codes.Ok {
|
||||
t.Errorf("a replica that serves is %v", start.Status())
|
||||
}
|
||||
events := start.Events()
|
||||
if len(events) == 0 || events[len(events)-1].Name != "shard ready" {
|
||||
t.Errorf("the shard's progress is not on the replica's start: %v", events)
|
||||
}
|
||||
if sc := h.r.Replicas()[0].StartTrace(); sc.SpanID() != start.SpanContext().SpanID() {
|
||||
t.Error("the replica does not name its start's span for a job to link to")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPassThatDoesNothingLeavesNoTrace(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
before := len(rec.Ended())
|
||||
|
||||
// Every pass from here finds the model as it should be.
|
||||
for range 3 {
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
if got := rec.Ended()[before:]; len(got) != 0 {
|
||||
var names []string
|
||||
for _, s := range got {
|
||||
names = append(names, s.Name())
|
||||
}
|
||||
t.Errorf("passes with nothing to do were traced: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlacementThatFitsNowhereSaysWhy(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
m := testModel(1, catalog.Profile{Name: "tp16", GPUClass: "h100-80g", TP: 16})
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
place := rec.One(t, [16]byte{}, "place")
|
||||
if place.Status().Code != codes.Error {
|
||||
t.Errorf("a placement that found nowhere is %v", place.Status())
|
||||
}
|
||||
if a := obstest.Attrs(place); a["llmgw.rejections"] == "0" || !strings.Contains(a["llmgw.explain"], "tp16") {
|
||||
t.Errorf("the placement does not say what it turned down: %v", a)
|
||||
}
|
||||
if a := obstest.Attrs(rec.One(t, place.SpanContext().TraceID(), "scale up")); a["llmgw.outcome"] != "no room" {
|
||||
t.Errorf("the scale-up says %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedStartEndsItsTraceWithTheEnginesWords(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
cmd := h.disp.startedShards()[0]
|
||||
|
||||
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Failed, "CUDA out of memory")
|
||||
start := rec.One(t, [16]byte{}, "replica start")
|
||||
if start.Status().Code != codes.Error || !strings.Contains(start.Status().Description, "CUDA out of memory") {
|
||||
t.Errorf("the failed start says %+v", start.Status())
|
||||
}
|
||||
|
||||
// The next pass reaps it, in a trace of its own, saying when it is tried
|
||||
// again.
|
||||
h.r.Reconcile(t.Context())
|
||||
reap := rec.One(t, [16]byte{}, "reap replica")
|
||||
a := obstest.Attrs(reap)
|
||||
if a["llmgw.replica"] != cmd.ReplicaID || a["llmgw.failures_in_a_row"] != "1" || a["llmgw.retry_in_seconds"] == "" {
|
||||
t.Errorf("the reaping does not say what happens next: %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADispatchThatFailsIsTracedAsFailed(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.disp.failNext = errors.New("no agent connected")
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
send := rec.One(t, [16]byte{}, "send start")
|
||||
if send.Status().Code != codes.Error {
|
||||
t.Errorf("a command that was not sent is %v", send.Status())
|
||||
}
|
||||
if start := rec.One(t, [16]byte{}, "replica start"); start.Status().Code != codes.Error {
|
||||
t.Errorf("a replica none of whose shards was sent is %v", start.Status())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPreemptionIsTracedWithItsVictims(t *testing.T) {
|
||||
h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8))
|
||||
rec := obstest.Record(t)
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
pre := rec.One(t, [16]byte{}, "preempt")
|
||||
a := obstest.Attrs(pre)
|
||||
if a["llmgw.for"] != "vital" || !strings.Contains(a["llmgw.victims"], "chat") {
|
||||
t.Errorf("the preemption does not say who gave way to whom: %v", a)
|
||||
}
|
||||
drain := rec.One(t, pre.SpanContext().TraceID(), "drain replica")
|
||||
if !obstest.ChildOf(drain, pre) || obstest.Attrs(drain)["llmgw.reason"] != "preempted for vital" {
|
||||
t.Errorf("the victim's drain is not beneath the preemption: %v", obstest.Attrs(drain))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStopSaysWhy(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
rec := obstest.Record(t)
|
||||
|
||||
h.setModels(t) // the model leaves the catalog
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
stop := rec.One(t, [16]byte{}, "stop replica")
|
||||
if a := obstest.Attrs(stop); a["llmgw.model"] != "llama" || a["llmgw.reason"] != "its model left the catalog" {
|
||||
t.Errorf("the stop does not say why: %v", a)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user