Merge C20 tracing beyond the router
Conflicts: config imports and docgen sources (both kept); agent.pb.go regenerated with buf from the merged proto (MIG devices and trace context). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
801e2d51c0
33 files changed
+1608
-144
No files matched your search
@@ -14,6 +14,8 @@ import (
|
||||
"time"
|
||||
|
||||
"connectrpc.com/connect"
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1/llmgwv1connect"
|
||||
@@ -22,6 +24,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/fabrictest"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// StatusSink receives shard transitions and recovers replica state from what
|
||||
@@ -311,7 +314,20 @@ func (s *Service) Connect(
|
||||
// unreachable in production while its unit tests, which call the reconciler
|
||||
// directly, kept passing.
|
||||
func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.RunningShard, c *conn) {
|
||||
// A trace of its own, and only for a node that reported something: a
|
||||
// node running nothing registering is not an event anyone looks for.
|
||||
tctx, span := ctx, trace.SpanFromContext(context.Background()) // the no-op span
|
||||
if len(shards) > 0 {
|
||||
tctx, span = obs.Tracer(tracerName).Start(ctx, "adopt",
|
||||
trace.WithNewRoot(),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", nodeID),
|
||||
attribute.Int("llmgw.shards", len(shards)),
|
||||
))
|
||||
}
|
||||
defer span.End()
|
||||
unclaimed := s.status.Adopt(nodeID, AdoptedFromRegister(nodeID, shards))
|
||||
span.SetAttributes(attribute.Int("llmgw.unclaimed", len(unclaimed)))
|
||||
for _, u := range unclaimed {
|
||||
// The model or profile left the catalog while this was running. Nobody
|
||||
// will ever route to it, and it is holding GPUs.
|
||||
@@ -319,7 +335,8 @@ func (s *Service) adopt(ctx context.Context, nodeID string, shards []*llmgwv1.Ru
|
||||
"node", nodeID, "shard", u.ShardID, "model", u.Model, "profile", u.Profile)
|
||||
select {
|
||||
case c.send <- &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: u.ShardID}},
|
||||
Trace: obs.Inject(tctx),
|
||||
}:
|
||||
case <-ctx.Done():
|
||||
return
|
||||
@@ -715,27 +732,37 @@ type Leader interface{ IsLeader() bool }
|
||||
// and what every deployment before leadership had.
|
||||
func (s *Service) SetLeadership(l Leader) { s.leader = l }
|
||||
|
||||
// tracerName names the spans this package opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
|
||||
// ErrNotConnected means no agent stream is open for that node.
|
||||
var ErrNotConnected = errors.New("no agent connected")
|
||||
|
||||
// StartShard implements reconciler.Dispatcher.
|
||||
func (s *Service) StartShard(_ context.Context, cmd reconciler.ShardCommand) error {
|
||||
//
|
||||
// The command carries the trace of ctx, so the agent's spans for this start
|
||||
// — the weights, the engine's launch, its readiness — are in the trace of
|
||||
// the replica they are starting rather than in one of their own.
|
||||
func (s *Service) StartShard(ctx context.Context, cmd reconciler.ShardCommand) error {
|
||||
return s.send(cmd.NodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
|
||||
Msg: &llmgwv1.ControlMessage_Start{Start: StartShardFromCommand(cmd)},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
// StopShard implements reconciler.Dispatcher.
|
||||
func (s *Service) StopShard(_ context.Context, nodeID, shardID string) error {
|
||||
func (s *Service) StopShard(ctx context.Context, nodeID, shardID string) error {
|
||||
return s.send(nodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Stop{Stop: &llmgwv1.StopShard{ShardId: shardID}},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
// DrainShard implements reconciler.Dispatcher.
|
||||
func (s *Service) DrainShard(_ context.Context, nodeID, shardID string) error {
|
||||
func (s *Service) DrainShard(ctx context.Context, nodeID, shardID string) error {
|
||||
return s.send(nodeID, &llmgwv1.ControlMessage{
|
||||
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
|
||||
Msg: &llmgwv1.ControlMessage_Drain{Drain: &llmgwv1.DrainShard{ShardId: shardID}},
|
||||
Trace: obs.Inject(ctx),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -178,6 +178,9 @@ type Job struct {
|
||||
// how fast it said it.
|
||||
Reply string `json:"reply,omitempty"`
|
||||
TokensPerSecond float64 `json:"tokens_per_second,omitempty"`
|
||||
|
||||
// trace is the job's spans, when tracing is on.
|
||||
trace jobTrace
|
||||
}
|
||||
|
||||
// Server runs jobs and remembers them.
|
||||
@@ -324,6 +327,7 @@ func (s *Server) snapshot(j *Job) Job {
|
||||
func (s *Server) step(j *Job, name, state, detail string) {
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
j.traceStep(name, state, detail)
|
||||
if n := len(j.Steps); n > 0 && j.Steps[n-1].Name == name {
|
||||
j.Steps[n-1].State, j.Steps[n-1].Detail, j.Steps[n-1].At = state, detail, time.Now()
|
||||
return
|
||||
@@ -335,6 +339,7 @@ func (s *Server) fail(j *Job, name string, err error) {
|
||||
s.step(j, name, Failed, err.Error())
|
||||
s.mu.Lock()
|
||||
j.State, j.Error = Failed, err.Error()
|
||||
j.traceEnd(err)
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Warn("serving a model failed", "job", j.ID, "repo", j.Request.Repo, "step", name, "err", err)
|
||||
}
|
||||
@@ -623,6 +628,7 @@ func (s *Server) addAndStart(ctx context.Context, j *Job, repo hf.Repo, set hf.F
|
||||
func (s *Server) finish(j *Job) {
|
||||
s.mu.Lock()
|
||||
j.State = Done
|
||||
j.traceEnd(nil)
|
||||
s.mu.Unlock()
|
||||
s.deps.Log.Info("model served", "job", j.ID, "repo", j.Request.Repo, "model", j.Model)
|
||||
}
|
||||
@@ -791,6 +797,7 @@ func (s *Server) waitReady(ctx context.Context, j *Job, name string, hardware bo
|
||||
continue
|
||||
}
|
||||
if r.State == reconciler.Ready {
|
||||
s.linkReplica(j, r)
|
||||
s.step(j, "placement", Done, fmt.Sprintf("%s ready on %s at %s", r.ID,
|
||||
strings.Join(r.NodeIDs(), ","), r.Endpoint))
|
||||
return r, nil
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// A job is a trace of its own, and each of its steps a span beneath it —
|
||||
// registry, mirror, check, catalog, placement, smoke test — opened when the
|
||||
// step begins and closed when it is done, failed or skipped. The steps are
|
||||
// what a job already records for `ctl model jobs`; the trace adds where the
|
||||
// time went, and from the placement step a link to the replica's own start,
|
||||
// whose trace goes on down to the node.
|
||||
//
|
||||
// Begun by the first step rather than where a job is made, so every kind of
|
||||
// job is traced by recording steps and nothing else.
|
||||
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/modelserve"
|
||||
|
||||
// jobTrace is a job's spans. Read and written with Server.mu held.
|
||||
type jobTrace struct {
|
||||
span trace.Span
|
||||
step trace.Span
|
||||
stepName string
|
||||
}
|
||||
|
||||
// traceStep records a step on the job's trace. Called with s.mu held.
|
||||
func (j *Job) traceStep(name, state, detail string) {
|
||||
if j.trace.span == nil {
|
||||
kind := j.Kind
|
||||
if kind == "" {
|
||||
kind = "serve"
|
||||
}
|
||||
_, j.trace.span = obs.Tracer(tracerName).Start(context.Background(), kind+" model",
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.job", j.ID),
|
||||
attribute.String("llmgw.repo", j.Request.Repo),
|
||||
attribute.String("llmgw.model", j.Model),
|
||||
))
|
||||
}
|
||||
if j.trace.step != nil && j.trace.stepName != name {
|
||||
// A step left running when the next began: over, whatever it said.
|
||||
j.trace.step.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
if j.trace.step == nil {
|
||||
ctx := trace.ContextWithSpan(context.Background(), j.trace.span)
|
||||
_, j.trace.step = obs.Tracer(tracerName).Start(ctx, name)
|
||||
j.trace.stepName = name
|
||||
}
|
||||
span := j.trace.step
|
||||
span.SetAttributes(attribute.String("llmgw.state", state), attribute.String("llmgw.detail", detail))
|
||||
switch state {
|
||||
case Running:
|
||||
return
|
||||
case Failed:
|
||||
span.SetStatus(codes.Error, detail)
|
||||
}
|
||||
span.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
|
||||
// traceEnd closes the job's trace, failed when err is set. Called with s.mu
|
||||
// held.
|
||||
func (j *Job) traceEnd(err error) {
|
||||
if j.trace.step != nil {
|
||||
j.trace.step.End()
|
||||
j.trace.step = nil
|
||||
}
|
||||
if j.trace.span == nil {
|
||||
return
|
||||
}
|
||||
if j.Model != "" {
|
||||
j.trace.span.SetAttributes(attribute.String("llmgw.model", j.Model))
|
||||
}
|
||||
obs.End(j.trace.span, err)
|
||||
j.trace.span = nil
|
||||
}
|
||||
|
||||
// linkReplica links the step in progress to the start of a replica it waited
|
||||
// for, which is in the reconciler's trace rather than this one.
|
||||
func (s *Server) linkReplica(j *Job, r reconciler.Replica) {
|
||||
sc := r.StartTrace()
|
||||
if !sc.IsValid() {
|
||||
return
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if j.trace.step != nil {
|
||||
j.trace.step.AddLink(trace.Link{SpanContext: sc,
|
||||
Attributes: []attribute.KeyValue{attribute.String("llmgw.replica", r.ID)}})
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package modelserve
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/hf"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAJobIsATraceWithASpanPerStep(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
files := []hf.File{{Path: "Qwen2.5-1.5B-Instruct-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "bb"}}
|
||||
s, _, _, _ := setup(t, files, map[string]string{"Qwen2.5-1.5B-Instruct-Q4_K_M.gguf": "BB"})
|
||||
job, err := s.Start(Request{Repo: "bartowski/Qwen2.5-1.5B-Instruct-GGUF", Replicas: 1})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if j := wait(t, s, job.ID); j.State != Done {
|
||||
t.Fatalf("job %s: %s", j.State, j.Error)
|
||||
}
|
||||
|
||||
root := rec.One(t, [16]byte{}, "serve model")
|
||||
tid := root.SpanContext().TraceID()
|
||||
if a := obstest.Attrs(root); a["llmgw.job"] != job.ID || a["llmgw.model"] != "qwen2.5-1.5b-instruct" {
|
||||
t.Errorf("the job's span does not say which job: %v", a)
|
||||
}
|
||||
var steps []string
|
||||
for _, sp := range rec.Ended() {
|
||||
if sp.SpanContext().TraceID() == tid && obstest.ChildOf(sp, root) {
|
||||
steps = append(steps, sp.Name())
|
||||
}
|
||||
}
|
||||
if strings.Join(steps, ",") != "registry,mirror,check,catalog,placement,smoke test" {
|
||||
t.Errorf("step spans %v", steps)
|
||||
}
|
||||
smoke := rec.One(t, tid, "smoke test")
|
||||
if a := obstest.Attrs(smoke); a["llmgw.state"] != Done || !strings.Contains(a["llmgw.detail"], "Hello!") {
|
||||
t.Errorf("the smoke test's span says %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedJobsTraceEndsAtTheStepThatFailed(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
files := []hf.File{{Path: "m-Q4_K_M.gguf", Bytes: 1 << 30, SHA256: "aaaaaaaaaaaaaaaa"}}
|
||||
s, _, _, _ := setup(t, files, map[string]string{"m-Q4_K_M.gguf": "bbbbbbbbbbbbbbbb"})
|
||||
job, _ := s.Start(Request{Repo: "a/b", Replicas: 1})
|
||||
if j := wait(t, s, job.ID); j.State != Failed {
|
||||
t.Fatalf("job %s", j.State)
|
||||
}
|
||||
|
||||
root := rec.One(t, [16]byte{}, "serve model")
|
||||
if root.Status().Code != codes.Error || !strings.Contains(root.Status().Description, "registry published") {
|
||||
t.Errorf("the failed job's span says %+v", root.Status())
|
||||
}
|
||||
mirror := rec.One(t, root.SpanContext().TraceID(), "mirror")
|
||||
if mirror.Status().Code != codes.Error {
|
||||
t.Errorf("the step that failed is %+v", mirror.Status())
|
||||
}
|
||||
if len(rec.Named("catalog")) != 0 {
|
||||
t.Error("a step after the failure was traced")
|
||||
}
|
||||
}
|
||||
@@ -636,6 +636,9 @@ func (s *Server) replace(ctx context.Context, j *Job, name string) error {
|
||||
}
|
||||
}
|
||||
if left == 0 && len(ready) > 0 {
|
||||
for _, r := range ready {
|
||||
s.linkReplica(j, r)
|
||||
}
|
||||
s.step(j, "replicas", Done, fmt.Sprintf("%d replaced", len(old)))
|
||||
s.step(j, "smoke test", Running, "asking "+ready[0].ID+" to say hello")
|
||||
reply, tps, err := s.smoke(ctx, ready[0], name)
|
||||
|
||||
@@ -6,6 +6,8 @@ import (
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
@@ -53,7 +55,15 @@ func (r *Reconciler) makeRoomFor(ctx context.Context, m catalog.Model) bool {
|
||||
r.log.Info("evicting an idle replica to make room",
|
||||
"stopping", best.ID, "idle_model", best.Model,
|
||||
"idle", best.Idle.Round(time.Second), "gpus", best.GPUs, "for", m.Name)
|
||||
r.stopReplica(ctx, best.ID)
|
||||
ctx, span := startSpan(ctx, "evict",
|
||||
attribute.String("llmgw.for", m.Name),
|
||||
attribute.String("llmgw.replica", best.ID),
|
||||
attribute.String("llmgw.model", best.Model),
|
||||
attribute.Float64("llmgw.idle_seconds", best.Idle.Seconds()),
|
||||
attribute.Int("llmgw.gpus", best.GPUs),
|
||||
attribute.Int("llmgw.candidates", len(candidates)))
|
||||
defer span.End()
|
||||
r.stopReplica(ctx, best.ID, "evicted, idle, to make room for "+m.Name)
|
||||
return true
|
||||
}
|
||||
|
||||
|
||||
@@ -2,10 +2,13 @@ package reconciler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"slices"
|
||||
"sort"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||
@@ -113,12 +116,24 @@ func (r *Reconciler) preempt(ctx context.Context, m catalog.Model, chosen []vict
|
||||
r.preempting[m.Name] = ids
|
||||
r.mu.Unlock()
|
||||
|
||||
victims := make([]string, 0, len(chosen))
|
||||
for _, v := range chosen {
|
||||
r.log.Warn("preempting a replica to make room",
|
||||
"replica", v.ID, "model", v.Model, "class", v.Class, "above_min", v.AboveMin,
|
||||
"for", m.Name, "for_class", m.EffectiveClass())
|
||||
desc := fmt.Sprintf("%s (%s, %s)", v.ID, v.Model, v.Class)
|
||||
if v.AboveMin {
|
||||
desc = fmt.Sprintf("%s (%s, %s, above its minimum)", v.ID, v.Model, v.Class)
|
||||
}
|
||||
victims = append(victims, desc)
|
||||
}
|
||||
r.scaleDown(ctx, ids)
|
||||
ctx, span := startSpan(ctx, "preempt",
|
||||
attribute.String("llmgw.for", m.Name),
|
||||
attribute.String("llmgw.for_class", m.EffectiveClass()),
|
||||
attribute.StringSlice("llmgw.victims", victims),
|
||||
attribute.String("llmgw.hold", r.cfg.PreemptHold.String()))
|
||||
defer span.End()
|
||||
r.scaleDown(ctx, ids, "preempted for "+m.Name)
|
||||
}
|
||||
|
||||
// awaitingPreemption reports whether replicas stopped for a model are still
|
||||
|
||||
@@ -11,6 +11,10 @@ import (
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
@@ -18,6 +22,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/placement"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/quota"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/notify"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// ShardCommand is everything an agent needs to launch one shard.
|
||||
@@ -405,7 +410,9 @@ func (r *Reconciler) Reconcile(ctx context.Context) {
|
||||
if r.leader != nil && !r.leader.IsLeader() {
|
||||
return
|
||||
}
|
||||
r.expireNodes()
|
||||
ctx, p := beginPass(ctx)
|
||||
defer p.end()
|
||||
r.expireNodes(ctx)
|
||||
r.expireStuckReplicas()
|
||||
r.drainCordonedNodes(ctx)
|
||||
r.reapFailed(ctx)
|
||||
@@ -452,7 +459,7 @@ func (r *Reconciler) stopReplicasOfRemovedModels(ctx context.Context) {
|
||||
for _, rep := range gone {
|
||||
r.log.Info("stopping a replica whose model left the catalog",
|
||||
"replica", rep.ID, "model", rep.Model)
|
||||
r.stopReplica(ctx, rep.ID)
|
||||
r.stopReplica(ctx, rep.ID, "its model left the catalog")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -523,6 +530,7 @@ func (r *Reconciler) expireStuckReplicas() {
|
||||
}
|
||||
rep.State = Failed
|
||||
rep.Err = fmt.Sprintf("did not come up within %s", r.startupTimeout(rep))
|
||||
rep.startDone(now)
|
||||
stuck = append(stuck, rep)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
@@ -555,7 +563,7 @@ func (r *Reconciler) startupTimeout(rep *Replica) time.Duration {
|
||||
// The two-stage delay is deliberate: a brief network blip must not tear down
|
||||
// replicas that are still serving traffic perfectly well, and rebuilding a
|
||||
// multi-node replica costs tens of minutes of model loading.
|
||||
func (r *Reconciler) expireNodes() {
|
||||
func (r *Reconciler) expireNodes(ctx context.Context) {
|
||||
now := r.now()
|
||||
for _, n := range r.inv.Nodes() {
|
||||
silent := now.Sub(n.LastSeen)
|
||||
@@ -563,14 +571,21 @@ func (r *Reconciler) expireNodes() {
|
||||
case n.State == inventory.Ready && silent > r.cfg.UnreachableAfter:
|
||||
r.inv.SetState(n.ID, inventory.Unreachable)
|
||||
r.log.Warn("node is unreachable", "node", n.ID, "silent_for", silent.Round(time.Second))
|
||||
_, span := startSpan(ctx, "node unreachable", attribute.String("llmgw.node", n.ID),
|
||||
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
|
||||
span.End()
|
||||
case n.State == inventory.Unreachable && silent > r.cfg.LostAfter:
|
||||
_, span := startSpan(ctx, "node lost", attribute.String("llmgw.node", n.ID),
|
||||
attribute.Float64("llmgw.silent_seconds", silent.Seconds()))
|
||||
r.inv.SetState(n.ID, inventory.Lost)
|
||||
r.log.Error("node lost; its replicas will be rebuilt elsewhere",
|
||||
"node", n.ID, "silent_for", silent.Round(time.Second))
|
||||
r.notify.Load().Notify(notify.NodeLost, n.ID,
|
||||
fmt.Sprintf("node %s was lost: silent for %s; its replicas are rebuilt elsewhere", n.ID, silent.Round(time.Second)),
|
||||
map[string]any{"node": n.ID, "silent_seconds": int(silent.Seconds())})
|
||||
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")
|
||||
span.SetAttributes(attribute.StringSlice("llmgw.replicas_failed",
|
||||
r.failReplicasOn(n.ID, "node "+n.ID+" was lost")))
|
||||
span.End()
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -609,24 +624,26 @@ func (r *Reconciler) drainCordonedNodes(ctx context.Context) {
|
||||
for _, rep := range toDrain {
|
||||
r.log.Info("draining replica off a node being taken out of service",
|
||||
"replica", rep.ID, "nodes", rep.NodeIDs())
|
||||
for _, s := range shards[rep.ID] {
|
||||
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
||||
}
|
||||
}
|
||||
r.drainShards(ctx, rep.ID, shards[rep.ID], "its node is being drained")
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) failReplicasOn(nodeID, reason string) {
|
||||
// failReplicasOn fails every replica on a node, and says which.
|
||||
func (r *Reconciler) failReplicasOn(nodeID, reason string) []string {
|
||||
r.mu.Lock()
|
||||
defer r.mu.Unlock()
|
||||
var ids []string
|
||||
for _, rep := range r.replicas {
|
||||
if rep.State.Terminal() || !rep.OnNode(nodeID) {
|
||||
continue
|
||||
}
|
||||
rep.State = Failed
|
||||
rep.Err = reason
|
||||
rep.startDone(r.now())
|
||||
ids = append(ids, rep.ID)
|
||||
}
|
||||
slices.Sort(ids)
|
||||
return ids
|
||||
}
|
||||
|
||||
// reapFailed releases the resources of terminal replicas and tells the agents
|
||||
@@ -639,8 +656,10 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
doomed = append(doomed, rep)
|
||||
}
|
||||
}
|
||||
parkedNow := map[string]bool{}
|
||||
for _, rep := range doomed {
|
||||
delete(r.replicas, rep.ID)
|
||||
rep.startDone(r.now())
|
||||
if rep.State == Failed {
|
||||
// Carry the failure count on the model, not the replica: the
|
||||
// replica is about to disappear, and the backoff has to survive it.
|
||||
@@ -652,6 +671,7 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
r.parked[rep.Model] = parkRecord{
|
||||
Since: r.now(), Reason: rep.Err, Failures: n, fp: modelFingerprint(m),
|
||||
}
|
||||
parkedNow[rep.ID] = true
|
||||
r.log.Error("model parked: it failed every time it started, and is not retried "+
|
||||
"until its definition changes or `llmgw ctl retry` is run",
|
||||
"model", rep.Model, "failures", n, "last_error", rep.Err)
|
||||
@@ -667,23 +687,52 @@ func (r *Reconciler) reapFailed(ctx context.Context) {
|
||||
|
||||
for _, rep := range doomed {
|
||||
// rep is out of the map now, so no agent report can reach it any more.
|
||||
sctx, span := startSpan(ctx, "reap replica", replicaAttrs(rep)...)
|
||||
span.SetAttributes(attribute.String("llmgw.state", rep.State.String()))
|
||||
if rep.State == Failed {
|
||||
r.mu.Lock()
|
||||
retry := r.backoff(r.attempts[rep.Model])
|
||||
attempts := r.attempts[rep.Model]
|
||||
retry := r.backoff(attempts)
|
||||
r.mu.Unlock()
|
||||
r.log.Error("replica failed",
|
||||
"replica", rep.ID, "model", rep.Model, "err", rep.Err,
|
||||
"retry_in", retry.Round(time.Second))
|
||||
span.SetAttributes(
|
||||
attribute.String("llmgw.error", rep.Err),
|
||||
attribute.Int("llmgw.failures_in_a_row", attempts),
|
||||
attribute.Float64("llmgw.retry_in_seconds", retry.Seconds()),
|
||||
attribute.Bool("llmgw.parked", parkedNow[rep.ID]),
|
||||
)
|
||||
}
|
||||
// Stop every shard, including the ones that looked healthy: a partially
|
||||
// alive replica holds GPUs while serving nothing.
|
||||
for _, s := range rep.Shards {
|
||||
if err := r.disp.StopShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
if err := r.disp.StopShard(sctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("stopping a shard of a dead replica failed",
|
||||
"replica", rep.ID, "shard", s.ID, "err", err)
|
||||
}
|
||||
}
|
||||
r.inv.ReleaseReplica(rep.ID)
|
||||
span.End()
|
||||
}
|
||||
}
|
||||
|
||||
// drainShards tells a replica's nodes to stop routing to its shards, as one
|
||||
// "drain replica" span saying why.
|
||||
func (r *Reconciler) drainShards(ctx context.Context, id string, shards []Shard, why string) {
|
||||
nodes := make([]string, 0, len(shards))
|
||||
for _, s := range shards {
|
||||
nodes = append(nodes, s.NodeID)
|
||||
}
|
||||
ctx, span := startSpan(ctx, "drain replica",
|
||||
attribute.String("llmgw.replica", id),
|
||||
attribute.StringSlice("llmgw.nodes", nodes),
|
||||
attribute.String("llmgw.reason", why))
|
||||
defer span.End()
|
||||
for _, s := range shards {
|
||||
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -727,12 +776,14 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
}
|
||||
}
|
||||
// The next restart an operator queued, once the last one is done.
|
||||
restarting := ""
|
||||
if len(retiringIDs) == 0 {
|
||||
if id, ok := r.nextRestartLocked(m.Name); ok {
|
||||
r.replicas[id].RetiringAt = r.now()
|
||||
retiringIDs = append(retiringIDs, id)
|
||||
liveIDs = slices.DeleteFunc(liveIDs, func(x string) bool { return x == id })
|
||||
readyLive--
|
||||
restarting = id
|
||||
r.log.Info("restarting a replica: placing its replacement first",
|
||||
"replica", id, "model", m.Name)
|
||||
}
|
||||
@@ -749,6 +800,11 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
r.log.Info("model unparked: its definition changed", "model", m.Name)
|
||||
}
|
||||
r.mu.Unlock()
|
||||
if restarting != "" {
|
||||
_, span := startSpan(ctx, "restart replica",
|
||||
attribute.String("llmgw.replica", restarting), attribute.String("llmgw.model", m.Name))
|
||||
span.End()
|
||||
}
|
||||
|
||||
desired := r.desiredFor(m)
|
||||
outcome := notTried
|
||||
@@ -765,9 +821,20 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
// the real error in noise.
|
||||
return
|
||||
}
|
||||
outcome = r.scaleUp(ctx, m, avoid)
|
||||
sctx, span := startSpan(ctx, "scale up",
|
||||
attribute.String("llmgw.model", m.Name),
|
||||
attribute.Int("llmgw.replicas", len(liveIDs)),
|
||||
attribute.Int("llmgw.desired", desired))
|
||||
outcome = r.scaleUp(sctx, m, avoid)
|
||||
span.SetAttributes(attribute.String("llmgw.outcome", outcome.String()))
|
||||
span.End()
|
||||
case len(liveIDs) > desired:
|
||||
r.scaleDown(ctx, liveIDs[desired:])
|
||||
sctx, span := startSpan(ctx, "scale down",
|
||||
attribute.String("llmgw.model", m.Name),
|
||||
attribute.Int("llmgw.replicas", len(liveIDs)),
|
||||
attribute.Int("llmgw.desired", desired))
|
||||
r.scaleDown(sctx, liveIDs[desired:], "more replicas than desired")
|
||||
span.End()
|
||||
}
|
||||
|
||||
// A drained replica is stopped once its replacement is serving, or once the
|
||||
@@ -790,17 +857,17 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
switch {
|
||||
case !ok:
|
||||
case replaced:
|
||||
r.stopReplica(ctx, id)
|
||||
r.stopReplica(ctx, id, "drained, and its replacement serves")
|
||||
case expired && preemptedFor != "":
|
||||
r.log.Info("stopping a preempted replica", "replica", id,
|
||||
"for", preemptedFor, "after", grace)
|
||||
r.stopReplica(ctx, id)
|
||||
r.stopReplica(ctx, id, "preempted for "+preemptedFor+", and its grace is over")
|
||||
case expired:
|
||||
// The replacement never arrived — often because nowhere else can
|
||||
// host it. Draining forever would leave the node undrainable.
|
||||
r.log.Warn("stopping a drained replica without a replacement",
|
||||
"replica", id, "after", grace)
|
||||
r.stopReplica(ctx, id)
|
||||
r.stopReplica(ctx, id, "drained, and no replacement arrived within "+grace.String())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -813,6 +880,9 @@ func (r *Reconciler) reconcileModel(ctx context.Context, m catalog.Model) {
|
||||
// allocated, and starting several at once from the same inventory snapshot
|
||||
// would hand them the same GPUs.
|
||||
func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []string) scaleOutcome {
|
||||
_, pspan := startSpan(ctx, "place",
|
||||
attribute.String("llmgw.model", m.Name),
|
||||
attribute.StringSlice("llmgw.avoid", avoid))
|
||||
profiles := r.orderedProfiles(m)
|
||||
res := r.sched.Place(profiles, avoid)
|
||||
if res.Placement == nil && len(avoid) > 0 {
|
||||
@@ -820,7 +890,10 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
|
||||
// is the only thing standing in the way, take the placement anyway:
|
||||
// a second replica on one node still survives an engine crash.
|
||||
res = r.sched.Place(profiles, nil)
|
||||
pspan.SetAttributes(attribute.Bool("llmgw.spread_given_up", res.Placement != nil))
|
||||
}
|
||||
placementAttrs(pspan, res)
|
||||
pspan.End()
|
||||
|
||||
r.mu.Lock()
|
||||
r.lastExplain[m.Name] = res
|
||||
@@ -852,10 +925,12 @@ func (r *Reconciler) scaleUp(ctx context.Context, m catalog.Model, avoid []strin
|
||||
{Profile: res.Placement.Profile.Name, Reason: why}}}
|
||||
r.mu.Unlock()
|
||||
r.notePlacementFailure(m.Name, why)
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.over_limit", why))
|
||||
return overLimit
|
||||
}
|
||||
if err := r.startReplica(ctx, m, res.Placement); err != nil {
|
||||
r.log.Error("starting replica failed", "model", m.Name, "err", err)
|
||||
trace.SpanFromContext(ctx).SetStatus(codes.Error, err.Error())
|
||||
return noRoom
|
||||
}
|
||||
return placed
|
||||
@@ -872,6 +947,51 @@ const (
|
||||
overLimit // there is room, but the owner's quota does not allow it
|
||||
)
|
||||
|
||||
// String names an outcome for a trace.
|
||||
func (o scaleOutcome) String() string {
|
||||
switch o {
|
||||
case placed:
|
||||
return "placed"
|
||||
case makingRoom:
|
||||
return "making room"
|
||||
case noRoom:
|
||||
return "no room"
|
||||
case overLimit:
|
||||
return "over its quota"
|
||||
default:
|
||||
return "not tried"
|
||||
}
|
||||
}
|
||||
|
||||
// placementAttrs puts a placement's answer on its span: where it went, or
|
||||
// every candidate it turned down and why, which is what `ctl
|
||||
// explain-placement` would have printed at that moment.
|
||||
func placementAttrs(span trace.Span, res placement.Result) {
|
||||
span.SetAttributes(attribute.Int("llmgw.rejections", len(res.Rejections)))
|
||||
pl := res.Placement
|
||||
if pl == nil {
|
||||
explain := res.Explain()
|
||||
if len(explain) > 4096 {
|
||||
explain = explain[:4096] + "…"
|
||||
}
|
||||
span.SetAttributes(attribute.String("llmgw.explain", explain))
|
||||
span.SetStatus(codes.Error, "fits nowhere")
|
||||
return
|
||||
}
|
||||
nodes := make([]string, 0, len(pl.Nodes))
|
||||
for _, a := range pl.Nodes {
|
||||
nodes = append(nodes, a.NodeID)
|
||||
}
|
||||
span.SetAttributes(
|
||||
attribute.String("llmgw.profile", pl.Profile.Name),
|
||||
attribute.String("llmgw.engine", pl.Profile.Engine),
|
||||
attribute.String("llmgw.gpu_class", pl.Profile.GPUClass),
|
||||
attribute.String("llmgw.pool", pl.Pool),
|
||||
attribute.String("llmgw.fabric", pl.Fabric),
|
||||
attribute.StringSlice("llmgw.nodes", nodes),
|
||||
)
|
||||
}
|
||||
|
||||
func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *placement.Placement) error {
|
||||
prof, ok := m.ProfileByName(pl.Profile.Name)
|
||||
if !ok {
|
||||
@@ -936,6 +1056,19 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
slices.SortFunc(order, func(a, b Shard) int { return a.Rank - b.Rank })
|
||||
nodeIDs := rep.NodeIDs()
|
||||
|
||||
// Open until the replica serves, fails or is stopped, and the parent of
|
||||
// every node's part of the start: the agents' spans arrive under it.
|
||||
rctx, rspan := startSpan(ctx, "replica start", append(replicaAttrs(rep),
|
||||
attribute.String("llmgw.engine", prof.Engine),
|
||||
attribute.String("llmgw.gpu_class", prof.GPUClass),
|
||||
attribute.String("llmgw.pool", pl.Pool),
|
||||
attribute.String("llmgw.fabric", pl.Fabric),
|
||||
attribute.String("llmgw.endpoint", rep.Endpoint),
|
||||
attribute.Int("llmgw.attempt", attempt),
|
||||
attribute.String("llmgw.startup_timeout", r.startupTimeout(rep).String()),
|
||||
)...)
|
||||
rep.startSpan, rep.startTrace = rspan, rspan.SpanContext()
|
||||
|
||||
r.mu.Lock()
|
||||
r.replicas[id] = rep
|
||||
r.mu.Unlock()
|
||||
@@ -965,7 +1098,19 @@ func (r *Reconciler) startReplica(ctx context.Context, m catalog.Model, pl *plac
|
||||
Fabric: pl.Fabric,
|
||||
ProfileFP: prof.Fingerprint(),
|
||||
}
|
||||
if err := r.disp.StartShard(ctx, cmd); err != nil {
|
||||
// The command, sent: its context travels with it to the agent.
|
||||
sctx, sspan := tracer().Start(rctx, "send start",
|
||||
trace.WithSpanKind(trace.SpanKindProducer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", s.ID),
|
||||
attribute.String("llmgw.node", s.NodeID),
|
||||
attribute.Int("llmgw.rank", s.Rank),
|
||||
attribute.IntSlice("llmgw.gpus", s.GPUIndices),
|
||||
attribute.Int("llmgw.port", s.HTTPPort),
|
||||
))
|
||||
err := r.disp.StartShard(sctx, cmd)
|
||||
obs.End(sspan, err)
|
||||
if err != nil {
|
||||
r.markShard(id, s.ID, supervisor.Failed, err.Error())
|
||||
return fmt.Errorf("starting shard %s: %w", s.ID, err)
|
||||
}
|
||||
@@ -1031,7 +1176,8 @@ func (r *Reconciler) allocatePorts(pl *placement.Placement) int {
|
||||
return r.cfg.PortRange[0]
|
||||
}
|
||||
|
||||
func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
|
||||
// scaleDown drains replicas; why is for the trace.
|
||||
func (r *Reconciler) scaleDown(ctx context.Context, excess []string, why string) {
|
||||
for _, id := range excess {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
@@ -1045,15 +1191,12 @@ func (r *Reconciler) scaleDown(ctx context.Context, excess []string) {
|
||||
r.mu.Unlock()
|
||||
|
||||
r.log.Info("draining replica", "replica", id)
|
||||
for _, s := range shards {
|
||||
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
||||
}
|
||||
}
|
||||
r.drainShards(ctx, id, shards, why)
|
||||
}
|
||||
}
|
||||
|
||||
func (r *Reconciler) stopReplica(ctx context.Context, id string) {
|
||||
// stopReplica stops a replica's shards and forgets it; why is for the trace.
|
||||
func (r *Reconciler) stopReplica(ctx context.Context, id, why string) {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
if !ok {
|
||||
@@ -1062,6 +1205,10 @@ func (r *Reconciler) stopReplica(ctx context.Context, id string) {
|
||||
}
|
||||
shards := slices.Clone(rep.Shards)
|
||||
rep.State = Stopped
|
||||
rep.startDone(r.now())
|
||||
ctx, span := startSpan(ctx, "stop replica", append(replicaAttrs(rep),
|
||||
attribute.String("llmgw.reason", why))...)
|
||||
defer span.End()
|
||||
r.mu.Unlock()
|
||||
|
||||
for _, s := range shards {
|
||||
@@ -1130,6 +1277,18 @@ func (r *Reconciler) markShard(replicaID, shardID string, st supervisor.State, e
|
||||
|
||||
prev := rep.State
|
||||
rep.deriveState(r.now())
|
||||
if rep.startSpan != nil {
|
||||
ev := []attribute.KeyValue{
|
||||
attribute.String("llmgw.shard", shardID), attribute.String("llmgw.state", st.String()),
|
||||
}
|
||||
if errMsg != "" {
|
||||
ev = append(ev, attribute.String("llmgw.error", errMsg))
|
||||
}
|
||||
rep.startSpan.AddEvent("shard "+st.String(), trace.WithAttributes(ev...))
|
||||
}
|
||||
if rep.State == Ready || rep.State.Terminal() {
|
||||
rep.startDone(r.now())
|
||||
}
|
||||
if rep.State == Ready && prev != Ready {
|
||||
// A model that comes up clears its backoff; the next failure starts
|
||||
// counting from scratch rather than from an old streak.
|
||||
|
||||
@@ -11,6 +11,8 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
@@ -24,9 +26,12 @@ type fakeDispatcher struct {
|
||||
stopped []string
|
||||
drained []string
|
||||
failNext error
|
||||
// traces are the span each start was sent under, which is what the
|
||||
// real dispatcher puts on the command for the agent.
|
||||
traces []trace.SpanContext
|
||||
}
|
||||
|
||||
func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
|
||||
func (d *fakeDispatcher) StartShard(ctx context.Context, cmd ShardCommand) error {
|
||||
d.mu.Lock()
|
||||
defer d.mu.Unlock()
|
||||
if d.failNext != nil {
|
||||
@@ -35,6 +40,7 @@ func (d *fakeDispatcher) StartShard(_ context.Context, cmd ShardCommand) error {
|
||||
return err
|
||||
}
|
||||
d.started = append(d.started, cmd)
|
||||
d.traces = append(d.traces, trace.SpanContextFromContext(ctx))
|
||||
return nil
|
||||
}
|
||||
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
)
|
||||
|
||||
@@ -166,8 +168,19 @@ type Replica struct {
|
||||
// carried across rebuilds so the backoff does not reset on every attempt.
|
||||
Attempt int
|
||||
Err string
|
||||
|
||||
// startSpan is the "replica start" span, open from placement until the
|
||||
// replica serves, fails or is stopped; nil once closed, and for a replica
|
||||
// adopted rather than started. startTrace names it for good.
|
||||
startSpan trace.Span
|
||||
startTrace trace.SpanContext
|
||||
}
|
||||
|
||||
// StartTrace names the span of this replica's start, for something that
|
||||
// waited on it — a job serving a model — to link to. Invalid for a replica
|
||||
// adopted rather than started, and whenever tracing is off.
|
||||
func (r Replica) StartTrace() trace.SpanContext { return r.startTrace }
|
||||
|
||||
// NodeIDs lists the nodes this replica occupies.
|
||||
func (r Replica) NodeIDs() []string {
|
||||
out := make([]string, 0, len(r.Shards))
|
||||
|
||||
@@ -7,6 +7,8 @@ import (
|
||||
"slices"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
)
|
||||
|
||||
@@ -48,39 +50,39 @@ func (r *Reconciler) retire(ctx context.Context, m catalog.Model, ids []string,
|
||||
}
|
||||
switch {
|
||||
case readyLive >= desired:
|
||||
r.drainForRestart(ctx, id)
|
||||
r.drainForRestart(ctx, id, "restarted: its replacement serves")
|
||||
case inPlace:
|
||||
r.log.Warn("restarting a replica in place, as asked: stopped before its replacement starts", "replica", id)
|
||||
r.stopForRestart(ctx, id)
|
||||
r.stopForRestart(ctx, id, "restarted in place, as asked")
|
||||
case live >= desired:
|
||||
// A replacement is placed and loading; it is waited for.
|
||||
case parked:
|
||||
r.refuseRestart(id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
|
||||
r.refuseRestart(ctx, id, "its replacements fail to start; the model is parked (llmgw ctl retry "+m.Name+")")
|
||||
case outcome == noRoom && r.fitsWithout(m, nil, []victim{{ID: id, Model: m.Name}}):
|
||||
r.log.Warn("restarting a replica in place: it is the only thing in its replacement's way",
|
||||
"replica", id, "model", m.Name)
|
||||
r.stopForRestart(ctx, id)
|
||||
r.stopForRestart(ctx, id, "restarted in place: it was the only thing in its replacement's way")
|
||||
case outcome == noRoom || outcome == overLimit:
|
||||
r.refuseRestart(id, "its replacement fits on no node even with this replica stopped "+
|
||||
r.refuseRestart(ctx, id, "its replacement fits on no node even with this replica stopped "+
|
||||
"(llmgw ctl explain-placement "+m.Name+")")
|
||||
case outcome == notTried && r.now().Sub(since) > r.cfg.DrainGrace:
|
||||
r.log.Warn("restarting a replica in place: no replacement beside it",
|
||||
"replica", id, "after", r.cfg.DrainGrace)
|
||||
r.stopForRestart(ctx, id)
|
||||
r.stopForRestart(ctx, id, "restarted in place: no replacement beside it within "+r.cfg.DrainGrace.String())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// stopForRestart drains a replica and stops it at once, for its replacement
|
||||
// to take its place.
|
||||
func (r *Reconciler) stopForRestart(ctx context.Context, id string) {
|
||||
r.drainForRestart(ctx, id)
|
||||
r.stopReplica(ctx, id)
|
||||
func (r *Reconciler) stopForRestart(ctx context.Context, id, why string) {
|
||||
r.drainForRestart(ctx, id, why)
|
||||
r.stopReplica(ctx, id, why)
|
||||
}
|
||||
|
||||
// refuseRestart gives up a replica's restart, leaving it serving, and keeps
|
||||
// why for whoever looks at the replica.
|
||||
func (r *Reconciler) refuseRestart(id, why string) {
|
||||
func (r *Reconciler) refuseRestart(ctx context.Context, id, why string) {
|
||||
r.mu.Lock()
|
||||
if rep, ok := r.replicas[id]; ok {
|
||||
rep.RetiringAt = time.Time{}
|
||||
@@ -89,9 +91,12 @@ func (r *Reconciler) refuseRestart(id, why string) {
|
||||
r.restartRefused[id] = why
|
||||
r.mu.Unlock()
|
||||
r.log.Warn("not restarting a replica: it keeps serving", "replica", id, "because", why)
|
||||
_, span := startSpan(ctx, "restart refused",
|
||||
attribute.String("llmgw.replica", id), attribute.String("llmgw.reason", why))
|
||||
span.End()
|
||||
}
|
||||
|
||||
func (r *Reconciler) drainForRestart(ctx context.Context, id string) {
|
||||
func (r *Reconciler) drainForRestart(ctx context.Context, id, why string) {
|
||||
r.mu.Lock()
|
||||
rep, ok := r.replicas[id]
|
||||
var shards []Shard
|
||||
@@ -101,10 +106,8 @@ func (r *Reconciler) drainForRestart(ctx context.Context, id string) {
|
||||
}
|
||||
delete(r.restartInPlace, id)
|
||||
r.mu.Unlock()
|
||||
for _, s := range shards {
|
||||
if err := r.disp.DrainShard(ctx, s.NodeID, s.ID); err != nil {
|
||||
r.log.Warn("draining a shard failed", "shard", s.ID, "err", err)
|
||||
}
|
||||
if ok {
|
||||
r.drainShards(ctx, id, shards, why)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,122 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// What is traced here, and why it is shaped this way.
|
||||
//
|
||||
// A pass runs every few seconds and nearly always finds nothing to do. A
|
||||
// trace per pass would bury the few that placed or stopped something under
|
||||
// seventeen thousand empty ones a day, so a pass's span is begun only when
|
||||
// the pass first does something, and backdated to when the pass began.
|
||||
//
|
||||
// A replica's start outlives the pass that placed it by minutes. Its span,
|
||||
// "replica start", is opened beneath the placement and closed when the
|
||||
// replica serves, fails, or is stopped, and the command to each node carries
|
||||
// it, so the agent's spans for that shard — weights, launch, readiness — are
|
||||
// its descendants. That one trace is the answer to "why did this model take
|
||||
// four minutes to start".
|
||||
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/control/reconciler"
|
||||
|
||||
func tracer() trace.Tracer { return obs.Tracer(tracerName) }
|
||||
|
||||
// pass is one reconcile pass's span, begun on first use.
|
||||
type pass struct {
|
||||
start time.Time
|
||||
base context.Context
|
||||
|
||||
once sync.Once
|
||||
ctx context.Context
|
||||
span trace.Span
|
||||
}
|
||||
|
||||
type passKey struct{}
|
||||
|
||||
// beginPass marks ctx as a pass's, for startSpan to hang its spans on.
|
||||
func beginPass(ctx context.Context) (context.Context, *pass) {
|
||||
p := &pass{start: time.Now(), base: ctx}
|
||||
return context.WithValue(ctx, passKey{}, p), p
|
||||
}
|
||||
|
||||
// context is the pass's span's context, beginning the span if it has not
|
||||
// been.
|
||||
func (p *pass) context() context.Context {
|
||||
p.once.Do(func() {
|
||||
p.ctx, p.span = tracer().Start(p.base, "reconcile", trace.WithTimestamp(p.start))
|
||||
})
|
||||
return p.ctx
|
||||
}
|
||||
|
||||
// end closes the pass's span, if anything began it.
|
||||
func (p *pass) end() {
|
||||
p.once.Do(func() {}) // nothing may begin it after this
|
||||
if p.span != nil {
|
||||
p.span.End()
|
||||
}
|
||||
}
|
||||
|
||||
// startSpan opens a span for something the reconciler did: beneath the span
|
||||
// already in ctx, or beneath the pass's — begun now — when ctx holds the
|
||||
// pass and nothing else.
|
||||
func startSpan(ctx context.Context, name string, attrs ...attribute.KeyValue) (context.Context, trace.Span) {
|
||||
if !trace.SpanContextFromContext(ctx).IsValid() {
|
||||
if p, ok := ctx.Value(passKey{}).(*pass); ok {
|
||||
if pctx := p.context(); pctx != nil {
|
||||
ctx = pctx
|
||||
}
|
||||
}
|
||||
}
|
||||
return tracer().Start(ctx, name, trace.WithAttributes(attrs...))
|
||||
}
|
||||
|
||||
// endSpan closes a span, marking it failed when why is not empty.
|
||||
func endSpan(span trace.Span, why string) {
|
||||
if why != "" {
|
||||
obs.End(span, errors.New(why))
|
||||
return
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
|
||||
// replicaAttrs are what every span about one replica says.
|
||||
func replicaAttrs(rep *Replica) []attribute.KeyValue {
|
||||
return []attribute.KeyValue{
|
||||
attribute.String("llmgw.replica", rep.ID),
|
||||
attribute.String("llmgw.model", rep.Model),
|
||||
attribute.String("llmgw.profile", rep.Profile),
|
||||
attribute.StringSlice("llmgw.nodes", rep.NodeIDs()),
|
||||
}
|
||||
}
|
||||
|
||||
// startDone closes a replica's "replica start" span, once: when it serves,
|
||||
// with how long that took, or when it fails or is stopped first, with why.
|
||||
// Called with r.mu held, or before the replica is in the map.
|
||||
func (rep *Replica) startDone(now time.Time) {
|
||||
if rep.startSpan == nil {
|
||||
return
|
||||
}
|
||||
span := rep.startSpan
|
||||
rep.startSpan = nil
|
||||
span.SetAttributes(attribute.String("llmgw.state", rep.State.String()))
|
||||
switch rep.State {
|
||||
case Ready:
|
||||
span.SetAttributes(attribute.Float64("llmgw.ready_after_seconds", now.Sub(rep.CreatedAt).Seconds()))
|
||||
span.SetStatus(codes.Ok, "")
|
||||
case Failed:
|
||||
span.SetStatus(codes.Error, rep.Err)
|
||||
default:
|
||||
span.SetStatus(codes.Error, "stopped before it was ready")
|
||||
}
|
||||
span.End()
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
package reconciler
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/catalog"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAPlacementIsTracedFromThePassToTheCommandAndTheReplicaServing(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
pass := rec.One(t, [16]byte{}, "reconcile")
|
||||
tid := pass.SpanContext().TraceID()
|
||||
up := rec.One(t, tid, "scale up")
|
||||
place := rec.One(t, tid, "place")
|
||||
if !obstest.ChildOf(up, pass) || !obstest.ChildOf(place, up) {
|
||||
t.Fatal("the placement is not beneath the scale-up, beneath the pass")
|
||||
}
|
||||
if a := obstest.Attrs(place); a["llmgw.model"] != "llama" || a["llmgw.profile"] != "tp4" ||
|
||||
a["llmgw.gpu_class"] != "h100-80g" || a["llmgw.nodes"] != `["1"]` {
|
||||
t.Errorf("the placement does not say where it went: %v", a)
|
||||
}
|
||||
if a := obstest.Attrs(up); a["llmgw.outcome"] != "placed" || a["llmgw.desired"] != "1" {
|
||||
t.Errorf("the scale-up does not say what it did: %v", a)
|
||||
}
|
||||
|
||||
// The command went under a span of its own, beneath the replica's start,
|
||||
// which is still open: the replica is loading.
|
||||
send := rec.One(t, tid, "send start")
|
||||
if got := h.disp.traces; len(got) != 1 || got[0].SpanID() != send.SpanContext().SpanID() {
|
||||
t.Fatalf("the command was not sent under its span: %v", got)
|
||||
}
|
||||
if len(rec.Named("replica start")) != 0 {
|
||||
t.Fatal("the replica's start ended before the replica served")
|
||||
}
|
||||
|
||||
h.markReady()
|
||||
start := rec.One(t, tid, "replica start")
|
||||
if !obstest.ChildOf(start, up) || !obstest.ChildOf(send, start) {
|
||||
t.Error("the replica's start is not beneath the scale-up, or the command beneath it")
|
||||
}
|
||||
a := obstest.Attrs(start)
|
||||
if a["llmgw.state"] != "ready" || a["llmgw.profile"] != "tp4" || a["llmgw.ready_after_seconds"] == "" {
|
||||
t.Errorf("the replica's start does not say how it ended: %v", a)
|
||||
}
|
||||
if start.Status().Code != codes.Ok {
|
||||
t.Errorf("a replica that serves is %v", start.Status())
|
||||
}
|
||||
events := start.Events()
|
||||
if len(events) == 0 || events[len(events)-1].Name != "shard ready" {
|
||||
t.Errorf("the shard's progress is not on the replica's start: %v", events)
|
||||
}
|
||||
if sc := h.r.Replicas()[0].StartTrace(); sc.SpanID() != start.SpanContext().SpanID() {
|
||||
t.Error("the replica does not name its start's span for a job to link to")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPassThatDoesNothingLeavesNoTrace(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
before := len(rec.Ended())
|
||||
|
||||
// Every pass from here finds the model as it should be.
|
||||
for range 3 {
|
||||
h.r.Reconcile(t.Context())
|
||||
}
|
||||
if got := rec.Ended()[before:]; len(got) != 0 {
|
||||
var names []string
|
||||
for _, s := range got {
|
||||
names = append(names, s.Name())
|
||||
}
|
||||
t.Errorf("passes with nothing to do were traced: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPlacementThatFitsNowhereSaysWhy(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
m := testModel(1, catalog.Profile{Name: "tp16", GPUClass: "h100-80g", TP: 16})
|
||||
h := newHarness(t, m, gpuNode("1", 8))
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
place := rec.One(t, [16]byte{}, "place")
|
||||
if place.Status().Code != codes.Error {
|
||||
t.Errorf("a placement that found nowhere is %v", place.Status())
|
||||
}
|
||||
if a := obstest.Attrs(place); a["llmgw.rejections"] == "0" || !strings.Contains(a["llmgw.explain"], "tp16") {
|
||||
t.Errorf("the placement does not say what it turned down: %v", a)
|
||||
}
|
||||
if a := obstest.Attrs(rec.One(t, place.SpanContext().TraceID(), "scale up")); a["llmgw.outcome"] != "no room" {
|
||||
t.Errorf("the scale-up says %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAFailedStartEndsItsTraceWithTheEnginesWords(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
cmd := h.disp.startedShards()[0]
|
||||
|
||||
h.r.OnShardStatus(cmd.ReplicaID, cmd.ShardID, supervisor.Failed, "CUDA out of memory")
|
||||
start := rec.One(t, [16]byte{}, "replica start")
|
||||
if start.Status().Code != codes.Error || !strings.Contains(start.Status().Description, "CUDA out of memory") {
|
||||
t.Errorf("the failed start says %+v", start.Status())
|
||||
}
|
||||
|
||||
// The next pass reaps it, in a trace of its own, saying when it is tried
|
||||
// again.
|
||||
h.r.Reconcile(t.Context())
|
||||
reap := rec.One(t, [16]byte{}, "reap replica")
|
||||
a := obstest.Attrs(reap)
|
||||
if a["llmgw.replica"] != cmd.ReplicaID || a["llmgw.failures_in_a_row"] != "1" || a["llmgw.retry_in_seconds"] == "" {
|
||||
t.Errorf("the reaping does not say what happens next: %v", a)
|
||||
}
|
||||
}
|
||||
|
||||
func TestADispatchThatFailsIsTracedAsFailed(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.disp.failNext = errors.New("no agent connected")
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
send := rec.One(t, [16]byte{}, "send start")
|
||||
if send.Status().Code != codes.Error {
|
||||
t.Errorf("a command that was not sent is %v", send.Status())
|
||||
}
|
||||
if start := rec.One(t, [16]byte{}, "replica start"); start.Status().Code != codes.Error {
|
||||
t.Errorf("a replica none of whose shards was sent is %v", start.Status())
|
||||
}
|
||||
}
|
||||
|
||||
func TestAPreemptionIsTracedWithItsVictims(t *testing.T) {
|
||||
h := fillThenWant(t, gpuModel("vital", catalog.ClassCritical, 1), gpuModel("chat", "", 1), gpuNode("1", 8))
|
||||
rec := obstest.Record(t)
|
||||
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
pre := rec.One(t, [16]byte{}, "preempt")
|
||||
a := obstest.Attrs(pre)
|
||||
if a["llmgw.for"] != "vital" || !strings.Contains(a["llmgw.victims"], "chat") {
|
||||
t.Errorf("the preemption does not say who gave way to whom: %v", a)
|
||||
}
|
||||
drain := rec.One(t, pre.SpanContext().TraceID(), "drain replica")
|
||||
if !obstest.ChildOf(drain, pre) || obstest.Attrs(drain)["llmgw.reason"] != "preempted for vital" {
|
||||
t.Errorf("the victim's drain is not beneath the preemption: %v", obstest.Attrs(drain))
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStopSaysWhy(t *testing.T) {
|
||||
h := newHarness(t, testModel(1), gpuNode("1", 8))
|
||||
h.r.Reconcile(t.Context())
|
||||
h.markReady()
|
||||
rec := obstest.Record(t)
|
||||
|
||||
h.setModels(t) // the model leaves the catalog
|
||||
h.r.Reconcile(t.Context())
|
||||
|
||||
stop := rec.One(t, [16]byte{}, "stop replica")
|
||||
if a := obstest.Attrs(stop); a["llmgw.model"] != "llama" || a["llmgw.reason"] != "its model left the catalog" {
|
||||
t.Errorf("the stop does not say why: %v", a)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user