Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
e725adc2f0
33 files changed
+1607
-143
No files matched your search
+57
-4
@@ -23,6 +23,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
@@ -33,6 +35,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
@@ -419,6 +422,17 @@ func int32Slice(v []int) []int32 {
|
||||
return out
|
||||
}
|
||||
|
||||
func int32Ints(v []int32) []int {
|
||||
out := make([]int, len(v))
|
||||
for i, x := range v {
|
||||
out[i] = int(x)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tracerName names the spans the agent opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent"
|
||||
|
||||
// scan refreshes the node's hardware inventory.
|
||||
func (a *Agent) scan(ctx context.Context) error {
|
||||
// Carried from the agent's own configuration rather than duplicated into
|
||||
@@ -786,15 +800,23 @@ func (a *Agent) removeWeights(paths []string) {
|
||||
}
|
||||
|
||||
func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) {
|
||||
// The control plane's span that sent this, when it is tracing: what is
|
||||
// done here for a start or a stop goes into the same trace.
|
||||
tctx := obs.Extract(ctx, msg.GetTrace())
|
||||
switch m := msg.Msg.(type) {
|
||||
case *llmgwv1.ControlMessage_Start:
|
||||
go a.startShard(ctx, m.Start, out)
|
||||
go a.startShard(tctx, m.Start, out)
|
||||
case *llmgwv1.ControlMessage_Stop:
|
||||
go a.stopShard(ctx, m.Stop.GetShardId(), out)
|
||||
go a.stopShard(tctx, m.Stop.GetShardId(), out)
|
||||
case *llmgwv1.ControlMessage_Drain:
|
||||
if err := a.sup.Drain(m.Drain.GetShardId()); err != nil {
|
||||
_, span := obs.Tracer(tracerName).Start(tctx, "drain shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(attribute.String("llmgw.shard", m.Drain.GetShardId())))
|
||||
err := a.sup.Drain(m.Drain.GetShardId())
|
||||
if err != nil {
|
||||
a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err)
|
||||
}
|
||||
obs.End(span, err)
|
||||
case *llmgwv1.ControlMessage_RemoveWeights:
|
||||
a.removeWeights(m.RemoveWeights.GetPaths())
|
||||
case *llmgwv1.ControlMessage_Provision:
|
||||
@@ -927,6 +949,25 @@ func localAddr(ip netip.Addr) bool {
|
||||
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
|
||||
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
|
||||
|
||||
// This node's whole part of the start, from the command arriving to the
|
||||
// engine answering, beneath the control plane's span that sent it. The
|
||||
// steps — weights, launch, readiness — are its children.
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "start shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", a.cfg.NodeID),
|
||||
attribute.String("llmgw.replica", replicaID),
|
||||
attribute.String("llmgw.shard", shardID),
|
||||
attribute.Int("llmgw.rank", int(m.GetRank())),
|
||||
attribute.String("llmgw.model", m.GetProfile().GetServedModelName()),
|
||||
attribute.String("llmgw.profile", m.GetProfile().GetName()),
|
||||
attribute.String("llmgw.engine", m.GetProfile().GetEngine()),
|
||||
attribute.String("llmgw.gpu_class", m.GetProfile().GetGpuClass()),
|
||||
attribute.IntSlice("llmgw.gpus", int32Ints(m.GetGpuIndices())),
|
||||
))
|
||||
var failed error
|
||||
defer func() { obs.End(span, failed) }()
|
||||
|
||||
a.mu.Lock()
|
||||
a.shardReplica[shardID] = replicaID
|
||||
a.mu.Unlock()
|
||||
@@ -939,6 +980,9 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
}
|
||||
}
|
||||
report := func(st supervisor.State, errMsg string, tail []string) {
|
||||
if st == supervisor.Failed {
|
||||
failed = errors.New(errMsg)
|
||||
}
|
||||
send(&llmgwv1.ShardStatus{
|
||||
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
|
||||
})
|
||||
@@ -1109,9 +1153,18 @@ func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgw
|
||||
delete(a.shardReplica, shardID)
|
||||
a.mu.Unlock()
|
||||
|
||||
if err := a.sup.Stop(ctx, shardID); err != nil {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", a.cfg.NodeID),
|
||||
attribute.String("llmgw.replica", replicaID),
|
||||
attribute.String("llmgw.shard", shardID),
|
||||
))
|
||||
err := a.sup.Stop(ctx, shardID)
|
||||
if err != nil {
|
||||
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
|
||||
}
|
||||
obs.End(span, err)
|
||||
a.sup.Remove(shardID)
|
||||
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
@@ -94,6 +95,13 @@ type Config struct {
|
||||
Engines engines.Config `koanf:"engines"`
|
||||
Superv supervisor.Config `koanf:"supervisor"`
|
||||
|
||||
// Tracing sends this node's part of each replica start — the weights
|
||||
// fetched, the engine launched, the wait for it to answer — to an
|
||||
// OpenTelemetry collector, inside the control plane's trace of that
|
||||
// start. Give it the collector server.yaml's tracing names. Off unless
|
||||
// an endpoint is set here or in OTEL_EXPORTER_OTLP_ENDPOINT.
|
||||
Tracing obs.Tracing `koanf:"tracing"`
|
||||
|
||||
Version string
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// Want is one engine at one version.
|
||||
@@ -251,7 +255,17 @@ func (in *Installer) ready(st Status, b Build, bin, version string) Status {
|
||||
//
|
||||
// Into a scratch directory renamed into place at the end, so a failure halfway
|
||||
// leaves nothing that looks installed.
|
||||
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
|
||||
//
|
||||
// Traced as "install engine", a trace of its own: an install is not part of
|
||||
// any one start, and runs once per version rather than once per replica.
|
||||
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) (err error) {
|
||||
ctx, span := obs.Tracer("gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision").Start(ctx, "install engine",
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.engine", w.Engine),
|
||||
attribute.String("llmgw.engine_version", w.Version),
|
||||
attribute.String("llmgw.engine_build", w.Variant),
|
||||
))
|
||||
defer func() { obs.End(span, err) }()
|
||||
if b.Python != "" {
|
||||
return in.installPython(ctx, r, w, b, dir, st)
|
||||
}
|
||||
|
||||
@@ -18,9 +18,19 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// tracerName names the spans the supervisor opens. There is no restart span
|
||||
// here because the supervisor never restarts an engine on its own: a restart
|
||||
// is the control plane's decision, traced there, and reaches a node as a stop
|
||||
// and a start.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
|
||||
// Config configures the supervisor.
|
||||
type Config struct {
|
||||
// WorkDir holds pid files and per-shard scratch space.
|
||||
@@ -195,7 +205,25 @@ var ErrAlreadyRunning = errors.New("shard is already running")
|
||||
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
|
||||
// rather than a second process, because the control plane retries commands
|
||||
// (plan §5.6).
|
||||
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (*Shard, error) {
|
||||
//
|
||||
// Traced as "launch engine": from the first process to the engine's being
|
||||
// started, which for a multi-node worker includes waiting for its leader's
|
||||
// rendezvous point. Loading the model is the readiness wait's, not this.
|
||||
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (_ *Shard, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "launch engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", id),
|
||||
attribute.String("llmgw.engine", spec.Profile.Engine),
|
||||
attribute.Int("llmgw.processes", len(cmds)),
|
||||
))
|
||||
defer func() {
|
||||
if errors.Is(err, ErrAlreadyRunning) {
|
||||
// A repeated command, which is not this start failing.
|
||||
span.SetAttributes(attribute.Bool("llmgw.already_running", true))
|
||||
span.End()
|
||||
return
|
||||
}
|
||||
obs.End(span, err)
|
||||
}()
|
||||
if len(cmds) == 0 {
|
||||
return nil, fmt.Errorf("shard %s: no commands to run", id)
|
||||
}
|
||||
@@ -301,6 +329,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
sh.pid = c.Process.Pid
|
||||
sh.state = Starting
|
||||
sh.mu.Unlock()
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.Int("llmgw.pid", c.Process.Pid))
|
||||
|
||||
if err := s.writeState(sh); err != nil {
|
||||
s.log.Warn("could not write pid file; adoption after a restart will not work",
|
||||
@@ -337,7 +366,13 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
|
||||
}
|
||||
|
||||
// waitForPort blocks until addr accepts a connection.
|
||||
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
|
||||
//
|
||||
// Traced: a worker's start that took ten minutes because its leader was
|
||||
// still loading reads as exactly that.
|
||||
func waitForPort(ctx context.Context, addr string, timeout time.Duration) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for rendezvous",
|
||||
trace.WithAttributes(attribute.String("llmgw.rendezvous", addr)))
|
||||
defer func() { obs.End(span, err) }()
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -416,7 +451,26 @@ func (s *Supervisor) wait(sh *Shard) {
|
||||
// The timeout comes from the profile because it is model-dependent: loading a
|
||||
// 70B model across two nodes legitimately takes tens of minutes, while a small
|
||||
// model failing to load for ten minutes means something is wrong.
|
||||
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
|
||||
//
|
||||
// Traced as "wait for engine": the model loading, which is usually most of a
|
||||
// start. The span says how many times the engine was asked and what it last
|
||||
// answered, so one that never came up says why.
|
||||
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", sh.ID),
|
||||
attribute.String("llmgw.readiness", "health check"),
|
||||
attribute.String("llmgw.timeout", timeout.String()),
|
||||
))
|
||||
polls := 0
|
||||
var answer error
|
||||
defer func() {
|
||||
span.SetAttributes(attribute.Int("llmgw.polls", polls))
|
||||
if answer != nil && err != nil {
|
||||
span.SetAttributes(attribute.String("llmgw.last_answer", answer.Error()))
|
||||
}
|
||||
obs.End(span, err)
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -432,7 +486,8 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
||||
last := strings.Join(sh.Logs(10), "\n")
|
||||
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
||||
case <-t.C:
|
||||
if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
|
||||
polls++
|
||||
if answer = s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); answer != nil {
|
||||
continue
|
||||
}
|
||||
sh.setState(Ready, nil)
|
||||
@@ -449,7 +504,13 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
||||
// leader answering, which only happens once every worker has joined. Polling a
|
||||
// worker's port instead would time out and fail a replica that is coming up
|
||||
// perfectly well.
|
||||
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) error {
|
||||
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) (err error) {
|
||||
_, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", sh.ID),
|
||||
attribute.String("llmgw.readiness", "stays alive"),
|
||||
attribute.String("llmgw.timeout", s.cfg.AliveGrace.String()),
|
||||
))
|
||||
defer func() { obs.End(span, err) }()
|
||||
sh.setState(Loading, nil)
|
||||
|
||||
select {
|
||||
@@ -483,7 +544,14 @@ func (s *Supervisor) Drain(id string) error {
|
||||
|
||||
// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
|
||||
// StopTimeout.
|
||||
func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
||||
//
|
||||
// Traced as "stop engine", saying whether it had to be killed: an engine
|
||||
// that ignores SIGTERM for a minute is a minute its replacement waits for
|
||||
// the GPU.
|
||||
func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop engine",
|
||||
trace.WithAttributes(attribute.String("llmgw.shard", id)))
|
||||
defer func() { obs.End(span, err) }()
|
||||
sh, err := s.Get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -527,6 +595,7 @@ func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
||||
// force kills the group outright. exec.Cmd's cancel does this for shards we
|
||||
// started; an adopted shard has none, so signal its group directly.
|
||||
force := func() {
|
||||
span.SetAttributes(attribute.Bool("llmgw.killed", true))
|
||||
if sh.cancel != nil {
|
||||
sh.cancel()
|
||||
return
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAStartOnTheNodeIsInTheControlPlanesTrace(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = servingEngine(t)
|
||||
})
|
||||
|
||||
// The control plane's side, as the reconciler sends it: the command
|
||||
// dispatched under a span of its own, over the real stream.
|
||||
ctx, sent := obs.Tracer("test").Start(t.Context(), "send start",
|
||||
trace.WithSpanKind(trace.SpanKindProducer))
|
||||
if err := h.svc.StartShard(ctx, shardCommand(t, "shard-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent.End()
|
||||
h.rec.await(t, 30*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-a" && s.state == supervisor.Ready
|
||||
})
|
||||
|
||||
// The node's part ends just after it reports the shard ready.
|
||||
tid := sent.SpanContext().TraceID()
|
||||
node := rec.Await(t, tid, "start shard", 5*time.Second)
|
||||
if node.Parent().SpanID() != sent.SpanContext().SpanID() {
|
||||
t.Fatal("the node's start is not beneath the control plane's command")
|
||||
}
|
||||
if node.SpanKind() != trace.SpanKindConsumer {
|
||||
t.Errorf("the node's start is a %v span", node.SpanKind())
|
||||
}
|
||||
if a := obstest.Attrs(node); a["llmgw.node"] != "n1" || a["llmgw.replica"] != "rep-1" ||
|
||||
a["llmgw.model"] != "test-model" || a["llmgw.engine"] != "vllm" {
|
||||
t.Errorf("the node's start does not say what it started: %v", a)
|
||||
}
|
||||
if node.Status().Code == codes.Error {
|
||||
t.Errorf("a start that served is %v", node.Status())
|
||||
}
|
||||
for _, name := range []string{"fetch weights", "launch engine", "wait for engine"} {
|
||||
if !obstest.ChildOf(rec.One(t, tid, name), node) {
|
||||
t.Errorf("%s is not beneath the node's start", name)
|
||||
}
|
||||
}
|
||||
if a := obstest.Attrs(rec.One(t, tid, "fetch weights")); a["llmgw.weights.outcome"] != "local" {
|
||||
t.Errorf("the weights do not say how they were found: %v", a)
|
||||
}
|
||||
if a := obstest.Attrs(rec.One(t, tid, "wait for engine")); a["llmgw.polls"] == "" || a["llmgw.polls"] == "0" {
|
||||
t.Errorf("the readiness wait does not say how often it asked: %v", a)
|
||||
}
|
||||
|
||||
// A stop joins the trace it was sent in, too.
|
||||
sctx, stop := obs.Tracer("test").Start(t.Context(), "stop replica")
|
||||
if err := h.svc.StopShard(sctx, "n1", "shard-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop.End()
|
||||
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-a" && s.state == supervisor.Stopped
|
||||
})
|
||||
stid := stop.SpanContext().TraceID()
|
||||
nodeStop := rec.Await(t, stid, "stop shard", 5*time.Second)
|
||||
if !obstest.ChildOf(nodeStop, rec.One(t, stid, "stop replica")) {
|
||||
t.Error("the node's stop is not beneath the control plane's")
|
||||
}
|
||||
if !obstest.ChildOf(rec.One(t, stid, "stop engine"), nodeStop) {
|
||||
t.Error("stopping the engine is not beneath the node's stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptionIsTracedAndItsStopsAreInIt(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
||||
})
|
||||
if err := h.svc.StartShard(t.Context(), shardCommand(t, "shard-orphaned")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-orphaned" && s.state == supervisor.Starting
|
||||
})
|
||||
|
||||
// The control plane, restarted, claims nothing the node is running.
|
||||
h.rec.markUnclaimed("shard-orphaned")
|
||||
h.ln.drop()
|
||||
h.rec.await(t, 20*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-orphaned" && s.state == supervisor.Stopped
|
||||
})
|
||||
|
||||
adopt := rec.Named("adopt")
|
||||
if len(adopt) != 1 {
|
||||
t.Fatalf("%d adoptions traced", len(adopt))
|
||||
}
|
||||
if a := obstest.Attrs(adopt[0]); a["llmgw.node"] != "n1" || a["llmgw.shards"] != "1" || a["llmgw.unclaimed"] != "1" {
|
||||
t.Errorf("the adoption does not say what it found: %v", a)
|
||||
}
|
||||
stop := rec.Await(t, adopt[0].SpanContext().TraceID(), "stop shard", 5*time.Second)
|
||||
if !obstest.ChildOf(stop, adopt[0]) {
|
||||
t.Error("stopping the unclaimed shard is not beneath the adoption")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStartThatFailsOnTheNodeSaysWhyInTheTrace(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = fakeEngineScript(t, `echo "CUDA out of memory" >&2; exit 1`)
|
||||
})
|
||||
|
||||
ctx, sent := obs.Tracer("test").Start(t.Context(), "send start")
|
||||
if err := h.svc.StartShard(ctx, shardCommand(t, "shard-b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent.End()
|
||||
h.rec.await(t, 30*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-b" && s.state == supervisor.Failed
|
||||
})
|
||||
|
||||
tid := sent.SpanContext().TraceID()
|
||||
node := rec.Await(t, tid, "start shard", 5*time.Second)
|
||||
if node.Status().Code != codes.Error {
|
||||
t.Errorf("a start that failed is %v", node.Status())
|
||||
}
|
||||
if wait := rec.One(t, tid, "wait for engine"); wait.Status().Code != codes.Error {
|
||||
t.Errorf("the readiness wait of an engine that exited is %v", wait.Status())
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,11 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// Progress reports how a fetch is going, for anything that wants to say so.
|
||||
@@ -80,13 +85,48 @@ var ErrNotLocal = errors.New("the weights are not on this node and the source sa
|
||||
// local disk, a Lustre mount or an image exported over the fabric, and a
|
||||
// downloader that second-guessed that would copy a hundred gigabytes that were
|
||||
// already present.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
//
|
||||
// Traced as "fetch weights", with what it came to — the file was local,
|
||||
// already cached, downloaded, or downloaded by another shard this one waited
|
||||
// for — since that is most of the difference between a replica that starts
|
||||
// in a minute and one that takes an hour.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string) (path string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "fetch weights",
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
defer func() {
|
||||
span.SetAttributes(attribute.String("llmgw.weights.path", path))
|
||||
obs.End(span, err)
|
||||
}()
|
||||
return f.ensure(ctx, source)
|
||||
}
|
||||
|
||||
// tracerName names the spans this package opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
|
||||
// outcome says on the current span how a source was made present.
|
||||
func outcome(ctx context.Context, how string) {
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.weights.outcome", how))
|
||||
}
|
||||
|
||||
// redact is a source fit for a trace: a URL without its credentials or
|
||||
// query, which is where a token would be.
|
||||
func redact(source string) string {
|
||||
u, err := url.Parse(source)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return source
|
||||
}
|
||||
u.User, u.RawQuery, u.Fragment = nil, "", ""
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (f *Fetcher) ensure(ctx context.Context, source string) (string, error) {
|
||||
if source == "" {
|
||||
return "", fmt.Errorf("no source given")
|
||||
}
|
||||
// A version rather than a file. Everything below assumes one URL is one
|
||||
// file, and a set has to be handled before any of it.
|
||||
if IsSet(source) {
|
||||
outcome(ctx, "set")
|
||||
return f.ensureSet(ctx, source)
|
||||
}
|
||||
if !isRemote(source) {
|
||||
@@ -95,9 +135,11 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
// was never meant to be one. vLLM and SGLang resolve those
|
||||
// themselves, and doing it for them would be both a duplicate
|
||||
// downloader and a change in documented behaviour.
|
||||
outcome(ctx, "left to the engine")
|
||||
return source, nil
|
||||
}
|
||||
if _, err := os.Stat(source); err == nil {
|
||||
outcome(ctx, "local")
|
||||
return source, nil
|
||||
}
|
||||
return "", fmt.Errorf("%w: %s", ErrNotLocal, source)
|
||||
@@ -108,6 +150,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
outcome(ctx, "cached")
|
||||
return target, nil
|
||||
}
|
||||
|
||||
@@ -117,6 +160,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
}
|
||||
if call, running := f.inflight[source]; running {
|
||||
f.mu.Unlock()
|
||||
outcome(ctx, "fetched by another shard")
|
||||
select {
|
||||
case <-call.done:
|
||||
return call.path, call.err
|
||||
@@ -128,6 +172,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
f.inflight[source] = call
|
||||
f.mu.Unlock()
|
||||
|
||||
outcome(ctx, "downloaded")
|
||||
call.path, call.err = f.fetch(ctx, source, target)
|
||||
close(call.done)
|
||||
|
||||
@@ -147,7 +192,15 @@ func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, err
|
||||
// The two differ for a set: the shards waiting on it asked for the version, not
|
||||
// for the fourth file in it, and progress reported under the file's own URL
|
||||
// reaches nobody. What they see is one number climbing across the whole set.
|
||||
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (string, error) {
|
||||
//
|
||||
// Each file is a "download" span, with how many bytes it took and where a
|
||||
// resumed one started from.
|
||||
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (_ string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "download",
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
defer func() { obs.End(span, err) }()
|
||||
|
||||
url, err := resolveURL(source)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -208,6 +261,10 @@ func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string)
|
||||
total := existing + resp.ContentLength
|
||||
f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total})
|
||||
written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total)
|
||||
span.SetAttributes(
|
||||
attribute.Int64("llmgw.weights.resumed_from", existing),
|
||||
attribute.Int64("llmgw.weights.bytes", written),
|
||||
)
|
||||
closeErr := out.Close()
|
||||
if err != nil {
|
||||
// The part file is kept on purpose, so the next attempt resumes.
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package weights
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAFetchSaysInItsTraceWhetherItDownloadedAndHowMuch(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
srv := serveBytes(t, []byte("forty gigabytes, give or take"), nil)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
src := strings.Replace(srv.URL, "http://", "http://user:secret@", 1) + "/m/model.gguf?token=hunter2"
|
||||
|
||||
if _, err := f.Ensure(t.Context(), src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetch := rec.One(t, [16]byte{}, "fetch weights")
|
||||
a := obstest.Attrs(fetch)
|
||||
if a["llmgw.weights.outcome"] != "downloaded" {
|
||||
t.Errorf("the first fetch says %v", a)
|
||||
}
|
||||
if strings.Contains(a["llmgw.weights.source"], "secret") || strings.Contains(a["llmgw.weights.source"], "hunter2") {
|
||||
t.Errorf("the trace holds the source's credentials: %s", a["llmgw.weights.source"])
|
||||
}
|
||||
dl := rec.One(t, fetch.SpanContext().TraceID(), "download")
|
||||
if !obstest.ChildOf(dl, fetch) || obstest.Attrs(dl)["llmgw.weights.bytes"] != "29" {
|
||||
t.Errorf("the download does not say how much it moved: %v", obstest.Attrs(dl))
|
||||
}
|
||||
|
||||
// The second start on this node finds it here.
|
||||
if _, err := f.Ensure(t.Context(), src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := rec.Named("fetch weights")
|
||||
if len(again) != 2 || obstest.Attrs(again[1])["llmgw.weights.outcome"] != "cached" {
|
||||
t.Errorf("the second fetch says %v", obstest.Attrs(again[len(again)-1]))
|
||||
}
|
||||
if len(rec.Named("download")) != 1 {
|
||||
t.Error("a cached file was downloaded again")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user