Merge C20 tracing beyond the router
Conflicts: config imports and docgen sources (both kept); agent.pb.go regenerated with buf from the merged proto (MIG devices and trace context). Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
801e2d51c0
33 files changed
+1608
-144
No files matched your search
+57
-4
@@ -23,6 +23,8 @@ import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
"golang.org/x/net/http2"
|
||||
|
||||
llmgwv1 "gitea.mynodes.xyz/ilker/llmgw/api/gen/llmgw/v1"
|
||||
@@ -33,6 +35,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/agentsvc"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
@@ -419,6 +422,17 @@ func int32Slice(v []int) []int32 {
|
||||
return out
|
||||
}
|
||||
|
||||
func int32Ints(v []int32) []int {
|
||||
out := make([]int, len(v))
|
||||
for i, x := range v {
|
||||
out[i] = int(x)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// tracerName names the spans the agent opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent"
|
||||
|
||||
// scan refreshes the node's hardware inventory.
|
||||
func (a *Agent) scan(ctx context.Context) error {
|
||||
// Carried from the agent's own configuration rather than duplicated into
|
||||
@@ -786,15 +800,23 @@ func (a *Agent) removeWeights(paths []string) {
|
||||
}
|
||||
|
||||
func (a *Agent) handle(ctx context.Context, msg *llmgwv1.ControlMessage, out chan<- *llmgwv1.AgentMessage) {
|
||||
// The control plane's span that sent this, when it is tracing: what is
|
||||
// done here for a start or a stop goes into the same trace.
|
||||
tctx := obs.Extract(ctx, msg.GetTrace())
|
||||
switch m := msg.Msg.(type) {
|
||||
case *llmgwv1.ControlMessage_Start:
|
||||
go a.startShard(ctx, m.Start, out)
|
||||
go a.startShard(tctx, m.Start, out)
|
||||
case *llmgwv1.ControlMessage_Stop:
|
||||
go a.stopShard(ctx, m.Stop.GetShardId(), out)
|
||||
go a.stopShard(tctx, m.Stop.GetShardId(), out)
|
||||
case *llmgwv1.ControlMessage_Drain:
|
||||
if err := a.sup.Drain(m.Drain.GetShardId()); err != nil {
|
||||
_, span := obs.Tracer(tracerName).Start(tctx, "drain shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(attribute.String("llmgw.shard", m.Drain.GetShardId())))
|
||||
err := a.sup.Drain(m.Drain.GetShardId())
|
||||
if err != nil {
|
||||
a.log.Warn("draining failed", "shard", m.Drain.GetShardId(), "err", err)
|
||||
}
|
||||
obs.End(span, err)
|
||||
case *llmgwv1.ControlMessage_RemoveWeights:
|
||||
a.removeWeights(m.RemoveWeights.GetPaths())
|
||||
case *llmgwv1.ControlMessage_Provision:
|
||||
@@ -927,6 +949,25 @@ func localAddr(ip netip.Addr) bool {
|
||||
func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<- *llmgwv1.AgentMessage) {
|
||||
shardID, replicaID := m.GetShardId(), m.GetReplicaId()
|
||||
|
||||
// This node's whole part of the start, from the command arriving to the
|
||||
// engine answering, beneath the control plane's span that sent it. The
|
||||
// steps — weights, launch, readiness — are its children.
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "start shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", a.cfg.NodeID),
|
||||
attribute.String("llmgw.replica", replicaID),
|
||||
attribute.String("llmgw.shard", shardID),
|
||||
attribute.Int("llmgw.rank", int(m.GetRank())),
|
||||
attribute.String("llmgw.model", m.GetProfile().GetServedModelName()),
|
||||
attribute.String("llmgw.profile", m.GetProfile().GetName()),
|
||||
attribute.String("llmgw.engine", m.GetProfile().GetEngine()),
|
||||
attribute.String("llmgw.gpu_class", m.GetProfile().GetGpuClass()),
|
||||
attribute.IntSlice("llmgw.gpus", int32Ints(m.GetGpuIndices())),
|
||||
))
|
||||
var failed error
|
||||
defer func() { obs.End(span, failed) }()
|
||||
|
||||
a.mu.Lock()
|
||||
a.shardReplica[shardID] = replicaID
|
||||
a.mu.Unlock()
|
||||
@@ -939,6 +980,9 @@ func (a *Agent) startShard(ctx context.Context, m *llmgwv1.StartShard, out chan<
|
||||
}
|
||||
}
|
||||
report := func(st supervisor.State, errMsg string, tail []string) {
|
||||
if st == supervisor.Failed {
|
||||
failed = errors.New(errMsg)
|
||||
}
|
||||
send(&llmgwv1.ShardStatus{
|
||||
State: agentsvc.StateToWire(st), Error: errMsg, LastOutput: tail,
|
||||
})
|
||||
@@ -1109,9 +1153,18 @@ func (a *Agent) stopShard(ctx context.Context, shardID string, out chan<- *llmgw
|
||||
delete(a.shardReplica, shardID)
|
||||
a.mu.Unlock()
|
||||
|
||||
if err := a.sup.Stop(ctx, shardID); err != nil {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop shard",
|
||||
trace.WithSpanKind(trace.SpanKindConsumer),
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.node", a.cfg.NodeID),
|
||||
attribute.String("llmgw.replica", replicaID),
|
||||
attribute.String("llmgw.shard", shardID),
|
||||
))
|
||||
err := a.sup.Stop(ctx, shardID)
|
||||
if err != nil {
|
||||
a.log.Warn("stopping a shard failed", "shard", shardID, "err", err)
|
||||
}
|
||||
obs.End(span, err)
|
||||
a.sup.Remove(shardID)
|
||||
|
||||
msg := &llmgwv1.AgentMessage{Msg: &llmgwv1.AgentMessage_Shard{Shard: &llmgwv1.ShardStatus{
|
||||
|
||||
@@ -14,6 +14,7 @@ import (
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/control/inventory"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/tlsconf"
|
||||
)
|
||||
|
||||
@@ -94,6 +95,13 @@ type Config struct {
|
||||
Engines engines.Config `koanf:"engines"`
|
||||
Superv supervisor.Config `koanf:"supervisor"`
|
||||
|
||||
// Tracing sends this node's part of each replica start — the weights
|
||||
// fetched, the engine launched, the wait for it to answer — to an
|
||||
// OpenTelemetry collector, inside the control plane's trace of that
|
||||
// start. Give it the collector server.yaml's tracing names. Off unless
|
||||
// an endpoint is set here or in OTEL_EXPORTER_OTLP_ENDPOINT.
|
||||
Tracing obs.Tracing `koanf:"tracing"`
|
||||
|
||||
Version string
|
||||
}
|
||||
|
||||
|
||||
@@ -20,7 +20,11 @@ import (
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/discovery"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// Want is one engine at one version.
|
||||
@@ -251,7 +255,17 @@ func (in *Installer) ready(st Status, b Build, bin, version string) Status {
|
||||
//
|
||||
// Into a scratch directory renamed into place at the end, so a failure halfway
|
||||
// leaves nothing that looks installed.
|
||||
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) error {
|
||||
//
|
||||
// Traced as "install engine", a trace of its own: an install is not part of
|
||||
// any one start, and runs once per version rather than once per replica.
|
||||
func (in *Installer) install(ctx context.Context, r Recipe, w Want, b Build, dir string, st *Status) (err error) {
|
||||
ctx, span := obs.Tracer("gitea.mynodes.xyz/ilker/llmgw/internal/agent/provision").Start(ctx, "install engine",
|
||||
trace.WithAttributes(
|
||||
attribute.String("llmgw.engine", w.Engine),
|
||||
attribute.String("llmgw.engine_version", w.Version),
|
||||
attribute.String("llmgw.engine_build", w.Variant),
|
||||
))
|
||||
defer func() { obs.End(span, err) }()
|
||||
if b.Python != "" {
|
||||
return in.installPython(ctx, r, w, b, dir, st)
|
||||
}
|
||||
|
||||
@@ -18,9 +18,19 @@ import (
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/engines"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// tracerName names the spans the supervisor opens. There is no restart span
|
||||
// here because the supervisor never restarts an engine on its own: a restart
|
||||
// is the control plane's decision, traced there, and reaches a node as a stop
|
||||
// and a start.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
|
||||
// Config configures the supervisor.
|
||||
type Config struct {
|
||||
// WorkDir holds pid files and per-shard scratch space.
|
||||
@@ -195,7 +205,25 @@ var ErrAlreadyRunning = errors.New("shard is already running")
|
||||
// It is idempotent by ID: a repeated StartShard for the same shard is a no-op
|
||||
// rather than a second process, because the control plane retries commands
|
||||
// (plan §5.6).
|
||||
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (*Shard, error) {
|
||||
//
|
||||
// Traced as "launch engine": from the first process to the engine's being
|
||||
// started, which for a multi-node worker includes waiting for its leader's
|
||||
// rendezvous point. Loading the model is the readiness wait's, not this.
|
||||
func (s *Supervisor) Start(ctx context.Context, id string, spec engines.ShardSpec, cmds []engines.Command) (_ *Shard, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "launch engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", id),
|
||||
attribute.String("llmgw.engine", spec.Profile.Engine),
|
||||
attribute.Int("llmgw.processes", len(cmds)),
|
||||
))
|
||||
defer func() {
|
||||
if errors.Is(err, ErrAlreadyRunning) {
|
||||
// A repeated command, which is not this start failing.
|
||||
span.SetAttributes(attribute.Bool("llmgw.already_running", true))
|
||||
span.End()
|
||||
return
|
||||
}
|
||||
obs.End(span, err)
|
||||
}()
|
||||
if len(cmds) == 0 {
|
||||
return nil, fmt.Errorf("shard %s: no commands to run", id)
|
||||
}
|
||||
@@ -301,6 +329,7 @@ func (s *Supervisor) launch(ctx context.Context, sh *Shard, cmds []engines.Comma
|
||||
sh.pid = c.Process.Pid
|
||||
sh.state = Starting
|
||||
sh.mu.Unlock()
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.Int("llmgw.pid", c.Process.Pid))
|
||||
|
||||
if err := s.writeState(sh); err != nil {
|
||||
s.log.Warn("could not write pid file; adoption after a restart will not work",
|
||||
@@ -337,7 +366,13 @@ func (s *Supervisor) spawn(ctx context.Context, sh *Shard, c engines.Command) (*
|
||||
}
|
||||
|
||||
// waitForPort blocks until addr accepts a connection.
|
||||
func waitForPort(ctx context.Context, addr string, timeout time.Duration) error {
|
||||
//
|
||||
// Traced: a worker's start that took ten minutes because its leader was
|
||||
// still loading reads as exactly that.
|
||||
func waitForPort(ctx context.Context, addr string, timeout time.Duration) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for rendezvous",
|
||||
trace.WithAttributes(attribute.String("llmgw.rendezvous", addr)))
|
||||
defer func() { obs.End(span, err) }()
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -416,7 +451,26 @@ func (s *Supervisor) wait(sh *Shard) {
|
||||
// The timeout comes from the profile because it is model-dependent: loading a
|
||||
// 70B model across two nodes legitimately takes tens of minutes, while a small
|
||||
// model failing to load for ten minutes means something is wrong.
|
||||
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) error {
|
||||
//
|
||||
// Traced as "wait for engine": the model loading, which is usually most of a
|
||||
// start. The span says how many times the engine was asked and what it last
|
||||
// answered, so one that never came up says why.
|
||||
func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model string, timeout time.Duration) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", sh.ID),
|
||||
attribute.String("llmgw.readiness", "health check"),
|
||||
attribute.String("llmgw.timeout", timeout.String()),
|
||||
))
|
||||
polls := 0
|
||||
var answer error
|
||||
defer func() {
|
||||
span.SetAttributes(attribute.Int("llmgw.polls", polls))
|
||||
if answer != nil && err != nil {
|
||||
span.SetAttributes(attribute.String("llmgw.last_answer", answer.Error()))
|
||||
}
|
||||
obs.End(span, err)
|
||||
}()
|
||||
|
||||
ctx, cancel := context.WithTimeout(ctx, timeout)
|
||||
defer cancel()
|
||||
|
||||
@@ -432,7 +486,8 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
||||
last := strings.Join(sh.Logs(10), "\n")
|
||||
return fmt.Errorf("shard %s not ready within %v; last output:\n%s", sh.ID, timeout, last)
|
||||
case <-t.C:
|
||||
if err := s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); err != nil {
|
||||
polls++
|
||||
if answer = s.health.Ready(ctx, baseURL, model, sh.Spec.APIKey); answer != nil {
|
||||
continue
|
||||
}
|
||||
sh.setState(Ready, nil)
|
||||
@@ -449,7 +504,13 @@ func (s *Supervisor) WaitReady(ctx context.Context, sh *Shard, baseURL, model st
|
||||
// leader answering, which only happens once every worker has joined. Polling a
|
||||
// worker's port instead would time out and fail a replica that is coming up
|
||||
// perfectly well.
|
||||
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) error {
|
||||
func (s *Supervisor) WaitAlive(ctx context.Context, sh *Shard) (err error) {
|
||||
_, span := obs.Tracer(tracerName).Start(ctx, "wait for engine", trace.WithAttributes(
|
||||
attribute.String("llmgw.shard", sh.ID),
|
||||
attribute.String("llmgw.readiness", "stays alive"),
|
||||
attribute.String("llmgw.timeout", s.cfg.AliveGrace.String()),
|
||||
))
|
||||
defer func() { obs.End(span, err) }()
|
||||
sh.setState(Loading, nil)
|
||||
|
||||
select {
|
||||
@@ -483,7 +544,14 @@ func (s *Supervisor) Drain(id string) error {
|
||||
|
||||
// Stop terminates a shard: SIGTERM to the whole group, then SIGKILL after
|
||||
// StopTimeout.
|
||||
func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
||||
//
|
||||
// Traced as "stop engine", saying whether it had to be killed: an engine
|
||||
// that ignores SIGTERM for a minute is a minute its replacement waits for
|
||||
// the GPU.
|
||||
func (s *Supervisor) Stop(ctx context.Context, id string) (err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "stop engine",
|
||||
trace.WithAttributes(attribute.String("llmgw.shard", id)))
|
||||
defer func() { obs.End(span, err) }()
|
||||
sh, err := s.Get(id)
|
||||
if err != nil {
|
||||
return err
|
||||
@@ -527,6 +595,7 @@ func (s *Supervisor) Stop(ctx context.Context, id string) error {
|
||||
// force kills the group outright. exec.Cmd's cancel does this for shards we
|
||||
// started; an adopted shard has none, so signal its group directly.
|
||||
force := func() {
|
||||
span.SetAttributes(attribute.Bool("llmgw.killed", true))
|
||||
if sh.cancel != nil {
|
||||
sh.cancel()
|
||||
return
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
package agent
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/codes"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/agent/supervisor"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAStartOnTheNodeIsInTheControlPlanesTrace(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = servingEngine(t)
|
||||
})
|
||||
|
||||
// The control plane's side, as the reconciler sends it: the command
|
||||
// dispatched under a span of its own, over the real stream.
|
||||
ctx, sent := obs.Tracer("test").Start(t.Context(), "send start",
|
||||
trace.WithSpanKind(trace.SpanKindProducer))
|
||||
if err := h.svc.StartShard(ctx, shardCommand(t, "shard-a")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent.End()
|
||||
h.rec.await(t, 30*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-a" && s.state == supervisor.Ready
|
||||
})
|
||||
|
||||
// The node's part ends just after it reports the shard ready.
|
||||
tid := sent.SpanContext().TraceID()
|
||||
node := rec.Await(t, tid, "start shard", 5*time.Second)
|
||||
if node.Parent().SpanID() != sent.SpanContext().SpanID() {
|
||||
t.Fatal("the node's start is not beneath the control plane's command")
|
||||
}
|
||||
if node.SpanKind() != trace.SpanKindConsumer {
|
||||
t.Errorf("the node's start is a %v span", node.SpanKind())
|
||||
}
|
||||
if a := obstest.Attrs(node); a["llmgw.node"] != "n1" || a["llmgw.replica"] != "rep-1" ||
|
||||
a["llmgw.model"] != "test-model" || a["llmgw.engine"] != "vllm" {
|
||||
t.Errorf("the node's start does not say what it started: %v", a)
|
||||
}
|
||||
if node.Status().Code == codes.Error {
|
||||
t.Errorf("a start that served is %v", node.Status())
|
||||
}
|
||||
for _, name := range []string{"fetch weights", "launch engine", "wait for engine"} {
|
||||
if !obstest.ChildOf(rec.One(t, tid, name), node) {
|
||||
t.Errorf("%s is not beneath the node's start", name)
|
||||
}
|
||||
}
|
||||
if a := obstest.Attrs(rec.One(t, tid, "fetch weights")); a["llmgw.weights.outcome"] != "local" {
|
||||
t.Errorf("the weights do not say how they were found: %v", a)
|
||||
}
|
||||
if a := obstest.Attrs(rec.One(t, tid, "wait for engine")); a["llmgw.polls"] == "" || a["llmgw.polls"] == "0" {
|
||||
t.Errorf("the readiness wait does not say how often it asked: %v", a)
|
||||
}
|
||||
|
||||
// A stop joins the trace it was sent in, too.
|
||||
sctx, stop := obs.Tracer("test").Start(t.Context(), "stop replica")
|
||||
if err := h.svc.StopShard(sctx, "n1", "shard-a"); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
stop.End()
|
||||
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-a" && s.state == supervisor.Stopped
|
||||
})
|
||||
stid := stop.SpanContext().TraceID()
|
||||
nodeStop := rec.Await(t, stid, "stop shard", 5*time.Second)
|
||||
if !obstest.ChildOf(nodeStop, rec.One(t, stid, "stop replica")) {
|
||||
t.Error("the node's stop is not beneath the control plane's")
|
||||
}
|
||||
if !obstest.ChildOf(rec.One(t, stid, "stop engine"), nodeStop) {
|
||||
t.Error("stopping the engine is not beneath the node's stop")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAnAdoptionIsTracedAndItsStopsAreInIt(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = fakeEngineScript(t, `sleep 60`)
|
||||
})
|
||||
if err := h.svc.StartShard(t.Context(), shardCommand(t, "shard-orphaned")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
h.rec.await(t, 15*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-orphaned" && s.state == supervisor.Starting
|
||||
})
|
||||
|
||||
// The control plane, restarted, claims nothing the node is running.
|
||||
h.rec.markUnclaimed("shard-orphaned")
|
||||
h.ln.drop()
|
||||
h.rec.await(t, 20*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-orphaned" && s.state == supervisor.Stopped
|
||||
})
|
||||
|
||||
adopt := rec.Named("adopt")
|
||||
if len(adopt) != 1 {
|
||||
t.Fatalf("%d adoptions traced", len(adopt))
|
||||
}
|
||||
if a := obstest.Attrs(adopt[0]); a["llmgw.node"] != "n1" || a["llmgw.shards"] != "1" || a["llmgw.unclaimed"] != "1" {
|
||||
t.Errorf("the adoption does not say what it found: %v", a)
|
||||
}
|
||||
stop := rec.Await(t, adopt[0].SpanContext().TraceID(), "stop shard", 5*time.Second)
|
||||
if !obstest.ChildOf(stop, adopt[0]) {
|
||||
t.Error("stopping the unclaimed shard is not beneath the adoption")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAStartThatFailsOnTheNodeSaysWhyInTheTrace(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
h := start(t, func(c *Config) {
|
||||
c.Engines.VLLM.Bin = fakeEngineScript(t, `echo "CUDA out of memory" >&2; exit 1`)
|
||||
})
|
||||
|
||||
ctx, sent := obs.Tracer("test").Start(t.Context(), "send start")
|
||||
if err := h.svc.StartShard(ctx, shardCommand(t, "shard-b")); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sent.End()
|
||||
h.rec.await(t, 30*time.Second, func(s status) bool {
|
||||
return s.shardID == "shard-b" && s.state == supervisor.Failed
|
||||
})
|
||||
|
||||
tid := sent.SpanContext().TraceID()
|
||||
node := rec.Await(t, tid, "start shard", 5*time.Second)
|
||||
if node.Status().Code != codes.Error {
|
||||
t.Errorf("a start that failed is %v", node.Status())
|
||||
}
|
||||
if wait := rec.One(t, tid, "wait for engine"); wait.Status().Code != codes.Error {
|
||||
t.Errorf("the readiness wait of an engine that exited is %v", wait.Status())
|
||||
}
|
||||
}
|
||||
@@ -26,6 +26,11 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// Progress reports how a fetch is going, for anything that wants to say so.
|
||||
@@ -80,13 +85,48 @@ var ErrNotLocal = errors.New("the weights are not on this node and the source sa
|
||||
// local disk, a Lustre mount or an image exported over the fabric, and a
|
||||
// downloader that second-guessed that would copy a hundred gigabytes that were
|
||||
// already present.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
//
|
||||
// Traced as "fetch weights", with what it came to — the file was local,
|
||||
// already cached, downloaded, or downloaded by another shard this one waited
|
||||
// for — since that is most of the difference between a replica that starts
|
||||
// in a minute and one that takes an hour.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string) (path string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "fetch weights",
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
defer func() {
|
||||
span.SetAttributes(attribute.String("llmgw.weights.path", path))
|
||||
obs.End(span, err)
|
||||
}()
|
||||
return f.ensure(ctx, source)
|
||||
}
|
||||
|
||||
// tracerName names the spans this package opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
|
||||
// outcome says on the current span how a source was made present.
|
||||
func outcome(ctx context.Context, how string) {
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.weights.outcome", how))
|
||||
}
|
||||
|
||||
// redact is a source fit for a trace: a URL without its credentials or
|
||||
// query, which is where a token would be.
|
||||
func redact(source string) string {
|
||||
u, err := url.Parse(source)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return source
|
||||
}
|
||||
u.User, u.RawQuery, u.Fragment = nil, "", ""
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (f *Fetcher) ensure(ctx context.Context, source string) (string, error) {
|
||||
if source == "" {
|
||||
return "", fmt.Errorf("no source given")
|
||||
}
|
||||
// A version rather than a file. Everything below assumes one URL is one
|
||||
// file, and a set has to be handled before any of it.
|
||||
if IsSet(source) {
|
||||
outcome(ctx, "set")
|
||||
return f.ensureSet(ctx, source)
|
||||
}
|
||||
if !isRemote(source) {
|
||||
@@ -95,9 +135,11 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
// was never meant to be one. vLLM and SGLang resolve those
|
||||
// themselves, and doing it for them would be both a duplicate
|
||||
// downloader and a change in documented behaviour.
|
||||
outcome(ctx, "left to the engine")
|
||||
return source, nil
|
||||
}
|
||||
if _, err := os.Stat(source); err == nil {
|
||||
outcome(ctx, "local")
|
||||
return source, nil
|
||||
}
|
||||
return "", fmt.Errorf("%w: %s", ErrNotLocal, source)
|
||||
@@ -108,6 +150,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
outcome(ctx, "cached")
|
||||
return target, nil
|
||||
}
|
||||
|
||||
@@ -117,6 +160,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
}
|
||||
if call, running := f.inflight[source]; running {
|
||||
f.mu.Unlock()
|
||||
outcome(ctx, "fetched by another shard")
|
||||
select {
|
||||
case <-call.done:
|
||||
return call.path, call.err
|
||||
@@ -128,6 +172,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
f.inflight[source] = call
|
||||
f.mu.Unlock()
|
||||
|
||||
outcome(ctx, "downloaded")
|
||||
call.path, call.err = f.fetch(ctx, source, target)
|
||||
close(call.done)
|
||||
|
||||
@@ -147,7 +192,15 @@ func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, err
|
||||
// The two differ for a set: the shards waiting on it asked for the version, not
|
||||
// for the fourth file in it, and progress reported under the file's own URL
|
||||
// reaches nobody. What they see is one number climbing across the whole set.
|
||||
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (string, error) {
|
||||
//
|
||||
// Each file is a "download" span, with how many bytes it took and where a
|
||||
// resumed one started from.
|
||||
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (_ string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "download",
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
defer func() { obs.End(span, err) }()
|
||||
|
||||
url, err := resolveURL(source)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -208,6 +261,10 @@ func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string)
|
||||
total := existing + resp.ContentLength
|
||||
f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total})
|
||||
written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total)
|
||||
span.SetAttributes(
|
||||
attribute.Int64("llmgw.weights.resumed_from", existing),
|
||||
attribute.Int64("llmgw.weights.bytes", written),
|
||||
)
|
||||
closeErr := out.Close()
|
||||
if err != nil {
|
||||
// The part file is kept on purpose, so the next attempt resumes.
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
package weights
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs/obstest"
|
||||
)
|
||||
|
||||
func TestAFetchSaysInItsTraceWhetherItDownloadedAndHowMuch(t *testing.T) {
|
||||
rec := obstest.Record(t)
|
||||
srv := serveBytes(t, []byte("forty gigabytes, give or take"), nil)
|
||||
f := &Fetcher{CacheDir: t.TempDir()}
|
||||
src := strings.Replace(srv.URL, "http://", "http://user:secret@", 1) + "/m/model.gguf?token=hunter2"
|
||||
|
||||
if _, err := f.Ensure(t.Context(), src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fetch := rec.One(t, [16]byte{}, "fetch weights")
|
||||
a := obstest.Attrs(fetch)
|
||||
if a["llmgw.weights.outcome"] != "downloaded" {
|
||||
t.Errorf("the first fetch says %v", a)
|
||||
}
|
||||
if strings.Contains(a["llmgw.weights.source"], "secret") || strings.Contains(a["llmgw.weights.source"], "hunter2") {
|
||||
t.Errorf("the trace holds the source's credentials: %s", a["llmgw.weights.source"])
|
||||
}
|
||||
dl := rec.One(t, fetch.SpanContext().TraceID(), "download")
|
||||
if !obstest.ChildOf(dl, fetch) || obstest.Attrs(dl)["llmgw.weights.bytes"] != "29" {
|
||||
t.Errorf("the download does not say how much it moved: %v", obstest.Attrs(dl))
|
||||
}
|
||||
|
||||
// The second start on this node finds it here.
|
||||
if _, err := f.Ensure(t.Context(), src); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
again := rec.Named("fetch weights")
|
||||
if len(again) != 2 || obstest.Attrs(again[1])["llmgw.weights.outcome"] != "cached" {
|
||||
t.Errorf("the second fetch says %v", obstest.Attrs(again[len(again)-1]))
|
||||
}
|
||||
if len(rec.Named("download")) != 1 {
|
||||
t.Error("a cached file was downloaded again")
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user