Tracing beyond the router: reconcile passes, placement and a replica's start traced down to the node
A reconcile pass that does something is a trace, with placement (where it went, or every candidate turned down), scale-up/down, drain, stop, eviction, preemption, reaping and restarts beneath it. A replica's start is a span open from placement until it serves; the start command carries its context in a new ControlMessage.trace field, and the agent continues the trace through the weights fetch, engine launch, rendezvous and readiness wait. Stops, drains, adoption and engine installs are traced on the node, model jobs are a trace with a span per step linked to the replica's start, and agent.yaml takes the same tracing block as server.yaml. Off, every span is the no-op one and commands carry nothing. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
e59ff52932
commit
e725adc2f0
33 files changed
+1607
-143
No files matched your search
@@ -26,6 +26,11 @@ import (
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.opentelemetry.io/otel/attribute"
|
||||
"go.opentelemetry.io/otel/trace"
|
||||
|
||||
"gitea.mynodes.xyz/ilker/llmgw/internal/obs"
|
||||
)
|
||||
|
||||
// Progress reports how a fetch is going, for anything that wants to say so.
|
||||
@@ -80,13 +85,48 @@ var ErrNotLocal = errors.New("the weights are not on this node and the source sa
|
||||
// local disk, a Lustre mount or an image exported over the fabric, and a
|
||||
// downloader that second-guessed that would copy a hundred gigabytes that were
|
||||
// already present.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
//
|
||||
// Traced as "fetch weights", with what it came to — the file was local,
|
||||
// already cached, downloaded, or downloaded by another shard this one waited
|
||||
// for — since that is most of the difference between a replica that starts
|
||||
// in a minute and one that takes an hour.
|
||||
func (f *Fetcher) Ensure(ctx context.Context, source string) (path string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "fetch weights",
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
defer func() {
|
||||
span.SetAttributes(attribute.String("llmgw.weights.path", path))
|
||||
obs.End(span, err)
|
||||
}()
|
||||
return f.ensure(ctx, source)
|
||||
}
|
||||
|
||||
// tracerName names the spans this package opens.
|
||||
const tracerName = "gitea.mynodes.xyz/ilker/llmgw/internal/agent/weights"
|
||||
|
||||
// outcome says on the current span how a source was made present.
|
||||
func outcome(ctx context.Context, how string) {
|
||||
trace.SpanFromContext(ctx).SetAttributes(attribute.String("llmgw.weights.outcome", how))
|
||||
}
|
||||
|
||||
// redact is a source fit for a trace: a URL without its credentials or
|
||||
// query, which is where a token would be.
|
||||
func redact(source string) string {
|
||||
u, err := url.Parse(source)
|
||||
if err != nil || u.Scheme == "" || u.Host == "" {
|
||||
return source
|
||||
}
|
||||
u.User, u.RawQuery, u.Fragment = nil, "", ""
|
||||
return u.String()
|
||||
}
|
||||
|
||||
func (f *Fetcher) ensure(ctx context.Context, source string) (string, error) {
|
||||
if source == "" {
|
||||
return "", fmt.Errorf("no source given")
|
||||
}
|
||||
// A version rather than a file. Everything below assumes one URL is one
|
||||
// file, and a set has to be handled before any of it.
|
||||
if IsSet(source) {
|
||||
outcome(ctx, "set")
|
||||
return f.ensureSet(ctx, source)
|
||||
}
|
||||
if !isRemote(source) {
|
||||
@@ -95,9 +135,11 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
// was never meant to be one. vLLM and SGLang resolve those
|
||||
// themselves, and doing it for them would be both a duplicate
|
||||
// downloader and a change in documented behaviour.
|
||||
outcome(ctx, "left to the engine")
|
||||
return source, nil
|
||||
}
|
||||
if _, err := os.Stat(source); err == nil {
|
||||
outcome(ctx, "local")
|
||||
return source, nil
|
||||
}
|
||||
return "", fmt.Errorf("%w: %s", ErrNotLocal, source)
|
||||
@@ -108,6 +150,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
return "", err
|
||||
}
|
||||
if _, err := os.Stat(target); err == nil {
|
||||
outcome(ctx, "cached")
|
||||
return target, nil
|
||||
}
|
||||
|
||||
@@ -117,6 +160,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
}
|
||||
if call, running := f.inflight[source]; running {
|
||||
f.mu.Unlock()
|
||||
outcome(ctx, "fetched by another shard")
|
||||
select {
|
||||
case <-call.done:
|
||||
return call.path, call.err
|
||||
@@ -128,6 +172,7 @@ func (f *Fetcher) Ensure(ctx context.Context, source string) (string, error) {
|
||||
f.inflight[source] = call
|
||||
f.mu.Unlock()
|
||||
|
||||
outcome(ctx, "downloaded")
|
||||
call.path, call.err = f.fetch(ctx, source, target)
|
||||
close(call.done)
|
||||
|
||||
@@ -147,7 +192,15 @@ func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, err
|
||||
// The two differ for a set: the shards waiting on it asked for the version, not
|
||||
// for the fourth file in it, and progress reported under the file's own URL
|
||||
// reaches nobody. What they see is one number climbing across the whole set.
|
||||
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (string, error) {
|
||||
//
|
||||
// Each file is a "download" span, with how many bytes it took and where a
|
||||
// resumed one started from.
|
||||
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (_ string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "download",
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
defer func() { obs.End(span, err) }()
|
||||
|
||||
url, err := resolveURL(source)
|
||||
if err != nil {
|
||||
return "", err
|
||||
@@ -208,6 +261,10 @@ func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string)
|
||||
total := existing + resp.ContentLength
|
||||
f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total})
|
||||
written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total)
|
||||
span.SetAttributes(
|
||||
attribute.Int64("llmgw.weights.resumed_from", existing),
|
||||
attribute.Int64("llmgw.weights.bytes", written),
|
||||
)
|
||||
closeErr := out.Close()
|
||||
if err != nil {
|
||||
// The part file is kept on purpose, so the next attempt resumes.
|
||||
|
||||
Reference in new issue
Block a user