A set's download reported as the whole set's, not the part being fetched
CI / test (push) Successful in 9m15s

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Ilker ManapandClaude Opus 5.5 committed 2026-10-30 14:48:45 +01:00
1 parent a5fefbdaaa
commit d5dcf78836
4 files changed
+66 -8

No files matched your search

+19 -7
View File
@@ -202,7 +202,7 @@ func (f *Fetcher) ensure(ctx context.Context, source string, allowPickle bool, w
// fetch downloads a source to a target, resuming and verifying.
func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, error) {
return f.fetchAs(ctx, source, source, target)
return f.fetchAs(ctx, source, source, target, setShare{})
}
// fetchChecked is fetch for a file whose sha256 is known: a download that
@@ -287,7 +287,7 @@ func ChecksumPath(file string) string { return checksumPath(file) }
//
// Each file is a "download" span, with how many bytes it took and where a
// resumed one started from.
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (_ string, err error) {
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string, share setShare) (_ string, err error) {
ctx, span := obs.Tracer(tracerName).Start(ctx, "download",
trace.WithSpanKind(trace.SpanKindClient),
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
@@ -351,8 +351,8 @@ func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string)
}
total := existing + resp.ContentLength
f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total})
written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total)
f.report(share.of(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total}))
written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total, share)
span.SetAttributes(
attribute.Int64("llmgw.weights.resumed_from", existing),
attribute.Int64("llmgw.weights.bytes", written),
@@ -376,7 +376,7 @@ func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string)
}
func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Reader,
source string, done, total int64,
source string, done, total int64, share setShare,
) (int64, error) {
buf := make([]byte, 4<<20)
last := time.Now()
@@ -394,12 +394,12 @@ func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Re
// hundred gigabytes is twenty-five thousand callbacks, and nobody
// needs to be told that often.
if time.Since(last) > 2*time.Second {
f.report(Progress{Source: source, BytesDone: done, BytesTotal: total})
f.report(share.of(Progress{Source: source, BytesDone: done, BytesTotal: total}))
last = time.Now()
}
}
if errors.Is(err, io.EOF) {
f.report(Progress{Source: source, BytesDone: done, BytesTotal: total})
f.report(share.of(Progress{Source: source, BytesDone: done, BytesTotal: total}))
return done, nil
}
if err != nil {
@@ -408,6 +408,18 @@ func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Re
}
}
// setShare places one file's download within its set's: the bytes of the
// files before it, and the whole set's. Zero for a file on its own.
type setShare struct{ before, total int64 }
// of is a file's progress as the set's.
func (s setShare) of(p Progress) Progress {
if s.total > 0 {
p.BytesDone, p.BytesTotal = s.before+p.BytesDone, s.total
}
return p
}
func (f *Fetcher) report(p Progress) {
if f.OnProgress != nil {
f.OnProgress(p)