A set's download reported as the whole set's, not the part being fetched
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a5fefbdaaa
commit
d5dcf78836
4 files changed
+66
-8
No files matched your search
@@ -202,7 +202,7 @@ func (f *Fetcher) ensure(ctx context.Context, source string, allowPickle bool, w
|
||||
|
||||
// fetch downloads a source to a target, resuming and verifying.
|
||||
func (f *Fetcher) fetch(ctx context.Context, source, target string) (string, error) {
|
||||
return f.fetchAs(ctx, source, source, target)
|
||||
return f.fetchAs(ctx, source, source, target, setShare{})
|
||||
}
|
||||
|
||||
// fetchChecked is fetch for a file whose sha256 is known: a download that
|
||||
@@ -287,7 +287,7 @@ func ChecksumPath(file string) string { return checksumPath(file) }
|
||||
//
|
||||
// Each file is a "download" span, with how many bytes it took and where a
|
||||
// resumed one started from.
|
||||
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string) (_ string, err error) {
|
||||
func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string, share setShare) (_ string, err error) {
|
||||
ctx, span := obs.Tracer(tracerName).Start(ctx, "download",
|
||||
trace.WithSpanKind(trace.SpanKindClient),
|
||||
trace.WithAttributes(attribute.String("llmgw.weights.source", redact(source))))
|
||||
@@ -351,8 +351,8 @@ func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string)
|
||||
}
|
||||
|
||||
total := existing + resp.ContentLength
|
||||
f.report(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total})
|
||||
written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total)
|
||||
f.report(share.of(Progress{Source: reportAs, BytesDone: existing, BytesTotal: total}))
|
||||
written, err := f.copyWithProgress(ctx, out, resp.Body, reportAs, existing, total, share)
|
||||
span.SetAttributes(
|
||||
attribute.Int64("llmgw.weights.resumed_from", existing),
|
||||
attribute.Int64("llmgw.weights.bytes", written),
|
||||
@@ -376,7 +376,7 @@ func (f *Fetcher) fetchAs(ctx context.Context, source, reportAs, target string)
|
||||
}
|
||||
|
||||
func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Reader,
|
||||
source string, done, total int64,
|
||||
source string, done, total int64, share setShare,
|
||||
) (int64, error) {
|
||||
buf := make([]byte, 4<<20)
|
||||
last := time.Now()
|
||||
@@ -394,12 +394,12 @@ func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Re
|
||||
// hundred gigabytes is twenty-five thousand callbacks, and nobody
|
||||
// needs to be told that often.
|
||||
if time.Since(last) > 2*time.Second {
|
||||
f.report(Progress{Source: source, BytesDone: done, BytesTotal: total})
|
||||
f.report(share.of(Progress{Source: source, BytesDone: done, BytesTotal: total}))
|
||||
last = time.Now()
|
||||
}
|
||||
}
|
||||
if errors.Is(err, io.EOF) {
|
||||
f.report(Progress{Source: source, BytesDone: done, BytesTotal: total})
|
||||
f.report(share.of(Progress{Source: source, BytesDone: done, BytesTotal: total}))
|
||||
return done, nil
|
||||
}
|
||||
if err != nil {
|
||||
@@ -408,6 +408,18 @@ func (f *Fetcher) copyWithProgress(ctx context.Context, dst io.Writer, src io.Re
|
||||
}
|
||||
}
|
||||
|
||||
// setShare places one file's download within its set's: the bytes of the
|
||||
// files before it, and the whole set's. Zero for a file on its own.
|
||||
type setShare struct{ before, total int64 }
|
||||
|
||||
// of is a file's progress as the set's.
|
||||
func (s setShare) of(p Progress) Progress {
|
||||
if s.total > 0 {
|
||||
p.BytesDone, p.BytesTotal = s.before+p.BytesDone, s.total
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func (f *Fetcher) report(p Progress) {
|
||||
if f.OnProgress != nil {
|
||||
f.OnProgress(p)
|
||||
|
||||
Reference in new issue
Block a user