A set wanted by several replicas on one node is fetched once
Single files were shared between the shards and replicas asking for them at the same time; sets were not, and four replicas on one node's four cards wrote the same .part files over one another until all but one failed. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
540b518fde
commit
a88624747e
3 files changed
+58
-3
No files matched your search
@@ -146,7 +146,11 @@ func (f *Fetcher) ensure(ctx context.Context, source string, allowPickle bool, w
|
||||
// file, and a set has to be handled before any of it.
|
||||
if IsSet(source) {
|
||||
outcome(ctx, "set")
|
||||
return f.ensureSet(ctx, source, allowPickle)
|
||||
// Shared like a single file: four replicas placed on one node's
|
||||
// four cards want the same set at the same instant, and each
|
||||
// fetching it wrote the same .part files over one another — three
|
||||
// failed renaming a file another had already renamed.
|
||||
return f.shared(ctx, source, func() (string, error) { return f.ensureSet(ctx, source, allowPickle) })
|
||||
}
|
||||
if !isRemote(source) {
|
||||
if !isPath(source) {
|
||||
@@ -178,6 +182,15 @@ func (f *Fetcher) ensure(ctx context.Context, source string, allowPickle bool, w
|
||||
// Not the file the catalog names: fetched again below, and checked.
|
||||
}
|
||||
|
||||
return f.shared(ctx, source, func() (string, error) {
|
||||
outcome(ctx, "downloaded")
|
||||
return f.fetchChecked(ctx, source, target, want)
|
||||
})
|
||||
}
|
||||
|
||||
// shared runs fetch once for every caller asking for source at the same
|
||||
// time; the others wait for its answer.
|
||||
func (f *Fetcher) shared(ctx context.Context, source string, fetch func() (string, error)) (string, error) {
|
||||
f.mu.Lock()
|
||||
if f.inflight == nil {
|
||||
f.inflight = map[string]*fetchCall{}
|
||||
@@ -196,8 +209,7 @@ func (f *Fetcher) ensure(ctx context.Context, source string, allowPickle bool, w
|
||||
f.inflight[source] = call
|
||||
f.mu.Unlock()
|
||||
|
||||
outcome(ctx, "downloaded")
|
||||
call.path, call.err = f.fetchChecked(ctx, source, target, want)
|
||||
call.path, call.err = fetch()
|
||||
close(call.done)
|
||||
|
||||
f.mu.Lock()
|
||||
|
||||
Reference in new issue
Block a user