A mirror download cut off carries on from where it got to, and is tried again
CI / test (push) Successful in 10m36s
CI / test (push) Successful in 10m36s
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
812eb8aac1
commit
be54f1617b
3 files changed
+229
-30
No files matched your search
@@ -37,6 +37,10 @@ APIs change between minors without a deprecation period.
|
|||||||
against the mirror's checksum, 195 GB on one core for eight minutes, while the replica still read "fetching 100%".
|
against the mirror's checksum, 195 GB on one core for eight minutes, while the replica still read "fetching 100%".
|
||||||
It now checks four at once and says so: `checking 40% of 195 GB` in `ctl replicas`, "Checking the weights" in the
|
It now checks four at once and says so: `checking 40% of 195 GB` in `ctl replicas`, "Checking the weights" in the
|
||||||
chat page; and a check that takes long is not taken for a stuck download.
|
chat page; and a check that takes long is not taken for a stuck download.
|
||||||
|
- **A mirror download the registry cut off started again from nothing, or not at all.** Hugging Face's CDN ended a
|
||||||
|
60 GB transfer at 23 GB (`stream error … CANCEL`), the mirror deleted what had arrived, and `ctl model serve`
|
||||||
|
failed. The mirror now tries again up to six times, carrying on from where it got to (`Range`), keeps what arrived
|
||||||
|
when it gives up, and carries on from it after a restart too.
|
||||||
|
|
||||||
## v0.16.1 — 2026-10-30
|
## v0.16.1 — 2026-10-30
|
||||||
|
|
||||||
|
|||||||
+131
-30
@@ -2,11 +2,15 @@ package mirror
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
|
"net"
|
||||||
"net/http"
|
"net/http"
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -54,40 +58,11 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En
|
|||||||
return Added{}, err
|
return Added{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
|
|
||||||
if err != nil {
|
|
||||||
return Added{}, err
|
|
||||||
}
|
|
||||||
resp, err := client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return Added{}, fmt.Errorf("fetching %s: %w", source, err)
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode != http.StatusOK {
|
|
||||||
return Added{}, fmt.Errorf("%s answered %s", source, resp.Status)
|
|
||||||
}
|
|
||||||
|
|
||||||
part := final + partSuffix
|
part := final + partSuffix
|
||||||
f, err := os.OpenFile(part, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) //nolint:gosec // inside the mirror root
|
written, err := downloadResuming(ctx, client, source, part, progress)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return Added{}, err
|
return Added{}, err
|
||||||
}
|
}
|
||||||
written, err := copyWithProgress(f, resp.Body, resp.ContentLength, progress)
|
|
||||||
closeErr := f.Close()
|
|
||||||
if err != nil {
|
|
||||||
_ = os.Remove(part)
|
|
||||||
return Added{}, err
|
|
||||||
}
|
|
||||||
if closeErr != nil {
|
|
||||||
_ = os.Remove(part)
|
|
||||||
return Added{}, closeErr
|
|
||||||
}
|
|
||||||
// A truncated download that the server ended cleanly would otherwise be
|
|
||||||
// renamed into place and copied to every node.
|
|
||||||
if resp.ContentLength > 0 && written != resp.ContentLength {
|
|
||||||
_ = os.Remove(part)
|
|
||||||
return Added{}, fmt.Errorf("%s: got %d bytes of %d", source, written, resp.ContentLength)
|
|
||||||
}
|
|
||||||
|
|
||||||
sum, err := sha256File(part)
|
sum, err := sha256File(part)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -112,6 +87,132 @@ func addWith(ctx context.Context, client *http.Client, root, source string, e En
|
|||||||
return Added{Entry: e, SHA256: sum}, nil
|
return Added{Entry: e, SHA256: sum}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// downloadAttempts is how many times a download is tried before it fails:
|
||||||
|
// a registry's CDN ends a long transfer now and then ("stream error …
|
||||||
|
// CANCEL"), and a 60 GB file failed at 23 GB and started again from nothing.
|
||||||
|
var downloadAttempts = 6
|
||||||
|
|
||||||
|
// downloadPause is the wait before the next attempt, doubled each time.
|
||||||
|
var downloadPause = 2 * time.Second
|
||||||
|
|
||||||
|
// downloadResuming fetches source into part, carrying on from what part
|
||||||
|
// already holds — a download cut short, or one a restarted mirror began —
|
||||||
|
// and trying again, from where it got to, when the transfer breaks. The part
|
||||||
|
// is kept when it fails, for the next fetch of the same file to carry on.
|
||||||
|
func downloadResuming(ctx context.Context, client *http.Client, source, part string,
|
||||||
|
progress func(done, total int64),
|
||||||
|
) (int64, error) {
|
||||||
|
var have int64
|
||||||
|
if st, err := os.Stat(part); err == nil {
|
||||||
|
have = st.Size()
|
||||||
|
}
|
||||||
|
pause := downloadPause
|
||||||
|
var last error
|
||||||
|
for attempt := 0; attempt < downloadAttempts; attempt++ {
|
||||||
|
if attempt > 0 {
|
||||||
|
select {
|
||||||
|
case <-ctx.Done():
|
||||||
|
return have, ctx.Err()
|
||||||
|
case <-time.After(pause):
|
||||||
|
}
|
||||||
|
pause = min(pause*2, 30*time.Second)
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, source, nil)
|
||||||
|
if err != nil {
|
||||||
|
return have, err
|
||||||
|
}
|
||||||
|
if have > 0 {
|
||||||
|
req.Header.Set("Range", fmt.Sprintf("bytes=%d-", have))
|
||||||
|
}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return have, ctx.Err()
|
||||||
|
}
|
||||||
|
// A refused address, a name that does not resolve: asking
|
||||||
|
// again changes nothing.
|
||||||
|
if !transient(err) {
|
||||||
|
return have, fmt.Errorf("fetching %s: %w", source, err)
|
||||||
|
}
|
||||||
|
last = fmt.Errorf("fetching %s: %w", source, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
flags := os.O_CREATE | os.O_WRONLY
|
||||||
|
switch {
|
||||||
|
case resp.StatusCode == http.StatusPartialContent && have > 0:
|
||||||
|
flags |= os.O_APPEND
|
||||||
|
case resp.StatusCode == http.StatusOK:
|
||||||
|
have = 0 // the server ignored the range: from the start
|
||||||
|
flags |= os.O_TRUNC
|
||||||
|
case resp.StatusCode == http.StatusRequestedRangeNotSatisfiable && have > 0:
|
||||||
|
// Everything was there already.
|
||||||
|
resp.Body.Close()
|
||||||
|
return have, nil
|
||||||
|
case resp.StatusCode >= 500 || resp.StatusCode == http.StatusTooManyRequests:
|
||||||
|
resp.Body.Close()
|
||||||
|
last = fmt.Errorf("%s answered %s", source, resp.Status)
|
||||||
|
continue
|
||||||
|
default:
|
||||||
|
resp.Body.Close()
|
||||||
|
return have, fmt.Errorf("%s answered %s", source, resp.Status)
|
||||||
|
}
|
||||||
|
want := resp.ContentLength
|
||||||
|
f, err := os.OpenFile(part, flags, 0o644) //nolint:gosec // inside the mirror root
|
||||||
|
if err != nil {
|
||||||
|
resp.Body.Close()
|
||||||
|
return have, err
|
||||||
|
}
|
||||||
|
offset := have
|
||||||
|
var report func(done, total int64)
|
||||||
|
if progress != nil {
|
||||||
|
report = func(done, total int64) {
|
||||||
|
if total > 0 {
|
||||||
|
total += offset
|
||||||
|
}
|
||||||
|
progress(offset+done, total)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
n, err := copyWithProgress(f, resp.Body, want, report)
|
||||||
|
resp.Body.Close()
|
||||||
|
closeErr := f.Close()
|
||||||
|
have += n
|
||||||
|
if err == nil && closeErr != nil {
|
||||||
|
return have, closeErr
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
if ctx.Err() != nil {
|
||||||
|
return have, ctx.Err()
|
||||||
|
}
|
||||||
|
last = fmt.Errorf("fetching %s after %d bytes: %w", source, have, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// Ended cleanly but short: the rest is asked for again.
|
||||||
|
if want > 0 && n != want {
|
||||||
|
last = fmt.Errorf("%s: got %d bytes of %d", source, n, want)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
return have, nil
|
||||||
|
}
|
||||||
|
return have, fmt.Errorf("%w (tried %d times; what arrived is kept and the next fetch carries on from it)",
|
||||||
|
last, downloadAttempts)
|
||||||
|
}
|
||||||
|
|
||||||
|
// transient says whether a failure to reach a server may pass if asked
|
||||||
|
// again: a timeout, a connection the far end reset or ended, an HTTP/2
|
||||||
|
// stream it cancelled.
|
||||||
|
func transient(err error) bool {
|
||||||
|
var ne net.Error
|
||||||
|
if errors.As(err, &ne) && ne.Timeout() {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if errors.Is(err, syscall.ECONNRESET) || errors.Is(err, io.ErrUnexpectedEOF) || errors.Is(err, io.EOF) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
msg := err.Error()
|
||||||
|
return strings.Contains(msg, "stream error") || strings.Contains(msg, "connection reset") ||
|
||||||
|
strings.Contains(msg, "GOAWAY")
|
||||||
|
}
|
||||||
|
|
||||||
// copyWithProgress copies and reports occasionally.
|
// copyWithProgress copies and reports occasionally.
|
||||||
//
|
//
|
||||||
// Occasionally rather than continuously: a forty gigabyte download is a
|
// Occasionally rather than continuously: a forty gigabyte download is a
|
||||||
|
|||||||
@@ -0,0 +1,94 @@
|
|||||||
|
package mirror
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"os"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync/atomic"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cutServer serves body, honouring Range, and cuts the first transfer off
|
||||||
|
// halfway, as a registry's CDN does now and then.
|
||||||
|
func cutServer(t *testing.T, body string) (*httptest.Server, *atomic.Int32, *atomic.Value) {
|
||||||
|
t.Helper()
|
||||||
|
var calls atomic.Int32
|
||||||
|
var ranges atomic.Value
|
||||||
|
ranges.Store("")
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
n := calls.Add(1)
|
||||||
|
from := 0
|
||||||
|
if rg := r.Header.Get("Range"); rg != "" {
|
||||||
|
ranges.Store(rg)
|
||||||
|
from, _ = strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(rg, "bytes="), "-"))
|
||||||
|
w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", from, len(body)-1, len(body)))
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(len(body)-from))
|
||||||
|
w.WriteHeader(http.StatusPartialContent)
|
||||||
|
} else {
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(len(body)))
|
||||||
|
}
|
||||||
|
if n == 1 {
|
||||||
|
_, _ = w.Write([]byte(body[from : from+len(body)/2]))
|
||||||
|
w.(http.Flusher).Flush()
|
||||||
|
hj, _ := w.(http.Hijacker)
|
||||||
|
conn, _, _ := hj.Hijack()
|
||||||
|
_ = conn.Close()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
_, _ = w.Write([]byte(body[from:]))
|
||||||
|
}))
|
||||||
|
t.Cleanup(srv.Close)
|
||||||
|
return srv, &calls, &ranges
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADownloadCutOffCarriesOnFromWhereItGot(t *testing.T) {
|
||||||
|
old := downloadPause
|
||||||
|
downloadPause = 10 * time.Millisecond
|
||||||
|
t.Cleanup(func() { downloadPause = old })
|
||||||
|
body := strings.Repeat("weights ", 1000)
|
||||||
|
srv, calls, ranges := cutServer(t, body)
|
||||||
|
root := t.TempDir()
|
||||||
|
e := Entry{Model: "m", Version: "v", File: "w.gguf"}
|
||||||
|
added, err := addWith(t.Context(), http.DefaultClient, root, srv.URL+"/w.gguf", e, nil)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
got, _ := os.ReadFile(e.Path(root))
|
||||||
|
if string(got) != body || added.Entry.Bytes != int64(len(body)) {
|
||||||
|
t.Fatalf("got %d bytes", len(got))
|
||||||
|
}
|
||||||
|
if calls.Load() != 2 || ranges.Load().(string) != fmt.Sprintf("bytes=%d-", len(body)/2) {
|
||||||
|
t.Errorf("calls %d, range %q", calls.Load(), ranges.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestADownloadARestartedMirrorBeganIsCarriedOn(t *testing.T) {
|
||||||
|
body := strings.Repeat("x", 4000)
|
||||||
|
var asked atomic.Value
|
||||||
|
asked.Store("")
|
||||||
|
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
rg := r.Header.Get("Range")
|
||||||
|
asked.Store(rg)
|
||||||
|
from, _ := strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(rg, "bytes="), "-"))
|
||||||
|
w.Header().Set("Content-Length", strconv.Itoa(len(body)-from))
|
||||||
|
w.WriteHeader(http.StatusPartialContent)
|
||||||
|
_, _ = w.Write([]byte(body[from:]))
|
||||||
|
}))
|
||||||
|
defer srv.Close()
|
||||||
|
root := t.TempDir()
|
||||||
|
e := Entry{Model: "m", Version: "v", File: "w.gguf"}
|
||||||
|
_ = os.MkdirAll(root+"/m/v", 0o755)
|
||||||
|
if err := os.WriteFile(e.Path(root)+partSuffix, []byte(body[:1500]), 0o644); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if _, err := addWith(t.Context(), http.DefaultClient, root, srv.URL+"/w.gguf", e, nil); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if got, _ := os.ReadFile(e.Path(root)); string(got) != body || asked.Load().(string) != "bytes=1500-" {
|
||||||
|
t.Errorf("%d bytes, asked %q", len(got), asked.Load())
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in new issue
Block a user