package mirror import ( "errors" "fmt" "net" "net/http" "net/netip" "net/url" "syscall" "time" ) // errPrivateSource is a fetch refused because it would reach this machine or // the network behind it rather than the internet. var errPrivateSource = errors.New("refusing to fetch from a loopback, private or link-local address") // fetchClient downloads sources for the management API. // // The API fetches whatever URL it is given and the file server then hands the // result to anyone. Without this, "fetch http://169.254.169.254/..." was a way // to read a cloud's instance metadata — or anything else on the network the // mirror sits in — through the mirror. The check is made on the address // actually dialled, after resolution and on every redirect, so a name that // resolves somewhere public and then somewhere private is caught too. func fetchClient(allowPrivate bool) *http.Client { d := &net.Dialer{Timeout: 30 * time.Second, KeepAlive: 30 * time.Second} if !allowPrivate { d.Control = func(_, address string, _ syscall.RawConn) error { host, _, err := net.SplitHostPort(address) if err != nil { return err } a, err := netip.ParseAddr(host) if err != nil { return err } if blockedAddr(a.Unmap()) { return fmt.Errorf("%w (%s)", errPrivateSource, a) } return nil } } return &http.Client{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: d.DialContext, TLSHandshakeTimeout: 30 * time.Second, ResponseHeaderTimeout: 2 * time.Minute, ForceAttemptHTTP2: true, }, // No overall timeout: a model is tens of gigabytes. The dial, the // handshake and the first byte are bounded above; after that a stall // is the context's to end. CheckRedirect: func(req *http.Request, via []*http.Request) error { if len(via) >= 10 { return errors.New("too many redirects") } return checkScheme(req.URL) }, } } func blockedAddr(a netip.Addr) bool { cgnat := netip.MustParsePrefix("100.64.0.0/10") return a.IsLoopback() || a.IsPrivate() || a.IsLinkLocalUnicast() || a.IsLinkLocalMulticast() || a.IsMulticast() || a.IsUnspecified() || cgnat.Contains(a) } // checkScheme allows the two schemes a model is published over. file:// and // the rest would read this machine's own disk. func checkScheme(u *url.URL) error { if u.Scheme != "http" && u.Scheme != "https" { return fmt.Errorf("%q is not an http or https source", u.Redacted()) } return nil } // validSource checks a source before a download is started for it. func validSource(source string) error { u, err := url.Parse(source) if err != nil || u.Host == "" { return fmt.Errorf("%q is not a URL", source) } return checkScheme(u) }