<commit_before># -*- coding: utf-8 -*-
"""functions for downloading from the web
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import namedtuple
import os
import tarfile
import zipfile
import requests
from requests.exceptions import HTTPError
from tqdm import tqdm

from mirdata.utils import md5


# destination dir should be a relative path to save the file/s, or None
RemoteFileMetadata = namedtuple(
    'RemoteFileMetadata', ['filename', 'url', 'checksum', 'destination_dir']
)


def downloader(
    save_dir,
    zip_downloads=None,
    tar_downloads=None,
    file_downloads=None,
    info_message=None,
    force_overwrite=False,
    cleanup=False,
):
    """Download data to `save_dir` and optionally print a message.

    Args:
        save_dir (str):
            The directory to download the data
        zip_downloads (list or None):
            A list of RemoteFileMetadata tuples of data in zip format.
            If None, there is no zip data to download
        tar_downloads (list or None):
            A list of RemoteFileMetadata tuples of data in tar format.
            If None, there is no tar data to download
        file_downloads (list or None):
            A list of  RemoteFileMetadata tuples of uncompressed data.
            If None, there is no uncompressed data to download.
        info_message (str or None):
            A string of info to print when this function is called.
            If None, no string is printed.
        force_overwrite (bool):
            If True, existing files are overwritten by the downloaded files.
        cleanup (bool):
            Whether to delete the zip/tar file after extracting.

    """
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    if zip_downloads is not None:
        for zip_download in zip_downloads:
            download_zip_file(zip_download, save_dir, force_overwrite, cleanup)

    if tar_downloads is not None:
        for tar_download in tar_downloads:
            download_tar_file(tar_download, save_dir, force_overwrite, cleanup)

    if file_downloads is not None:
        for file_download in file_downloads:
            download_from_remote(file_download, save_dir, force_overwrite)

    if info_message is not None:
        print(info_message)


class DownloadProgressBar(tqdm):
    """Wrap `tqdm` to show download progress"""

    def update_to(self, b=1, bsize=1, tsize=None):
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)


def _download_large_file(url, download_path, callback=lambda: None):
    """download large file stably (todo: ??)"""
    response = requests.get(url)
    response.raise_for_status()
    with open(download_path, 'wb') as handle:
        for block in response.iter_content(4096):
            handle.write(block)
            callback()
    return download_path


def download_from_remote(remote, save_dir, force_overwrite=False):
    """Download a remote dataset into path
    Fetch a dataset pointed by remote's url, save into path using remote's
    filename and ensure its integrity based on the MD5 Checksum of the
    downloaded file.

    Adapted from scikit-learn's sklearn.datasets.base._fetch_remote.

    Parameters
    -----------
    remote: RemoteFileMetadata
        Named tuple containing remote dataset meta information: url, filename
        and checksum
    save_dir: string
        Directory to save the file to. Usually `data_home`
    force_overwrite: bool
        If True, overwrite existing file with the downloaded file.
        If False, does not overwrite, but checks that checksum is consistent.

    Returns
    -------
    file_path: string
        Full path of the created file.
    """
    if remote.destination_dir is None:
        download_dir = save_dir
    else:
        download_dir = os.path.join(save_dir, remote.destination_dir)

    if not os.path.exists(download_dir):
        os.makedirs(download_dir)

    download_path = os.path.join(download_dir, remote.filename)
    if not os.path.exists(download_path) or force_overwrite:
        # If file doesn't exist or we want to overwrite, download it
        with DownloadProgressBar(
            unit='B', unit_scale=True, miniters=1, desc=remote.url.split('/')[-1]
        ) as t:
            try:
                _download_large_file(remote.url, download_path, t.update_to)
            except HTTPError:
                error_msg = """
                            mirdata failed to download the dataset!
                            Please try again in a few minutes.
                            If this error persists, please raise an issue at
                            https://github.com/mir-dataset-loaders/mirdata,
                            and tag it with 'broken-link'.
                            """
                raise HTTPError(error_msg)

    checksum = md5(download_path)
    if remote.checksum != checksum:
        raise IOError(
            '{} has an MD5 checksum ({}) '
            'differing from expected ({}), '
            'file may be corrupted.'.format(download_path, checksum, remote.checksum)
        )
    return download_path


def download_zip_file(zip_remote, save_dir, force_overwrite, cleanup=False):
    """Download and unzip a zip file.

    Parameters
    ----------
    zip_remote: RemoteFileMetadata
        Object containing download information
    save_dir: str
        Path to save downloaded file
    force_overwrite: bool
        If True, overwrites existing files
    cleanup: bool, default=False
        If True, remove zipfile after unziping.
    """
    zip_download_path = download_from_remote(zip_remote, save_dir, force_overwrite)
    unzip(zip_download_path, cleanup=cleanup)<commit_msg>Move back from requests to urllib (#134)<commit_after># -*- coding: utf-8 -*-
"""functions for downloading from the web
"""
from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

from collections import namedtuple
import os
import tarfile
import zipfile
from tqdm import tqdm
from six.moves import urllib
import six
import sys

from mirdata.utils import md5


# destination dir should be a relative path to save the file/s, or None
RemoteFileMetadata = namedtuple(
    'RemoteFileMetadata', ['filename', 'url', 'checksum', 'destination_dir']
)


def downloader(
    save_dir,
    zip_downloads=None,
    tar_downloads=None,
    file_downloads=None,
    info_message=None,
    force_overwrite=False,
    cleanup=False,
):
    """Download data to `save_dir` and optionally print a message.

    Args:
        save_dir (str):
            The directory to download the data
        zip_downloads (list or None):
            A list of RemoteFileMetadata tuples of data in zip format.
            If None, there is no zip data to download
        tar_downloads (list or None):
            A list of RemoteFileMetadata tuples of data in tar format.
            If None, there is no tar data to download
        file_downloads (list or None):
            A list of  RemoteFileMetadata tuples of uncompressed data.
            If None, there is no uncompressed data to download.
        info_message (str or None):
            A string of info to print when this function is called.
            If None, no string is printed.
        force_overwrite (bool):
            If True, existing files are overwritten by the downloaded files.
        cleanup (bool):
            Whether to delete the zip/tar file after extracting.

    """
    if not os.path.exists(save_dir):
        os.makedirs(save_dir)

    if zip_downloads is not None:
        for zip_download in zip_downloads:
            download_zip_file(zip_download, save_dir, force_overwrite, cleanup)

    if tar_downloads is not None:
        for tar_download in tar_downloads:
            download_tar_file(tar_download, save_dir, force_overwrite, cleanup)

    if file_downloads is not None:
        for file_download in file_downloads:
            download_from_remote(file_download, save_dir, force_overwrite)

    if info_message is not None:
        print(info_message)


class DownloadProgressBar(tqdm):
    """Wrap `tqdm` to show download progress"""

    def update_to(self, b=1, bsize=1, tsize=None):
        if tsize is not None:
            self.total = tsize
        self.update(b * bsize - self.n)


def download_from_remote(remote, save_dir, force_overwrite=False):
    """Download a remote dataset into path
    Fetch a dataset pointed by remote's url, save into path using remote's
    filename and ensure its integrity based on the MD5 Checksum of the
    downloaded file.

    Adapted from scikit-learn's sklearn.datasets.base._fetch_remote.

    Parameters
    -----------
    remote: RemoteFileMetadata
        Named tuple containing remote dataset meta information: url, filename
        and checksum
    save_dir: string
        Directory to save the file to. Usually `data_home`
    force_overwrite: bool
        If True, overwrite existing file with the downloaded file.
        If False, does not overwrite, but checks that checksum is consistent.

    Returns
    -------
    file_path: string
        Full path of the created file.
    """
    if remote.destination_dir is None:
        download_dir = save_dir
    else:
        download_dir = os.path.join(save_dir, remote.destination_dir)

    if not os.path.exists(download_dir):
        os.makedirs(download_dir)

    download_path = os.path.join(download_dir, remote.filename)
    if not os.path.exists(download_path) or force_overwrite:
        # If file doesn't exist or we want to overwrite, download it
        with DownloadProgressBar(
            unit='B', unit_scale=True, unit_divisor=1024, miniters=1
        ) as t:
            try:
                urllib.request.urlretrieve(
                    remote.url,
                    filename=download_path,
                    reporthook=t.update_to,
                    data=None,
                )
            except Exception as e:
                error_msg = """
                            mirdata failed to download the dataset!
                            Please try again in a few minutes.
                            If this error persists, please raise an issue at
                            https://github.com/mir-dataset-loaders/mirdata,
                            and tag it with 'broken-link'.
                            """
                print(error_msg)
                raise e

    checksum = md5(download_path)
    if remote.checksum != checksum:
        raise IOError(
            '{} has an MD5 checksum ({}) '
            'differing from expected ({}), '
            'file may be corrupted.'.format(download_path, checksum, remote.checksum)
        )
    return download_path


def download_zip_file(zip_remote, save_dir, force_overwrite, cleanup=False):
    """Download and unzip a zip file.

    Parameters
    ----------
    zip_remote: RemoteFileMetadata
        Object containing download information
    save_dir: str
        Path to save downloaded file
    force_overwrite: bool
        If True, overwrites existing files
    cleanup: bool, default=False
        If True, remove zipfile after unziping.
    """
    zip_download_path = download_from_remote(zip_remote, save_dir, force_overwrite)
    unzip(zip_download_path, cleanup=cleanup)
{'fraction_non_alphanumeric': 0.07032890132960112, 'fraction_numerical': 0.00244926522043387, 'mean_word_length': 2.361658335783593, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 120, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3483445', 'n_tokens_mistral': 3152, 'n_tokens_neox': 3059, 'n_words': 1068}