<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Utilities for data grabbers (from nilearn)
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import os.path as op
from pathlib import Path
import sys
import shutil
import time
import base64
import hashlib
import subprocess as sp
from io import open
from builtins import str

try:
    from urllib.parse import urlparse
    from urllib.request import urlopen, Request
    from urllib.error import HTTPError, URLError
except ImportError:
    from urlparse import urlparse
    from urllib2 import urlopen, Request, HTTPError, URLError

from .. import NIWORKFLOWS_LOG

PY3 = sys.version_info[0] > 2
MAX_RETRIES = 20
NIWORKFLOWS_CACHE_DIR = (Path.home() / '.cache' / 'stanford-crn')


def fetch_file(dataset_name, url, dataset_dir, dataset_prefix=None,
               default_paths=None, filetype=None, resume=True, overwrite=False,
               md5sum=None, username=None, password=None, retry=0,
               verbose=1, temp_downloads=None):
    """Load requested file, downloading it if needed or requested.

    :param str url: contains the url of the file to be downloaded.
    :param str dataset_dir: path of the data directory. Used for data
        storage in the specified location.
    :param bool resume: if true, try to resume partially downloaded files
    :param overwrite: if bool true and file already exists, delete it.
    :param str md5sum: MD5 sum of the file. Checked if download of the file
        is required
    :param str username: username used for basic HTTP authentication
    :param str password: password used for basic HTTP authentication
    :param int verbose: verbosity level (0 means no message).
    :returns: absolute path of downloaded file
    :rtype: str

    ..note::

      If, for any reason, the download procedure fails, all downloaded files are
      removed.


    """
    final_path, cached = _get_dataset(dataset_name, dataset_prefix=dataset_prefix,
                                      data_dir=dataset_dir, default_paths=default_paths,
                                      verbose=verbose)
    if cached and not overwrite:
        return final_path

    data_dir = final_path.parent

    if temp_downloads is None:
        temp_downloads = NIWORKFLOWS_CACHE_DIR / 'downloads'
    temp_downloads = Path(temp_downloads)

    temp_downloads.mkdir(parents=True, exist_ok=True)

    # Determine filename using URL
    parse = urlparse(url)
    file_name = op.basename(parse.path)
    if file_name == '':
        file_name = _md5_hash(parse.path)

        if filetype is not None:
            file_name += filetype

    temp_full_path = temp_downloads / file_name
    temp_part_path = temp_full_path.with_name(file_name + '.part')

    if overwrite:
        shutil.rmtree(dataset_dir, ignore_errors=True)

    if overwrite and temp_full_path.exists():
        temp_full_path.unlink()

    t_0 = time.time()
    local_file = None
    initial_size = 0

    # Download data
    request = Request(url)
    request.add_header('Connection', 'Keep-Alive')
    if username is not None and password is not None:<commit_msg>ENH: Make fetch_file resilient to Path dataset_dirs<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-
# emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
# vi: set ft=python sts=4 ts=4 sw=4 et:
"""
Utilities for data grabbers (from nilearn)
"""
from __future__ import absolute_import, division, print_function, unicode_literals

import os
import os.path as op
from pathlib import Path
import sys
import shutil
import time
import base64
import hashlib
import subprocess as sp
from io import open
from builtins import str

try:
    from urllib.parse import urlparse
    from urllib.request import urlopen, Request
    from urllib.error import HTTPError, URLError
except ImportError:
    from urlparse import urlparse
    from urllib2 import urlopen, Request, HTTPError, URLError

from .. import NIWORKFLOWS_LOG

PY3 = sys.version_info[0] > 2
MAX_RETRIES = 20
NIWORKFLOWS_CACHE_DIR = (Path.home() / '.cache' / 'stanford-crn')


def fetch_file(dataset_name, url, dataset_dir, dataset_prefix=None,
               default_paths=None, filetype=None, resume=True, overwrite=False,
               md5sum=None, username=None, password=<PASSWORD>, retry=0,
               verbose=1, temp_downloads=None):
    """Load requested file, downloading it if needed or requested.

    :param str url: contains the url of the file to be downloaded.
    :param str dataset_dir: path of the data directory. Used for data
        storage in the specified location.
    :param bool resume: if true, try to resume partially downloaded files
    :param overwrite: if bool true and file already exists, delete it.
    :param str md5sum: MD5 sum of the file. Checked if download of the file
        is required
    :param str username: username used for basic HTTP authentication
    :param str password: password used for basic HTTP authentication
    :param int verbose: verbosity level (0 means no message).
    :returns: absolute path of downloaded file
    :rtype: str

    ..note::

      If, for any reason, the download procedure fails, all downloaded files are
      removed.


    """
    final_path, cached = _get_dataset(dataset_name, dataset_prefix=dataset_prefix,
                                      data_dir=dataset_dir, default_paths=default_paths,
                                      verbose=verbose)
    if cached and not overwrite:
        return final_path

    data_dir = final_path.parent

    if temp_downloads is None:
        temp_downloads = NIWORKFLOWS_CACHE_DIR / 'downloads'
    temp_downloads = Path(temp_downloads)

    temp_downloads.mkdir(parents=True, exist_ok=True)

    # Determine filename using URL
    parse = urlparse(url)
    file_name = op.basename(parse.path)
    if file_name == '':
        file_name = _md5_hash(parse.path)

        if filetype is not None:
            file_name += filetype

    temp_full_path = temp_downloads / file_name
    temp_part_path = temp_full_path.with_name(file_name + '.part')

    if overwrite:
        shutil.rmtree(str(dataset_dir), ignore_errors=True)

    if overwrite and temp_full_path.exists():
        temp_full_path.unlink()

    t_0 = time.time()
    local_file = None
    initial_size = 0

    # Download data
    request = Request(url)
    request.add_header('Connection', 'Keep-Alive')
    if username is not None and password is not None:
{'fraction_non_alphanumeric': 0.07655141174694248, 'fraction_numerical': 0.0066435150234032915, 'mean_word_length': 3.1168427594779367, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 36, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2100875', 'n_tokens_mistral': 2000, 'n_tokens_neox': 1915, 'n_words': 655}