<commit_before>#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import six

import os
import sys
import shutil
import requests
import tqdm
import hashlib
import tarfile
import zipfile

from .cli import ColorTTY

import logging
logger = logging.getLogger(__name__)

__all__ = ['get_weights_path', 'get_dataset_path']

WEIGHTS_HOME = os.path.expanduser("~/.cache/paddle/weights")
DATASET_HOME = os.path.expanduser("~/.cache/paddle/dataset")

DATASETS = {
    'coco': [
        (
            'http://images.cocodataset.org/zips/train2017.zip',
            'cced6f7f71b7629ddf16f17bbcfab6b2', ),
        (
            'http://images.cocodataset.org/zips/val2017.zip',
            '442b8da7639aecaf257c1dceb8ba8c80', ),
        (
            'http://images.cocodataset.org/annotations/annotations_trainval2017.zip',
            'f4bbac642086de4f52a3fdda2de5fa2c', ),
    ],
    'pascal': [(
        'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar',
        '6cd6e144f989b92b3379bac3b3de84fd', )],
}

DOWNLOAD_RETRY_LIMIT = 3


def get_weights_path(url):
    """Get weights path from WEIGHT_HOME, if not exists,
    download it from url.
    """
    return get_path(url, WEIGHTS_HOME)


def get_dataset_path(path):
    """
    If path exists, return path.
    Otherwise, get dataset path from DATASET_HOME, if not exists,
    download it.
    """
    if os.path.exists(path):
        logger.debug("Data path: {}".format(os.path.realpath(path)))
        return path

    logger.info("DATASET_DIR {} not exitst, try searching {} or "
                "downloading dataset...".format(
                    os.path.realpath(path), DATASET_HOME))
    for name, dataset in DATASETS.items():
        if path.lower().find(name) >= 0:
            logger.info("Parse DATASET_DIR {} as dataset "
                        "{}".format(path, name))
            data_dir = os.path.join(DATASET_HOME, name)
            for url, md5sum in dataset:
                get_path(url, data_dir, md5sum)
            return data_dir

    # not match any dataset in DATASETS
    raise ValueError("{} not exists and unknow dataset type".format(path))


def get_path(url, root_dir, md5sum=None):
    """ Download from given url to root_dir.
    if file or directory specified by url is exists under
    root_dir, return the path directly, otherwise download
    from url and decompress it, return the path.

    url (str): download url
    root_dir (str): root dir for downloading, it should be
                    WEIGHTS_HOME or DATASET_HOME
    md5sum (str): md5 sum of download package
    """
    # parse path after download to decompress under root_dir
    fname = url.split('/')[-1]
    zip_formats = ['.zip', '.tar', '.gz']
    fpath = fname
    for zip_format in zip_formats:
        fpath = fpath.replace(zip_format, '')
    fullpath = os.path.join(root_dir, fpath)

    # For same zip file, decompressed directory name different
    # from zip file name, rename by following map
    decompress_name_map = {
        "VOCtrainval": "VOCdevkit",
        "annotations_trainval": "annotations"
    }
    for k, v in decompress_name_map.items():
        if fullpath.find(k) >= 0:
            fullpath = '/'.join(fullpath.split('/')[:-1] + [v])

    if os.path.exists(fullpath):
        logger.info("Found {}".format(fullpath))
    else:
        fullname = _download(url, root_dir, md5sum)
        _decompress(fullname)

    return fullpath


def _download(url, path, md5sum=None):
    """
    Download from url, save to path.

    url (str): download url
    path (str): download to given path
    """
    if not os.path.exists(path):
        os.makedirs(path)

    fname = url.split('/')[-1]
    fullname = os.path.join(path, fname)
    retry_cnt = 0

    print("will download file to {}, proceed? {}".format(
        fullname, ColorTTY().green("[Y/n]")))
    yes = {'yes', 'y', ''}
    if six.PY2:
        choice = raw_input().lower()
    else:
        choice = input().lower()
    if choice not in yes:
        print('download aborted....')
        sys.exit(0)

    while not (os.path.exists(fullname) and _md5check(fullname, md5sum)):
        if retry_cnt < DOWNLOAD_RETRY_LIMIT:
            retry_cnt += 1
        else:
            raise RuntimeError("Download from {} failed. "
                               "Retry limit reached".format(url))<commit_msg>Revert user prompt before downloading files (#2521)<commit_after>#   Copyright (c) 2019 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function


import os
import shutil
import requests
import tqdm
import hashlib
import tarfile
import zipfile

import logging
logger = logging.getLogger(__name__)

__all__ = ['get_weights_path', 'get_dataset_path']

WEIGHTS_HOME = os.path.expanduser("~/.cache/paddle/weights")
DATASET_HOME = os.path.expanduser("~/.cache/paddle/dataset")

DATASETS = {
    'coco': [
        (
            'http://images.cocodataset.org/zips/train2017.zip',
            'cced6f7f71b7629ddf16f17bbcfab6b2', ),
        (
            'http://images.cocodataset.org/zips/val2017.zip',
            '442b8da7639aecaf257c1dceb8ba8c80', ),
        (
            'http://images.cocodataset.org/annotations/annotations_trainval2017.zip',
            'f4bbac642086de4f52a3fdda2de5fa2c', ),
    ],
    'pascal': [(
        'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/VOCtrainval_11-May-2012.tar',
        '6cd6e144f989b92b3379bac3b3de84fd', )],
}

DOWNLOAD_RETRY_LIMIT = 3


def get_weights_path(url):
    """Get weights path from WEIGHT_HOME, if not exists,
    download it from url.
    """
    return get_path(url, WEIGHTS_HOME)


def get_dataset_path(path):
    """
    If path exists, return path.
    Otherwise, get dataset path from DATASET_HOME, if not exists,
    download it.
    """
    if os.path.exists(path):
        logger.debug("Data path: {}".format(os.path.realpath(path)))
        return path

    logger.info("DATASET_DIR {} not exitst, try searching {} or "
                "downloading dataset...".format(
                    os.path.realpath(path), DATASET_HOME))
    for name, dataset in DATASETS.items():
        if path.lower().find(name) >= 0:
            logger.info("Parse DATASET_DIR {} as dataset "
                        "{}".format(path, name))
            data_dir = os.path.join(DATASET_HOME, name)
            for url, md5sum in dataset:
                get_path(url, data_dir, md5sum)
            return data_dir

    # not match any dataset in DATASETS
    raise ValueError("{} not exists and unknow dataset type".format(path))


def get_path(url, root_dir, md5sum=None):
    """ Download from given url to root_dir.
    if file or directory specified by url is exists under
    root_dir, return the path directly, otherwise download
    from url and decompress it, return the path.

    url (str): download url
    root_dir (str): root dir for downloading, it should be
                    WEIGHTS_HOME or DATASET_HOME
    md5sum (str): md5 sum of download package
    """
    # parse path after download to decompress under root_dir
    fname = url.split('/')[-1]
    zip_formats = ['.zip', '.tar', '.gz']
    fpath = fname
    for zip_format in zip_formats:
        fpath = fpath.replace(zip_format, '')
    fullpath = os.path.join(root_dir, fpath)

    # For same zip file, decompressed directory name different
    # from zip file name, rename by following map
    decompress_name_map = {
        "VOCtrainval": "VOCdevkit",
        "annotations_trainval": "annotations"
    }
    for k, v in decompress_name_map.items():
        if fullpath.find(k) >= 0:
            fullpath = '/'.join(fullpath.split('/')[:-1] + [v])

    if os.path.exists(fullpath):
        logger.info("Found {}".format(fullpath))
    else:
        fullname = _download(url, root_dir, md5sum)
        _decompress(fullname)

    return fullpath


def _download(url, path, md5sum=None):
    """
    Download from url, save to path.

    url (str): download url
    path (str): download to given path
    """
    if not os.path.exists(path):
        os.makedirs(path)

    fname = url.split('/')[-1]
    fullname = os.path.join(path, fname)
    retry_cnt = 0

    while not (os.path.exists(fullname) and _md5check(fullname, md5sum)):
        if retry_cnt < DOWNLOAD_RETRY_LIMIT:
            retry_cnt += 1
        else:
            raise RuntimeError("Download from {} failed. "
                               "Retry limit reached".format(url))
{'fraction_non_alphanumeric': 0.1103009140392318, 'fraction_numerical': 0.023005032350826744, 'mean_word_length': 2.7224770642201834, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 33, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2267559', 'n_tokens_mistral': 3171, 'n_tokens_neox': 3037, 'n_words': 853}