<commit_before>import hashlib
import os
import shutil
import urllib2

__all__ = ['DATA_HOME', 'download']

DATA_HOME = os.path.expanduser('~/.cache/paddle_data_set')

if not os.path.exists(DATA_HOME):
    os.makedirs(DATA_HOME)


def download(url, md5):
    filename = os.path.split(url)[-1]
    assert DATA_HOME is not None
    filepath = os.path.join(DATA_HOME, md5)
    if not os.path.exists(filepath):
        os.makedirs(filepath)
    __full_file__ = os.path.join(filepath, filename)

    def __file_ok__():
        if not os.path.exists(__full_file__):
            return False
        md5_hash = hashlib.md5()
        with open(__full_file__, 'rb') as f:<commit_msg>Set data cache home directory to ~/.cache/paddle/dataset<commit_after>import hashlib
import os
import shutil
import urllib2

__all__ = ['DATA_HOME', 'download']

DATA_HOME = os.path.expanduser('~/.cache/paddle/dataset')

if not os.path.exists(DATA_HOME):
    os.makedirs(DATA_HOME)


def download(url, md5):
    filename = os.path.split(url)[-1]
    assert DATA_HOME is not None
    filepath = os.path.join(DATA_HOME, md5)
    if not os.path.exists(filepath):
        os.makedirs(filepath)
    __full_file__ = os.path.join(filepath, filename)

    def __file_ok__():
        if not os.path.exists(__full_file__):
            return False
        md5_hash = hashlib.md5()
        with open(__full_file__, 'rb') as f:
{'fraction_non_alphanumeric': 0.11959654178674352, 'fraction_numerical': 0.008645533141210375, 'mean_word_length': 3.313664596273292, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3609971', 'n_tokens_mistral': 494, 'n_tokens_neox': 489, 'n_words': 107}