<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function, absolute_import
import argparse
import os
import subprocess as sp
from contextlib import contextmanager
import tempfile
try:
    import urllib.request as urllib2
except ImportError:
    import urllib2

MINICONDA_URL = 'https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh'


@contextmanager
def change_directory(path):
    old_cwd = os.getcwd()
    try:
        os.chdir(path)
        yield
    finally:
        os.chdir(old_cwd)


def main(args):
    pass


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('-d', '--directory',
                        required=False,
                        default=os.getcwd())
    main(parser.parse_args())<commit_msg>Add download function<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division, print_function, absolute_import
import argparse
import os
import subprocess as sp
from contextlib import contextmanager
import tempfile
try:
    import urllib.request as urllib2
except ImportError:
    import urllib2

MINICONDA_URL = 'https://repo.continuum.io/miniconda/Miniconda-latest-Linux-x86_64.sh'


@contextmanager
def change_directory(path):
    old_cwd = os.getcwd()
    try:
        os.chdir(path)
        yield
    finally:
        os.chdir(old_cwd)


def download_install_script():
    location = os.path.join(
        tempfile.gettempdir(),
        os.path.split(MINICONDA_URL)[-1])

    with open(location, 'wb') as outfile:
        response = urllib2.urlopen(MINICONDA_URL)
        data = response.read()
        outfile.write(data)
    return location


{'fraction_non_alphanumeric': 0.10245901639344263, 'fraction_numerical': 0.00936768149882904, 'mean_word_length': 3.088516746411483, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '306646', 'n_tokens_mistral': 572, 'n_tokens_neox': 556, 'n_words': 122}