<commit_before>import os
import re
import config
import urllib3
import argparse

def download(url, dbdir):
        http = urllib3.PoolManager()
        r = http.request('GET', url, preload_content = False)
        path = dbdir + url.split('/')[-1]
        print("downloading", url, "to", path)
        chunk_size = 16 * 1024
        with open(path, 'wb') as out:
                while True:
                        data = r.read(chunk_size)
                        if not data:
                                break
                        out.write(data)
        r.release_conn()

def mkdir(dbname):
        cfg = config.config()
        dbdir = cfg.dbdir + "/" + dbname + "/"
        os.makedirs(dbdir, exist_ok = True)
        return dbdir

def download_iris():
        dbdir = mkdir("iris")
        download("http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data", dbdir)

def download_wine():
        dbdir = mkdir("wine")
        download("http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data", dbdir)

def download_svhn():
        dbdir = mkdir("svhn")
        download("http://ufldl.stanford.edu/housenumbers/train_32x32.mat", dbdir)
        download("http://ufldl.stanford.edu/housenumbers/extra_32x32.mat", dbdir)
        download("http://ufldl.stanford.edu/housenumbers/test_32x32.mat", dbdir)<commit_msg>self-contained task downloading script<commit_after>import os
import re
import urllib3
import argparse

def download(url, dbdir):
        http = urllib3.PoolManager()
        r = http.request('GET', url, preload_content = False)
        path = dbdir + url.split('/')[-1]
        print("downloading", url, "to", path)
        chunk_size = 16 * 1024
        with open(path, 'wb') as out:
                while True:
                        data = r.read(chunk_size)
                        if not data:
                                break
                        out.write(data)
        r.release_conn()

def get_dbdir():
        homedir = os.path.expanduser('~')
        return os.path.join(homedir, "experiments", "databases")

def mkdir(dbname):
        dbdir = get_dbdir() + "/" + dbname + "/"
        os.makedirs(dbdir, exist_ok = True)
        return dbdir

def download_iris():
        dbdir = mkdir("iris")
        download("http://archive.ics.uci.edu/ml/machine-learning-databases/iris/iris.data", dbdir)

def download_wine():
        dbdir = mkdir("wine")
        download("http://archive.ics.uci.edu/ml/machine-learning-databases/wine/wine.data", dbdir)

def download_svhn():
        dbdir = mkdir("svhn")
        download("http://ufldl.stanford.edu/housenumbers/train_32x32.mat", dbdir)
        download("http://ufldl.stanford.edu/housenumbers/extra_32x32.mat", dbdir)
        download("http://ufldl.stanford.edu/housenumbers/test_32x32.mat", dbdir)
{'fraction_non_alphanumeric': 0.13610518834399432, 'fraction_numerical': 0.014925373134328358, 'mean_word_length': 2.184389140271493, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 21, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5868597', 'n_tokens_mistral': 952, 'n_tokens_neox': 906, 'n_words': 164}