<commit_before>import os
import pickle
from datetime import datetime
from ConfigParser import ConfigParser
from IPython.parallel import Client

config = ConfigParser()
config.read('{0}/config.ini'.format(os.path.dirname(os.path.realpath(__file__))))

rc = Client(packer="pickle")
dview = rc[:]
print rc.ids


@dview.remote(block=True)
def makedir(node):
    import os
    if not os.path.exists(node):
        os.makedirs(node)


@dview.remote(block=True)
def fetch(master, node, node_base):
    import os
    for f in files:
        fname = f.split("/")[-1].split("."[0])
        os.chdir(master)
        if not os.path.exists("{0}/{1}.zip".format(master, fname)):
            os.system("wget {0}.zip".format(f))
        if not os.path.exists("{0}/{1}.xml".format(node_base, fname)):
            os.system("unzip {1} -d {0}.zip".format(node_base, fname))


fname = open("{0}/urls.pickle".format(config.get('directory', 'sqlalchemy')), "rb")
urls = pickle.load(fname)<commit_msg>fixed minor typo<commit_after>import os
import pickle
from datetime import datetime
from ConfigParser import ConfigParser
from IPython.parallel import Client

config = ConfigParser()
config.read('{0}/config.ini'.format(os.path.dirname(os.path.realpath(__file__))))

rc = Client(packer="pickle")
dview = rc[:]
print rc.ids


@dview.remote(block=True)
def makedir(node):
    import os
    if not os.path.exists(node):
        os.makedirs(node)


@dview.remote(block=True)
def fetch(master, node, node_base):
    import os
    for f in files:
        fname = f.split("/")[-1].split(".")[0]
        os.chdir(master)
        if not os.path.exists("{0}/{1}.zip".format(master, fname)):
            os.system("wget {0}.zip".format(f))
        if not os.path.exists("{0}/{1}.xml".format(node_base, fname)):
            os.system("unzip {1} -d {0}.zip".format(node_base, fname))


fname = open("{0}/urls.pickle".format(config.get('directory', 'sqlalchemy')), "rb")
urls = pickle.load(fname)
{'fraction_non_alphanumeric': 0.17662072485962227, 'fraction_numerical': 0.011230219499744768, 'mean_word_length': 3.7457627118644066, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2817242', 'n_tokens_mistral': 700, 'n_tokens_neox': 693, 'n_words': 139}