<commit_before>import os
import pickle
from datetime import datetime
from ConfigParser import ConfigParser
from IPython.parallel import Client

config = ConfigParser()
config.read('{0}/config.ini'.format(os.path.dirname(os.path.realpath(__file__))))

rc = Client(packer="pickle")
dview = rc[:]
print rc.ids


@dview.remote(block=True)
def fetch(directory):
    import os
    if not os.path.exists(directory):
        os.makedirs(directory)
    for f in files:
        fname = f.split("/")[-1]
        os.chdir(directory)
        if not os.path.exists(fname):
            os.system("wget {0}".format(f))
            os.system("unzip {0}".format(fname))


fname = open("{0}/urls.pickle".format(config.get('directory', 'sqlalchemy')), "rb")
urls = pickle.load(fname)

for year in xrange(2005, 2014):
    print year, datetime.now()
    dview.scatter("files", urls[year])
    directory = "{0}/{1}".format(config.get('directory', 'local'), year)
    print "  *", directory
    fetch(directory)<commit_msg>creating remote directories<commit_after>import os
import pickle
from datetime import datetime
from ConfigParser import ConfigParser
from IPython.parallel import Client

config = ConfigParser()
config.read('{0}/config.ini'.format(os.path.dirname(os.path.realpath(__file__))))

rc = Client(packer="pickle")
dview = rc[:]
print rc.ids


@dview.remote(block=True)
def makedir(directory):
    import os
    try:
        os.makedirs(directory)
    except:
        pass

@dview.remote(block=True)
def fetch(directory):
    import os
    for f in files:
        fname = f.split("/")[-1]
        os.chdir(directory)
        if not os.path.exists(fname):
            os.system("wget {0}".format(f))
            os.system("unzip {0}".format(fname))


fname = open("{0}/urls.pickle".format(config.get('directory', 'sqlalchemy')), "rb")
urls = pickle.load(fname)

for year in xrange(2005, 2014):
    print year, datetime.now()
    dview.scatter("files", urls[year])
    directory = "{0}/{1}".format(config.get('directory', 'local'), year)
    print "  *", directory
    makedir(directory)
    fetch(directory)
{'fraction_non_alphanumeric': 0.15791984732824427, 'fraction_numerical': 0.01431297709923664, 'mean_word_length': 3.6912751677852347, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2454099', 'n_tokens_mistral': 720, 'n_tokens_neox': 701, 'n_words': 149}