<commit_before>import os
import commands
import time
import logging
import sys

if len(sys.argv) != 3 or (sys.argv[1:][0] not in ['pt', 'en']):
    print "ERROR! Use:\n python scripts/data_download/higher_education/create_files.py en/pt output_path\n"
    exit()

for year in range(2009, 2015):
    print "python scripts/data_download/higher_education/create_files.py "+str(sys.argv[1])+" "+str(sys.argv[2])+" "+ str(year)
    # commands.getoutput("python scripts/data_download/higher_education/create_files.py "+sys.argv[1]+" "+sys.argv[2]+" "+sys.argv[3])
   <commit_msg>Add file to create all files to higher education.<commit_after>import os
import commands
import time
import logging
import sys

if len(sys.argv) != 3 or (sys.argv[1:][0] not in ['pt', 'en']):
    print "ERROR! Use:\n python scripts/data_download/higher_education/create_files.py en/pt output_path\n"
    exit()

logging.basicConfig(filename=os.path.abspath(os.path.join(sys.argv[2],str(sys.argv[0].split('/')[2]) + '-all-data-download.log' )),level=logging.DEBUG)

for year in range(2009, 2015):
    logging.info("python scripts/data_download/higher_education/create_files.py "+str(sys.argv[1])+" "+str(sys.argv[2])+" "+ str(year) + "\n")
    ret = commands.getoutput("python scripts/data_download/higher_education/create_files.py "+str(sys.argv[1])+" "+str(sys.argv[2])+" "+ str(year))
    logging.info(str(ret) + "\nYear: " + str(year) + " ok =D\n\n")
{'fraction_non_alphanumeric': 0.18302945301542778, 'fraction_numerical': 0.023842917251051893, 'mean_word_length': 5.286343612334802, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3399709', 'n_tokens_mistral': 534, 'n_tokens_neox': 499, 'n_words': 103}