<commit_before>import argparse
import datetime
from os import makedirs, getcwd
from os.path import isdir, exists
from project import corpus, knn_corpus, lda_corpus, word2vec_corpus

algorithms = {"lda": lda_corpus.LDACorpus,
              "knn": knn_corpus.KNNCorpus,
              "w2v": word2vec_corpus.W2VCorpus}

base_dir = getcwd()
output_loc = base_dir + "/%s.corpus_out"
dictionary_loc = output_loc + "/%dcorpus.dict"
corpus_loc = output_loc + "/%dcorpus.mm"
log_file = output_loc + "/runtimes.log"
sup_file_loc = output_loc + "/%d.%s"


def run_algo(directory, ints, algorithm):
    # Build corpus from largest int and directory (Check valid directory)
    max_corpus = max(ints)
    output_dir = output_loc % algorithm
    if not exists(output_dir):
        makedirs(output_dir)
    log = open(log_file % algorithm, 'a+')
    base_corpus_file = corpus_loc % (algorithm, max_corpus)
    max_dict = dictionary_loc % (algorithm, max_corpus)
    start_time = datetime.datetime.now()

    # Build corpus of size max_corpus and save to be reused
    base_corpus = corpus.Corpus(directory=directory, max_docs=max_corpus)
    base_corpus.save(max_dict, base_corpus_file)
    end_time = datetime.datetime.now()
    base_corpus_build_time = end_time - start_time
    log.write(("base_corpus_build_time %d:\t" % max_corpus) + str(base_corpus_build_time) + "\n")

    # For each int in the param list then apply the corpus algorithm using a sliced corpus
    for size in ints:
        if size is max_corpus: continue
        test_corpus = algorithms[algorithm](dictionary=max_dict, corpus=base_corpus_file, max_docs=size)

        # Save any mm, index files, etc to a directory so they can be used again.
        dict_loc = dictionary_loc % (algorithm, size)
        corp_loc = corpus_loc % (algorithm, size)
        sup_loc = sup_file_loc % (algorithm, size, algorithm)
        test_corpus.save(dictionary_file=dict_loc, corpus_file=corp_loc, sup_file=sup_loc)

        # Log temporal time
        log.write("%s %d train time:\t" % (algorithm, size) + str(test_corpus.get_train_time()) + "\n")
    log.close()


def main():
    parser = argparse.ArgumentParser(description='Build bow corpus on the arxiv corpus.')
    parser.add_argument('integers', metavar='N', type=int, nargs='+', help='size values for the corpus')
    parser.add_argument('directory', help='directory for the arxiv txt files')
    parser.add_argument('algorithm', help='algorithm to apply to the corpus', choices=algorithms)
    args = parser.parse_args()
    if isdir(args.directory):
        run_algo(args.directory, args.integers, args.algorithm)
    else:
        print "Directory argument should be a valid directory"


if __name__ == "__main__":
    main()<commit_msg>Rename corpus output files<commit_after>import argparse
import datetime
from os import makedirs, getcwd
from os.path import isdir, exists
from project import corpus, knn_corpus, lda_corpus, word2vec_corpus

algorithms = {"lda": lda_corpus.LDACorpus,
              "knn": knn_corpus.KNNCorpus,
              "w2v": word2vec_corpus.W2VCorpus}

base_dir = getcwd()
output_loc = base_dir + "/%s.corpus_out"
dictionary_loc = output_loc + "/%scorpus.dict"
corpus_loc = output_loc + "/%scorpus.mm"
log_file = output_loc + "/runtimes.log"
sup_file_loc = output_loc + "/%d.%s"


def run_algo(directory, ints, algorithm):
    # Build corpus from largest int and directory (Check valid directory)
    max_corpus = max(ints)
    output_dir = output_loc % algorithm
    if not exists(output_dir):
        makedirs(output_dir)
    log = open(log_file % algorithm, 'a+')
    base_corpus_file = corpus_loc % (algorithm, "base_")
    max_dict = dictionary_loc % (algorithm, "base_")
    start_time = datetime.datetime.now()

    # Build corpus of size max_corpus and save to be reused
    base_corpus = corpus.Corpus(directory=directory, max_docs=max_corpus)
    base_corpus.save(max_dict, base_corpus_file)
    end_time = datetime.datetime.now()
    base_corpus_build_time = end_time - start_time
    log.write(("base_corpus_build_time %d:\t" % max_corpus) + str(base_corpus_build_time) + "\n")

    # For each int in the param list then apply the corpus algorithm using a sliced corpus
    for size in ints:
        test_corpus = algorithms[algorithm](dictionary=max_dict, corpus=base_corpus_file, max_docs=size)

        # Save any mm, index files, etc to a directory so they can be used again.
        dict_loc = dictionary_loc % (algorithm, size)
        corp_loc = corpus_loc % (algorithm, size)
        sup_loc = sup_file_loc % (algorithm, size, algorithm)
        test_corpus.save(dictionary_file=dict_loc, corpus_file=corp_loc, sup_file=sup_loc)

        # Log temporal time
        log.write("%s %d train time:\t" % (algorithm, size) + str(test_corpus.get_train_time()) + "\n")
    log.close()


def main():
    parser = argparse.ArgumentParser(description='Build bow corpus on the arxiv corpus.')
    parser.add_argument('integers', metavar='N', type=int, nargs='+', help='size values for the corpus')
    parser.add_argument('directory', help='directory for the arxiv txt files')
    parser.add_argument('algorithm', help='algorithm to apply to the corpus', choices=algorithms)
    args = parser.parse_args()
    if isdir(args.directory):
        run_algo(args.directory, args.integers, args.algorithm)
    else:
        print "Directory argument should be a valid directory"


if __name__ == "__main__":
    main()
{'fraction_non_alphanumeric': 0.10544654318723638, 'fraction_numerical': 0.0014670823399963323, 'mean_word_length': 3.9446962828649137, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1638775', 'n_tokens_mistral': 1825, 'n_tokens_neox': 1761, 'n_words': 463}