# Path: autoencoder/preprocessing/preprocessing.py
# def construct_train_test_corpus(train_path, test_path, output, threshold=5, topn=None):
#     train_docs, vocab_dict, train_word_freq = construct_corpus(train_path, True, threshold=threshold, topn=topn, recursive=True)
#     train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
#     dump_json(train_corpus, os.path.join(output, 'train.corpus'))
#     print 'Generated training corpus'
# 
#     test_docs, _, _ = construct_corpus(test_path, False, vocab_dict=vocab_dict, recursive=True)
#     test_corpus = {'docs': test_docs, 'vocab': vocab_dict}
#     dump_json(test_corpus, os.path.join(output, 'test.corpus'))
#     print 'Generated test corpus'
# 
#     return train_corpus, test_corpus
# 
# def generate_20news_doc_labels(doc_names, output):
#     doc_labels = {}
#     for each in doc_names:
#        label = each.split('_')[0]
#        doc_labels[each] = label
# 
#     dump_json(doc_labels, output)
# 
#     return doc_labels


import os
import argparse
from autoencoder.preprocessing.preprocessing import construct_train_test_corpus, generate_20news_doc_labels#, generate_8k_doc_labels
'''
Created on Dec, 2016

@author: hugo

'''




def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-train', '--train_path', type=str, required=True, help='path to the training corpus')
    parser.add_argument('-test', '--test_path', type=str, required=True, help='path to the test corpus')
    parser.add_argument('-o', '--out_dir', type=str, required=True, help='path to the output dir')
    parser.add_argument('-threshold', '--threshold', type=int, default=5, help='word frequency threshold (default 5)')
    parser.add_argument('-topn', '--topn', type=int, default=2000, help='top n words (default 2000)')
    args = parser.parse_args()

    train_corpus, test_corpus = construct_train_test_corpus(args.train_path, args.test_path, args.out_dir, threshold=args.threshold, topn=args.topn)
    train_labels = generate_20news_doc_labels(train_corpus['docs'].keys(), os.path.join(args.out_dir, 'train.labels'))

# Path: autoencoder/preprocessing/preprocessing.py
# def construct_train_test_corpus(train_path, test_path, output, threshold=5, topn=None):
#     train_docs, vocab_dict, train_word_freq = construct_corpus(train_path, True, threshold=threshold, topn=topn, recursive=True)
#     train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
#     dump_json(train_corpus, os.path.join(output, 'train.corpus'))
#     print 'Generated training corpus'
# 
#     test_docs, _, _ = construct_corpus(test_path, False, vocab_dict=vocab_dict, recursive=True)
#     test_corpus = {'docs': test_docs, 'vocab': vocab_dict}
#     dump_json(test_corpus, os.path.join(output, 'test.corpus'))
#     print 'Generated test corpus'
# 
#     return train_corpus, test_corpus
# 
# def generate_20news_doc_labels(doc_names, output):
#     doc_labels = {}
#     for each in doc_names:
#        label = each.split('_')[0]
#        doc_labels[each] = label
# 
#     dump_json(doc_labels, output)
# 
#     return doc_labels


# Path: construct_20news.py
import os
import argparse
from autoencoder.preprocessing.preprocessing import construct_train_test_corpus, generate_20news_doc_labels#, generate_8k_doc_labels

'''
Created on Dec, 2016

@author: hugo

'''




def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-train', '--train_path', type=str, required=True, help='path to the training corpus')
    parser.add_argument('-test', '--test_path', type=str, required=True, help='path to the test corpus')
    parser.add_argument('-o', '--out_dir', type=str, required=True, help='path to the output dir')
    parser.add_argument('-threshold', '--threshold', type=int, default=5, help='word frequency threshold (default 5)')
    parser.add_argument('-topn', '--topn', type=int, default=2000, help='top n words (default 2000)')
    args = parser.parse_args()

    train_corpus, test_corpus = construct_train_test_corpus(args.train_path, args.test_path, args.out_dir, threshold=args.threshold, topn=args.topn)
    train_labels = generate_20news_doc_labels(train_corpus['docs'].keys(), os.path.join(args.out_dir, 'train.labels'))

    test_labels = generate_20news_doc_labels(test_corpus['docs'].keys(), os.path.join(args.out_dir, 'test.labels'))