# Path: autoencoder/datasets/movie_review_data.py
# def construct_train_test_corpus(file, test_split, output, threshold=10, topn=20000):
#     train_data, train_labels, test_data, test_labels = load_data(file, test_split)
#     train_word_freq = count_words(train_data.values())
# 
#     train_docs, vocab_dict, train_word_freq = construct_corpus(train_data, train_word_freq, True, threshold=threshold, topn=topn)
#     train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
#     dump_json(train_corpus, os.path.join(output, 'train.corpus'))
#     print 'Generated training corpus'
#     dump_json(train_labels, os.path.join(output, 'train.labels'))
#     print 'Generated training labels'
# 
#     test_word_freq = count_words(test_data.values())
#     test_docs, _, _ = construct_corpus(test_data, test_word_freq, False, vocab_dict=vocab_dict)
#     test_corpus = {'docs': test_docs, 'vocab': vocab_dict}
#     dump_json(test_corpus, os.path.join(output, 'test.corpus'))
#     print 'Generated test corpus'
#     dump_json(test_labels, os.path.join(output, 'test.labels'))
#     print 'Generated test labels'
#     import pdb;pdb.set_trace()


import argparse
from autoencoder.datasets.movie_review_data import construct_train_test_corpus
'''
Created on Jan, 2017

@author: hugo

'''



def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True, help='path to the input corpus file')
    parser.add_argument('-o', '--output', type=str, default='./', help='path to the output dir')
    parser.add_argument('-ts', '--test_split', type=float, required=True, help='fraction of the dataset to be used as test data')
    args = parser.parse_args()


# Path: autoencoder/datasets/movie_review_data.py
# def construct_train_test_corpus(file, test_split, output, threshold=10, topn=20000):
#     train_data, train_labels, test_data, test_labels = load_data(file, test_split)
#     train_word_freq = count_words(train_data.values())
# 
#     train_docs, vocab_dict, train_word_freq = construct_corpus(train_data, train_word_freq, True, threshold=threshold, topn=topn)
#     train_corpus = {'docs': train_docs, 'vocab': vocab_dict, 'word_freq': train_word_freq}
#     dump_json(train_corpus, os.path.join(output, 'train.corpus'))
#     print 'Generated training corpus'
#     dump_json(train_labels, os.path.join(output, 'train.labels'))
#     print 'Generated training labels'
# 
#     test_word_freq = count_words(test_data.values())
#     test_docs, _, _ = construct_corpus(test_data, test_word_freq, False, vocab_dict=vocab_dict)
#     test_corpus = {'docs': test_docs, 'vocab': vocab_dict}
#     dump_json(test_corpus, os.path.join(output, 'test.corpus'))
#     print 'Generated test corpus'
#     dump_json(test_labels, os.path.join(output, 'test.labels'))
#     print 'Generated test labels'
#     import pdb;pdb.set_trace()


# Path: construct_movie_review_data.py
import argparse
from autoencoder.datasets.movie_review_data import construct_train_test_corpus

'''
Created on Jan, 2017

@author: hugo

'''



def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True, help='path to the input corpus file')
    parser.add_argument('-o', '--output', type=str, default='./', help='path to the output dir')
    parser.add_argument('-ts', '--test_split', type=float, required=True, help='fraction of the dataset to be used as test data')
    args = parser.parse_args()


    construct_train_test_corpus(args.input, args.test_split, args.output, threshold=10, topn=2000)