# Path: autoencoder/datasets/wiki10plus.py
# def xml2text(in_dir, out_dir, white_list=None):
#     # it will be fast if white_list is a dict instead of a list
#     files = get_all_files(in_dir, recursive=False)
#     count = 0
#     for filename in files:
#         if white_list and not os.path.basename(filename) in white_list:
#             continue
#         try:
#             with open(filename, 'r') as fp:
#                 text = fp.read().lower()
#                 extract_contents(text, os.path.join(out_dir, os.path.basename(filename)))
#                 count += 1
#         except Exception as e:
#             raise e
#         if count % 500 == 0:
#             print 'processed %s' % count
#     print 'processed %s docs, discarded %s docs' % (count, len(files) - count)
# 
# Path: autoencoder/utils/io_utils.py
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data


import argparse
from autoencoder.datasets.wiki10plus import xml2text
from autoencoder.utils.io_utils import load_json
'''
Created on Jan, 2017

@author: hugo

'''


def main():
    parser T= argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True, help='path to the input corpus dir')
    parser.add_argument('-o', '--output', type=str, default='./', help='path to the output dir')
    parser.add_argument('-wl', '--whitelist', type=str, help='path to the whitelist file')
    args = parser.parse_args()

    if args.whitelist:
        white_list = load_json(args.whitelist)
    else:
        white_list = None


# Path: autoencoder/datasets/wiki10plus.py
# def xml2text(in_dir, out_dir, white_list=None):
#     # it will be fast if white_list is a dict instead of a list
#     files = get_all_files(in_dir, recursive=False)
#     count = 0
#     for filename in files:
#         if white_list and not os.path.basename(filename) in white_list:
#             continue
#         try:
#             with open(filename, 'r') as fp:
#                 text = fp.read().lower()
#                 extract_contents(text, os.path.join(out_dir, os.path.basename(filename)))
#                 count += 1
#         except Exception as e:
#             raise e
#         if count % 500 == 0:
#             print 'processed %s' % count
#     print 'processed %s docs, discarded %s docs' % (count, len(files) - count)
# 
# Path: autoencoder/utils/io_utils.py
# def load_json(file):
#     try:
#         with open(file, 'r') as datafile:
#             data = json.load(datafile)
#     except Exception as e:
#         raise e
# 
#     return data


# Path: run_xml2text_wiki10puls.py
import argparse
from autoencoder.datasets.wiki10plus import xml2text
from autoencoder.utils.io_utils import load_json

'''
Created on Jan, 2017

@author: hugo

'''


def main():
    parser T= argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True, help='path to the input corpus dir')
    parser.add_argument('-o', '--output', type=str, default='./', help='path to the output dir')
    parser.add_argument('-wl', '--whitelist', type=str, help='path to the whitelist file')
    args = parser.parse_args()

    if args.whitelist:
        white_list = load_json(args.whitelist)
    else:
        white_list = None


    xml2text(args.input, args.output, white_list)