# Path: autoencoder/datasets/wikitag_extractor.py
# def extract_labels(in_file, topn):
#     #XML PARSING
#     parser = xml.sax.make_parser()
#     parser.setContentHandler(XMLhandler())
#     parser.parse(in_file)
# 
#     #NOTE in the xml file "css"  and "files" also come in the <hash>....</hash>, so i remove them manually
#     labeldict.pop("css")
#     labeldict.pop("files")
# 
#     #TO TAKE LABELS WITH AT LEAST N OCCURENCES ONLY
#     freqs = Counter(labels)
#     pairs = sorted(freqs.items(), key=lambda item: item[1], reverse=True)
#     pairs = pairs[:topn]
#     newlabels = zip(*pairs)[0]
#     newlabeldict = {}
# 
#     #REMOVE EXTRA LABELS FROM LABEL DICTIONARY (HASH-LABEL MAPPING)
#     #ALSO CHECK IF WE LOST ANY DOCUMENT; IN CASE A DOCUMENT HAS 0 LABELS
# 
#     se = set(newlabels)
#     for i in labeldict:
#         newlabeldict[i] = list(se.intersection(labeldict[i]))
#         if len(newlabeldict[i]) == 0:
#             newlabeldict.pop(i)
# 
#     return newlabeldict
# 
# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


import argparse
from autoencoder.datasets.wikitag_extractor import extract_labels
from autoencoder.utils.io_utils import dump_json
'''
Created on Jan, 2017

@author: hugo

'''



def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True, help='path to the input source file')
    parser.add_argument('--topn', type=int, default=25, help='keep only topn most frequent labels')
    parser.add_argument('-o', '--output', type=str, help='path to the output file')
    args = parser.parse_args()


    labeldict = extract_labels(args.input, args.topn)

# Path: autoencoder/datasets/wikitag_extractor.py
# def extract_labels(in_file, topn):
#     #XML PARSING
#     parser = xml.sax.make_parser()
#     parser.setContentHandler(XMLhandler())
#     parser.parse(in_file)
# 
#     #NOTE in the xml file "css"  and "files" also come in the <hash>....</hash>, so i remove them manually
#     labeldict.pop("css")
#     labeldict.pop("files")
# 
#     #TO TAKE LABELS WITH AT LEAST N OCCURENCES ONLY
#     freqs = Counter(labels)
#     pairs = sorted(freqs.items(), key=lambda item: item[1], reverse=True)
#     pairs = pairs[:topn]
#     newlabels = zip(*pairs)[0]
#     newlabeldict = {}
# 
#     #REMOVE EXTRA LABELS FROM LABEL DICTIONARY (HASH-LABEL MAPPING)
#     #ALSO CHECK IF WE LOST ANY DOCUMENT; IN CASE A DOCUMENT HAS 0 LABELS
# 
#     se = set(newlabels)
#     for i in labeldict:
#         newlabeldict[i] = list(se.intersection(labeldict[i]))
#         if len(newlabeldict[i]) == 0:
#             newlabeldict.pop(i)
# 
#     return newlabeldict
# 
# Path: autoencoder/utils/io_utils.py
# def dump_json(data, file):
#     try:
#         with open(file, 'w') as datafile:
#             json.dump(data, datafile)
#     except Exception as e:
#         raise e


# Path: run_wikitag_extractor.py
import argparse
from autoencoder.datasets.wikitag_extractor import extract_labels
from autoencoder.utils.io_utils import dump_json

'''
Created on Jan, 2017

@author: hugo

'''



def main():
    parser = argparse.ArgumentParser()
    parser.add_argument('-i', '--input', type=str, required=True, help='path to the input source file')
    parser.add_argument('--topn', type=int, default=25, help='keep only topn most frequent labels')
    parser.add_argument('-o', '--output', type=str, help='path to the output file')
    args = parser.parse_args()


    labeldict = extract_labels(args.input, args.topn)

    dump_json(labeldict, args.output)