# Path: emolga/utils/generic_utils.py
# def get_from_module(identifier, module_params, module_name, instantiate=False, kwargs=None):
#     if isinstance(identifier, six.string_types):
#         res = module_params.get(identifier)
#         if not res:
#             raise Exception('Invalid ' + str(module_name) + ': ' + str(identifier))
#         if instantiate and not kwargs:
#             return res()
#         elif instantiate and kwargs:
#             return res(**kwargs)
#         else:
#             return res
#     return identifier
# 
# Path: emolga/dataset/build_dataset.py
# def deserialize_from_file(path):
#     f = open(path, 'rb')
#     obj = pickle.load(f)
#     f.close()
#     return obj
# 
# def serialize_to_file(obj, path, protocol=pickle.HIGHEST_PROTOCOL):
#     print('serialize to %s' % path)
#     f = open(path, 'wb')
#     pickle.dump(obj, f, protocol=protocol)
#     f.close()
# 
# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):
# 
# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):


from nltk.internals import find_jars_within_path
from nltk.tag import StanfordPOSTagger
from nltk import word_tokenize
from emolga.utils.generic_utils import get_from_module
from emolga.dataset.build_dataset import deserialize_from_file, serialize_to_file
from keyphrase.dataset import dataset_utils
from nltk.stem.porter import *
from keyphrase.dataset import dataset_utils
    from keyphrase.config import setup_keyphrase_stable
import os
import re
import shutil
import nltk
import xml.etree.ElementTree as ET
import keyphrase.dataset.dataset_utils as utils
import numpy as np
                import string
                import string
                import string
                import string
def check_data():
    config = setup_keyphrase_stable()
    train_set, validation_set, test_set, idx2word, word2idx = deserialize_from_file(config['dataset'])

    for dataset_name in config['testing_datasets']:
        print('*' * 50)
        print(dataset_name)

        number_groundtruth = 0
        number_present_groundtruth = 0

        loader = testing_data_loader(dataset_name, kwargs=dict(basedir = config['path']))

        if dataset_name == 'nus':
            docs   = loader.get_docs(only_abstract = True, return_dict=False)
        else:
            docs   = loader.get_docs(return_dict=False)

        stemmer = PorterStemmer()

        for id,doc in enumerate(docs):

            text_tokens = dataset_utils.get_tokens(doc.title.strip()+' '+ doc.text.strip())
            # if len(text_tokens) > 1500:
            #     text_tokens = text_tokens[:1500]
            print('[%d] length= %d' % (id, len(doc.text)))

            stemmed_input = [stemmer.stem(t).strip().lower() for t in text_tokens]

            phrase_str = ';'.join([l.strip() for l in doc.phrases])

# Path: emolga/utils/generic_utils.py
# def get_from_module(identifier, module_params, module_name, instantiate=False, kwargs=None):
#     if isinstance(identifier, six.string_types):
#         res = module_params.get(identifier)
#         if not res:
#             raise Exception('Invalid ' + str(module_name) + ': ' + str(identifier))
#         if instantiate and not kwargs:
#             return res()
#         elif instantiate and kwargs:
#             return res(**kwargs)
#         else:
#             return res
#     return identifier
# 
# Path: emolga/dataset/build_dataset.py
# def deserialize_from_file(path):
#     f = open(path, 'rb')
#     obj = pickle.load(f)
#     f.close()
#     return obj
# 
# def serialize_to_file(obj, path, protocol=pickle.HIGHEST_PROTOCOL):
#     print('serialize to %s' % path)
#     f = open(path, 'wb')
#     pickle.dump(obj, f, protocol=protocol)
#     f.close()
# 
# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):
# 
# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):


# Path: keyphrase/dataset/keyphrase_test_dataset.py
from nltk.internals import find_jars_within_path
from nltk.tag import StanfordPOSTagger
from nltk import word_tokenize
from emolga.utils.generic_utils import get_from_module
from emolga.dataset.build_dataset import deserialize_from_file, serialize_to_file
from keyphrase.dataset import dataset_utils
from nltk.stem.porter import *
from keyphrase.dataset import dataset_utils
    from keyphrase.config import setup_keyphrase_stable
import os
import re
import shutil
import nltk
import xml.etree.ElementTree as ET
import keyphrase.dataset.dataset_utils as utils
import numpy as np
                import string
                import string
                import string
                import string

def check_data():
    config = setup_keyphrase_stable()
    train_set, validation_set, test_set, idx2word, word2idx = deserialize_from_file(config['dataset'])

    for dataset_name in config['testing_datasets']:
        print('*' * 50)
        print(dataset_name)

        number_groundtruth = 0
        number_present_groundtruth = 0

        loader = testing_data_loader(dataset_name, kwargs=dict(basedir = config['path']))

        if dataset_name == 'nus':
            docs   = loader.get_docs(only_abstract = True, return_dict=False)
        else:
            docs   = loader.get_docs(return_dict=False)

        stemmer = PorterStemmer()

        for id,doc in enumerate(docs):

            text_tokens = dataset_utils.get_tokens(doc.title.strip()+' '+ doc.text.strip())
            # if len(text_tokens) > 1500:
            #     text_tokens = text_tokens[:1500]
            print('[%d] length= %d' % (id, len(doc.text)))

            stemmed_input = [stemmer.stem(t).strip().lower() for t in text_tokens]

            phrase_str = ';'.join([l.strip() for l in doc.phrases])

            phrases = dataset_utils.process_keyphrase(phrase_str)