# Path: emolga/utils/generic_utils.py
# def get_from_module(identifier, module_params, module_name, instantiate=False, kwargs=None):
#     if isinstance(identifier, six.string_types):
#         res = module_params.get(identifier)
#         if not res:
#             raise Exception('Invalid ' + str(module_name) + ': ' + str(identifier))
#         if instantiate and not kwargs:
#             return res()
#         elif instantiate and kwargs:
#             return res(**kwargs)
#         else:
#             return res
#     return identifier
# 
# Path: emolga/dataset/build_dataset.py
# def deserialize_from_file(path):
#     f = open(path, 'rb')
#     obj = pickle.load(f)
#     f.close()
#     return obj
# 
# def serialize_to_file(obj, path, protocol=pickle.HIGHEST_PROTOCOL):
#     print('serialize to %s' % path)
#     f = open(path, 'wb')
#     pickle.dump(obj, f, protocol=protocol)
#     f.close()
# 
# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):
# 
# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):


from nltk.internals import find_jars_within_path
from nltk.tag import StanfordPOSTagger
from nltk import word_tokenize
from emolga.utils.generic_utils import get_from_module
from emolga.dataset.build_dataset import deserialize_from_file, serialize_to_file
from keyphrase.dataset import dataset_utils
from nltk.stem.porter import *
from keyphrase.dataset import dataset_utils
    from keyphrase.config import setup_keyphrase_stable
import os
import re
import shutil
import nltk
import xml.etree.ElementTree as ET
import keyphrase.dataset.dataset_utils as utils
import numpy as np
                import string
                import string
                import string
                import string


def get_postag_with_index(sources, idx2word, word2idx):
    path = os.path.dirname(__file__)
    path = path[:path.rfind(os.sep, 0, len(path)-10)+1] + 'stanford-postagger/'
    print(path)
    # jar = '/Users/memray/Project/stanford/stanford-postagger/stanford-postagger.jar'
    jar = path + '/stanford-postagger.jar'
    model = path + '/models/english-bidirectional-distsim.tagger'
    pos_tagger = StanfordPOSTagger(model, jar)
    # model = '/Users/memray/Project/stanford/stanford-postagger/models/english-left3words-distsim.tagger'
    # model = '/Users/memray/Project/stanford/stanford-postagger/models/english-bidirectional-distsim.tagger'

    stanford_dir = jar.rpartition('/')[0]
    stanford_jars = find_jars_within_path(stanford_dir)
    pos_tagger._stanford_jar = ':'.join(stanford_jars)

    tagged_source = []
    # Predict on testing data
    for idx in xrange(len(sources)):  # len(test_data_plain)
        test_s_o = sources[idx]
        source_text = keyphrase_utils.cut_zero(test_s_o, idx2word)
        text = pos_tagger.tag(source_text)
        print('[%d/%d] : %s' % (idx, len(sources), str(text)))

        tagged_source.append(text)

    return tagged_source

def check_postag(config):

# Path: emolga/utils/generic_utils.py
# def get_from_module(identifier, module_params, module_name, instantiate=False, kwargs=None):
#     if isinstance(identifier, six.string_types):
#         res = module_params.get(identifier)
#         if not res:
#             raise Exception('Invalid ' + str(module_name) + ': ' + str(identifier))
#         if instantiate and not kwargs:
#             return res()
#         elif instantiate and kwargs:
#             return res(**kwargs)
#         else:
#             return res
#     return identifier
# 
# Path: emolga/dataset/build_dataset.py
# def deserialize_from_file(path):
#     f = open(path, 'rb')
#     obj = pickle.load(f)
#     f.close()
#     return obj
# 
# def serialize_to_file(obj, path, protocol=pickle.HIGHEST_PROTOCOL):
#     print('serialize to %s' % path)
#     f = open(path, 'wb')
#     pickle.dump(obj, f, protocol=protocol)
#     f.close()
# 
# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):
# 
# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):


# Path: keyphrase/dataset/keyphrase_test_dataset.py
from nltk.internals import find_jars_within_path
from nltk.tag import StanfordPOSTagger
from nltk import word_tokenize
from emolga.utils.generic_utils import get_from_module
from emolga.dataset.build_dataset import deserialize_from_file, serialize_to_file
from keyphrase.dataset import dataset_utils
from nltk.stem.porter import *
from keyphrase.dataset import dataset_utils
    from keyphrase.config import setup_keyphrase_stable
import os
import re
import shutil
import nltk
import xml.etree.ElementTree as ET
import keyphrase.dataset.dataset_utils as utils
import numpy as np
                import string
                import string
                import string
                import string



def get_postag_with_index(sources, idx2word, word2idx):
    path = os.path.dirname(__file__)
    path = path[:path.rfind(os.sep, 0, len(path)-10)+1] + 'stanford-postagger/'
    print(path)
    # jar = '/Users/memray/Project/stanford/stanford-postagger/stanford-postagger.jar'
    jar = path + '/stanford-postagger.jar'
    model = path + '/models/english-bidirectional-distsim.tagger'
    pos_tagger = StanfordPOSTagger(model, jar)
    # model = '/Users/memray/Project/stanford/stanford-postagger/models/english-left3words-distsim.tagger'
    # model = '/Users/memray/Project/stanford/stanford-postagger/models/english-bidirectional-distsim.tagger'

    stanford_dir = jar.rpartition('/')[0]
    stanford_jars = find_jars_within_path(stanford_dir)
    pos_tagger._stanford_jar = ':'.join(stanford_jars)

    tagged_source = []
    # Predict on testing data
    for idx in xrange(len(sources)):  # len(test_data_plain)
        test_s_o = sources[idx]
        source_text = keyphrase_utils.cut_zero(test_s_o, idx2word)
        text = pos_tagger.tag(source_text)
        print('[%d/%d] : %s' % (idx, len(sources), str(text)))

        tagged_source.append(text)

    return tagged_source

def check_postag(config):

    train_set, validation_set, test_set, idx2word, word2idx = deserialize_from_file(config['dataset'])