# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):


import math
import logging
import string
import scipy
import numpy as np
import os
import sys
import keyphrase.config as config
import keyphrase.config
from nltk.stem.porter import *
from keyphrase.dataset import dataset_utils

def evaluate_(text_dir, target_dir, prediction_dir, model_name, dataset_name, do_stem=True):
    '''
    '''
    stemmer = PorterStemmer()

    print('Evaluating on %s@%s' % (model_name, dataset_name))
    # Evaluation part
    macro_metrics = []
    macro_matches = []

    doc_names = [name[:name.index('.')] for name in os.listdir(text_dir)]

    number_groundtruth = 0
    number_present_groundtruth = 0

    for doc_name in doc_names:
        logger.info('[FILE]{0}'.format(text_dir+'/'+doc_name+'.txt'))
        with open(text_dir+'/'+doc_name+'.txt', 'r') as f:
            text_tokens = (' '.join(f.readlines())).split( )

            text    = [t.split('_')[0] for t in text_tokens]
            postag  = [t.split('_')[1] for t in text_tokens]

        targets = load_phrase(target_dir+'/'+doc_name+'.txt', True)

        predictions = load_phrase(prediction_dir+'/'+doc_name+'.txt.phrases', False)

        # do processing to baseline predictions
        if (not model_name.startswith('CopyRNN')) and (not model_name.startswith('RNN')):

# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):


# Path: keyphrase/baseline/evaluate.py
import math
import logging
import string
import scipy
import numpy as np
import os
import sys
import keyphrase.config as config
import keyphrase.config
from nltk.stem.porter import *
from keyphrase.dataset import dataset_utils


def evaluate_(text_dir, target_dir, prediction_dir, model_name, dataset_name, do_stem=True):
    '''
    '''
    stemmer = PorterStemmer()

    print('Evaluating on %s@%s' % (model_name, dataset_name))
    # Evaluation part
    macro_metrics = []
    macro_matches = []

    doc_names = [name[:name.index('.')] for name in os.listdir(text_dir)]

    number_groundtruth = 0
    number_present_groundtruth = 0

    for doc_name in doc_names:
        logger.info('[FILE]{0}'.format(text_dir+'/'+doc_name+'.txt'))
        with open(text_dir+'/'+doc_name+'.txt', 'r') as f:
            text_tokens = (' '.join(f.readlines())).split( )

            text    = [t.split('_')[0] for t in text_tokens]
            postag  = [t.split('_')[1] for t in text_tokens]

        targets = load_phrase(target_dir+'/'+doc_name+'.txt', True)

        predictions = load_phrase(prediction_dir+'/'+doc_name+'.txt.phrases', False)

        # do processing to baseline predictions
        if (not model_name.startswith('CopyRNN')) and (not model_name.startswith('RNN')):

            predictions = dataset_utils.process_keyphrase(';'.join([' '.join(p) for p in predictions]))