# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):


import math
import logging
import string
import scipy
import numpy as np
import os
import sys
import keyphrase.config as config
import keyphrase.config
from nltk.stem.porter import *
from keyphrase.dataset import dataset_utils


# prepare logging.

# config = keyphrase.config.setup_keyphrase_all()
config = keyphrase.config.setup_keyphrase_baseline()  # load settings.

def load_phrase(file_path, tokenize=True):
    phrases = []
    with open(file_path, 'r') as f:
        # TODO here the ground-truth is already after processing, contains <digit>, not good for baseline methods...
        if tokenize:
            phrase_str = ';'.join([l.strip() for l in f.readlines()])
            phrases = dataset_utils.process_keyphrase(phrase_str)

# Path: keyphrase/dataset/dataset_utils.py
# SENTENCEDELIMITER = '<eos>'
# DIGIT = '<digit>'
#         A = [word2idx[w] if w in word2idx else word2idx['<unk>'] for w in source]
#         B = [[word2idx[w] if w in word2idx else word2idx['<unk>'] for w in p] for p in target]
#         C = [[0 if w not in source else source.index(w) + Lmax for w in p] for p in target]
# def prepare_text(record, process_type=1):
# def get_tokens(text, process_type=1):
# def process_keyphrase(keyword_str):
# def build_data(data, idx2word, word2idx):
# def load_pairs(records, process_type=1 ,do_filter=False):
# def get_none_phrases(source_text, source_postag, max_len):


# Path: keyphrase/baseline/evaluate.py
import math
import logging
import string
import scipy
import numpy as np
import os
import sys
import keyphrase.config as config
import keyphrase.config
from nltk.stem.porter import *
from keyphrase.dataset import dataset_utils



# prepare logging.

# config = keyphrase.config.setup_keyphrase_all()
config = keyphrase.config.setup_keyphrase_baseline()  # load settings.

def load_phrase(file_path, tokenize=True):
    phrases = []
    with open(file_path, 'r') as f:
        # TODO here the ground-truth is already after processing, contains <digit>, not good for baseline methods...
        if tokenize:
            phrase_str = ';'.join([l.strip() for l in f.readlines()])
            phrases = dataset_utils.process_keyphrase(phrase_str)

        else: