# Path: emolga/dataset/build_dataset.py
# def deserialize_from_file(path):
#     f = open(path, 'rb')
#     obj = pickle.load(f)
#     f.close()
#     return obj
# 
# Path: keyphrase/dataset/keyphrase_test_dataset.py
# def load_additional_testing_data(testing_names, idx2word, word2idx, config, postagging=True, process_type=1):
#     test_sets           = {}
# 
#     # rule out the ones appear in testing data
#     for dataset_name in testing_names:
# 
#         if os.path.exists(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl'):
#             test_set = deserialize_from_file(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl')
#             print('Loading testing dataset %s from %s' % (dataset_name, config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl'))
#         else:
#             print('Creating testing dataset %s: %s' % (dataset_name, config['path'] + '/dataset/keyphrase/' + config[
#                 'data_process_name'] + dataset_name + '.testing.pkl'))
#             dataloader          = testing_data_loader(dataset_name, kwargs=dict(basedir=config['path']))
#             records             = dataloader.get_docs()
#             records, pairs, _   = utils.load_pairs(records, process_type=process_type, do_filter=False)
#             test_set            = utils.build_data(pairs, idx2word, word2idx)
# 
#             test_set['record']  = records
# 
#             if postagging:
#                 tagged_sources = get_postag_with_record(records, pairs)
#                 test_set['tagged_source']   = [[t[1] for t in s] for s in tagged_sources]
# 
#                 if hasattr(dataloader, 'text_postag_dir') and dataloader.__getattribute__('text_postag_dir') != None:
#                     print('Exporting postagged data to %s' % (dataloader.text_postag_dir))
#                     if not os.path.exists(dataloader.text_postag_dir):
#                         os.makedirs(dataloader.text_postag_dir)
#                     for r_, p_, s_ in zip(records, pairs, tagged_sources):
#                         with open(dataloader.text_postag_dir+ '/' + r_['name'] + '.txt', 'w') as f:
#                             output_str = ' '.join([w+'_'+t for w,t in s_])
#                             f.write(output_str)
#                 else:
#                     print('text_postag_dir not found, no export of postagged data')
# 
#             serialize_to_file(test_set, config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl')
# 
#         test_sets[dataset_name] = test_set
# 
#     return test_sets


import os
import numpy
import shutil
import keyphrase.config
            import string
    import emolga,string
from emolga.dataset.build_dataset import deserialize_from_file
from keyphrase.dataset.keyphrase_test_dataset import load_additional_testing_data
    from keyphrase.dataset import keyphrase_test_dataset
        if not os.path.exists(text_dir):
            os.makedirs(text_dir)
        if target_dir!="" and not os.path.exists(target_dir):
            os.makedirs(target_dir)
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        for i, file_name in enumerate(file_names):
            print('Exporting %d. %s - %s' % (i, dataset_name, file_name))
            print('Text file %s' % (text_dir + str(i) + '.txt'))
            with open(text_dir + file_name + '.txt', 'r') as inf:
                text = inf.read()
                with open(output_dir + str(i) + '.txt', 'w') as outf:
                    outf.write(text)

            print('Target file %s' % (target_dir + file_name + '.keyphrases'))
            targets = []
            if target_dir.strip() != '':
                with open(target_dir + file_name + '.keyphrases', 'r') as inf:
                    targets.extend([l.strip() for l in inf.readlines()])
                with open(target_dir + file_name + '.keywords', 'r') as inf:
                    targets.extend([l.strip() for l in inf.readlines()])
                with open(output_dir + str(i) + '.key', 'w') as outf:
                    outf.write('\n'.join([t + '\t1' for t in targets]))

def export_krapivin_maui():
    # prepare logging.
    config  = keyphrase.config.setup_keyphrase_all()   # load settings.

    train_set, validation_set, test_sets, idx2word, word2idx = deserialize_from_file(config['dataset'])

# Path: emolga/dataset/build_dataset.py
# def deserialize_from_file(path):
#     f = open(path, 'rb')
#     obj = pickle.load(f)
#     f.close()
#     return obj
# 
# Path: keyphrase/dataset/keyphrase_test_dataset.py
# def load_additional_testing_data(testing_names, idx2word, word2idx, config, postagging=True, process_type=1):
#     test_sets           = {}
# 
#     # rule out the ones appear in testing data
#     for dataset_name in testing_names:
# 
#         if os.path.exists(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl'):
#             test_set = deserialize_from_file(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl')
#             print('Loading testing dataset %s from %s' % (dataset_name, config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl'))
#         else:
#             print('Creating testing dataset %s: %s' % (dataset_name, config['path'] + '/dataset/keyphrase/' + config[
#                 'data_process_name'] + dataset_name + '.testing.pkl'))
#             dataloader          = testing_data_loader(dataset_name, kwargs=dict(basedir=config['path']))
#             records             = dataloader.get_docs()
#             records, pairs, _   = utils.load_pairs(records, process_type=process_type, do_filter=False)
#             test_set            = utils.build_data(pairs, idx2word, word2idx)
# 
#             test_set['record']  = records
# 
#             if postagging:
#                 tagged_sources = get_postag_with_record(records, pairs)
#                 test_set['tagged_source']   = [[t[1] for t in s] for s in tagged_sources]
# 
#                 if hasattr(dataloader, 'text_postag_dir') and dataloader.__getattribute__('text_postag_dir') != None:
#                     print('Exporting postagged data to %s' % (dataloader.text_postag_dir))
#                     if not os.path.exists(dataloader.text_postag_dir):
#                         os.makedirs(dataloader.text_postag_dir)
#                     for r_, p_, s_ in zip(records, pairs, tagged_sources):
#                         with open(dataloader.text_postag_dir+ '/' + r_['name'] + '.txt', 'w') as f:
#                             output_str = ' '.join([w+'_'+t for w,t in s_])
#                             f.write(output_str)
#                 else:
#                     print('text_postag_dir not found, no export of postagged data')
# 
#             serialize_to_file(test_set, config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl')
# 
#         test_sets[dataset_name] = test_set
# 
#     return test_sets


# Path: keyphrase/baseline/export_dataset.py
import os
import numpy
import shutil
import keyphrase.config
            import string
    import emolga,string
from emolga.dataset.build_dataset import deserialize_from_file
from keyphrase.dataset.keyphrase_test_dataset import load_additional_testing_data
    from keyphrase.dataset import keyphrase_test_dataset

        if not os.path.exists(text_dir):
            os.makedirs(text_dir)
        if target_dir!="" and not os.path.exists(target_dir):
            os.makedirs(target_dir)
        if not os.path.exists(output_dir):
            os.makedirs(output_dir)

        for i, file_name in enumerate(file_names):
            print('Exporting %d. %s - %s' % (i, dataset_name, file_name))
            print('Text file %s' % (text_dir + str(i) + '.txt'))
            with open(text_dir + file_name + '.txt', 'r') as inf:
                text = inf.read()
                with open(output_dir + str(i) + '.txt', 'w') as outf:
                    outf.write(text)

            print('Target file %s' % (target_dir + file_name + '.keyphrases'))
            targets = []
            if target_dir.strip() != '':
                with open(target_dir + file_name + '.keyphrases', 'r') as inf:
                    targets.extend([l.strip() for l in inf.readlines()])
                with open(target_dir + file_name + '.keywords', 'r') as inf:
                    targets.extend([l.strip() for l in inf.readlines()])
                with open(output_dir + str(i) + '.key', 'w') as outf:
                    outf.write('\n'.join([t + '\t1' for t in targets]))

def export_krapivin_maui():
    # prepare logging.
    config  = keyphrase.config.setup_keyphrase_all()   # load settings.

    train_set, validation_set, test_sets, idx2word, word2idx = deserialize_from_file(config['dataset'])

    test_sets = load_additional_testing_data(config['testing_datasets'], idx2word, word2idx, config)