# Path: emolga/dataset/build_dataset.py
# def deserialize_from_file(path):
#     f = open(path, 'rb')
#     obj = pickle.load(f)
#     f.close()
#     return obj
# 
# Path: keyphrase/dataset/keyphrase_test_dataset.py
# def load_additional_testing_data(testing_names, idx2word, word2idx, config, postagging=True, process_type=1):
#     test_sets           = {}
# 
#     # rule out the ones appear in testing data
#     for dataset_name in testing_names:
# 
#         if os.path.exists(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl'):
#             test_set = deserialize_from_file(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl')
#             print('Loading testing dataset %s from %s' % (dataset_name, config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl'))
#         else:
#             print('Creating testing dataset %s: %s' % (dataset_name, config['path'] + '/dataset/keyphrase/' + config[
#                 'data_process_name'] + dataset_name + '.testing.pkl'))
#             dataloader          = testing_data_loader(dataset_name, kwargs=dict(basedir=config['path']))
#             records             = dataloader.get_docs()
#             records, pairs, _   = utils.load_pairs(records, process_type=process_type, do_filter=False)
#             test_set            = utils.build_data(pairs, idx2word, word2idx)
# 
#             test_set['record']  = records
# 
#             if postagging:
#                 tagged_sources = get_postag_with_record(records, pairs)
#                 test_set['tagged_source']   = [[t[1] for t in s] for s in tagged_sources]
# 
#                 if hasattr(dataloader, 'text_postag_dir') and dataloader.__getattribute__('text_postag_dir') != None:
#                     print('Exporting postagged data to %s' % (dataloader.text_postag_dir))
#                     if not os.path.exists(dataloader.text_postag_dir):
#                         os.makedirs(dataloader.text_postag_dir)
#                     for r_, p_, s_ in zip(records, pairs, tagged_sources):
#                         with open(dataloader.text_postag_dir+ '/' + r_['name'] + '.txt', 'w') as f:
#                             output_str = ' '.join([w+'_'+t for w,t in s_])
#                             f.write(output_str)
#                 else:
#                     print('text_postag_dir not found, no export of postagged data')
# 
#             serialize_to_file(test_set, config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl')
# 
#         test_sets[dataset_name] = test_set
# 
#     return test_sets


import os
import numpy
import shutil
import keyphrase.config
            import string
    import emolga,string
from emolga.dataset.build_dataset import deserialize_from_file
from keyphrase.dataset.keyphrase_test_dataset import load_additional_testing_data
    from keyphrase.dataset import keyphrase_test_dataset
        with open(test_dir+str(i)+'.txt', 'w') as f:
            f.write(' '.join(test_text))
        with open(test_dir + str(i) + '.key', 'w') as f:
            f.write('\n'.join([' '.join(t)+'\t1' for t in test_target]))

def export_ke20k_testing_maui():
    target_dir = '/Users/memray/Project/seq2seq-keyphrase/dataset/keyphrase/baseline-data/maui/ke20k/'

    config  = keyphrase.config.setup_keyphrase_all()   # load settings.
    doc_list = keyphrase_test_dataset.testing_data_loader('ke20k', kwargs=dict(basedir = config['path'])).get_docs(False)

    for d in doc_list:
        d_id = d.name[:d.name.find('.txt')]
        print(d_id)
        with open(target_dir+d_id+'.txt', 'w') as textfile:
            textfile.write(d.title+'\n'+d.text)
        with open(target_dir + d_id + '.key', 'w') as phrasefile:
            for p in d.phrases:
                phrasefile.write('%s\t1\n' % p)

def export_ke20k_train_maui():
    '''
    just use the validation dataset
    :return:
    '''
    config  = keyphrase.config.setup_keyphrase_all()   # load settings.
    target_dir = '/Users/memray/Project/seq2seq-keyphrase/dataset/keyphrase/baseline-data/maui/ke20k/train/'


    printable = set(string.printable)

# Path: emolga/dataset/build_dataset.py
# def deserialize_from_file(path):
#     f = open(path, 'rb')
#     obj = pickle.load(f)
#     f.close()
#     return obj
# 
# Path: keyphrase/dataset/keyphrase_test_dataset.py
# def load_additional_testing_data(testing_names, idx2word, word2idx, config, postagging=True, process_type=1):
#     test_sets           = {}
# 
#     # rule out the ones appear in testing data
#     for dataset_name in testing_names:
# 
#         if os.path.exists(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl'):
#             test_set = deserialize_from_file(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl')
#             print('Loading testing dataset %s from %s' % (dataset_name, config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl'))
#         else:
#             print('Creating testing dataset %s: %s' % (dataset_name, config['path'] + '/dataset/keyphrase/' + config[
#                 'data_process_name'] + dataset_name + '.testing.pkl'))
#             dataloader          = testing_data_loader(dataset_name, kwargs=dict(basedir=config['path']))
#             records             = dataloader.get_docs()
#             records, pairs, _   = utils.load_pairs(records, process_type=process_type, do_filter=False)
#             test_set            = utils.build_data(pairs, idx2word, word2idx)
# 
#             test_set['record']  = records
# 
#             if postagging:
#                 tagged_sources = get_postag_with_record(records, pairs)
#                 test_set['tagged_source']   = [[t[1] for t in s] for s in tagged_sources]
# 
#                 if hasattr(dataloader, 'text_postag_dir') and dataloader.__getattribute__('text_postag_dir') != None:
#                     print('Exporting postagged data to %s' % (dataloader.text_postag_dir))
#                     if not os.path.exists(dataloader.text_postag_dir):
#                         os.makedirs(dataloader.text_postag_dir)
#                     for r_, p_, s_ in zip(records, pairs, tagged_sources):
#                         with open(dataloader.text_postag_dir+ '/' + r_['name'] + '.txt', 'w') as f:
#                             output_str = ' '.join([w+'_'+t for w,t in s_])
#                             f.write(output_str)
#                 else:
#                     print('text_postag_dir not found, no export of postagged data')
# 
#             serialize_to_file(test_set, config['path'] + '/dataset/keyphrase/'+config['data_process_name']+dataset_name+'.testing.pkl')
# 
#         test_sets[dataset_name] = test_set
# 
#     return test_sets


# Path: keyphrase/baseline/export_dataset.py
import os
import numpy
import shutil
import keyphrase.config
            import string
    import emolga,string
from emolga.dataset.build_dataset import deserialize_from_file
from keyphrase.dataset.keyphrase_test_dataset import load_additional_testing_data
    from keyphrase.dataset import keyphrase_test_dataset

        with open(test_dir+str(i)+'.txt', 'w') as f:
            f.write(' '.join(test_text))
        with open(test_dir + str(i) + '.key', 'w') as f:
            f.write('\n'.join([' '.join(t)+'\t1' for t in test_target]))

def export_ke20k_testing_maui():
    target_dir = '/Users/memray/Project/seq2seq-keyphrase/dataset/keyphrase/baseline-data/maui/ke20k/'

    config  = keyphrase.config.setup_keyphrase_all()   # load settings.
    doc_list = keyphrase_test_dataset.testing_data_loader('ke20k', kwargs=dict(basedir = config['path'])).get_docs(False)

    for d in doc_list:
        d_id = d.name[:d.name.find('.txt')]
        print(d_id)
        with open(target_dir+d_id+'.txt', 'w') as textfile:
            textfile.write(d.title+'\n'+d.text)
        with open(target_dir + d_id + '.key', 'w') as phrasefile:
            for p in d.phrases:
                phrasefile.write('%s\t1\n' % p)

def export_ke20k_train_maui():
    '''
    just use the validation dataset
    :return:
    '''
    config  = keyphrase.config.setup_keyphrase_all()   # load settings.
    target_dir = '/Users/memray/Project/seq2seq-keyphrase/dataset/keyphrase/baseline-data/maui/ke20k/train/'


    printable = set(string.printable)

    validation_records = emolga.dataset.build_dataset.deserialize_from_file(config['path'] + '/dataset/keyphrase/'+config['data_process_name']+'validation_record_'+str(config['validation_size'])+'.pkl')