<jupyter_start><jupyter_text>Read MeWe recompute the scores based on the released dataset and outputs from [https://github.com/kenchan0226/keyphrase-generation-rl](https://github.com/kenchan0226/keyphrase-generation-rl) and [https://github.com/Chen-Wang-CUHK/KG-KE-KR-M](https://github.com/Chen-Wang-CUHK/KG-KE-KR-M).But we observe a significant difference on number of present/absent ground-truth phrases between our data and theirs. So all the scores here are basically not usable.  TODO - release a present/absent split following Chan's format. - release a script to help simplify and unify the research results.> @inproceedings{conf/acl/chan19keyphraseRL,  title={Neural Keyphrase Generation via Reinforcement Learning with Adaptive Rewards},  author={<NAME> and <NAME> and <NAME> and <NAME>},  booktitle={Proceedings of ACL},  year={2019}}> @inproceedings{chen-etal-2019-integrated,    title = "An Integrated Approach for Keyphrase Generation via Exploring the Power of Retrieval and Extraction",    author = "<NAME>  and      <NAME>  and      <NAME>  and      <NAME>  and      <NAME>",    booktitle = "Proceedings of the 2019 Conference of the North {A}merican Chapter of the Association for Computational Linguistics: Human Language Technologies, Volume 1 (Long and Short Papers)",    month = jun,    year = "2019",    address = "Minneapolis, Minnesota",    publisher = "Association for Computational Linguistics",    url = "https://www.aclweb.org/anthology/N19-1292",    pages = "2846--2856",}<jupyter_code>import os
import sys
import numpy as np
from collections import defaultdict

module_path = os.path.abspath(os.path.join('..'))
if module_path not in sys.path:
    sys.path.append(module_path)
module_path = os.path.abspath(os.path.join('../onmt'))
if module_path not in sys.path:
    sys.path.append(module_path)

import kp_evaluate
from onmt.keyphrase.utils import if_present_duplicate_phrases

def _gather_scores(gathered_scores, results_names, results_dicts):
    for result_name, result_dict in zip(results_names, results_dicts):
        for metric_name, score in result_dict.items():
            if metric_name.endswith('_num'):
                # if it's 'present_tgt_num' or 'absent_tgt_num', leave as is
                field_name = result_name
            else:
                # if it's other score like 'precision@5' is renamed to like 'present_exact_precision@'
                field_name = result_name+'_'+metric_name

            if field_name not in gathered_scores:
                gathered_scores[field_name] = []

            gathered_scores[field_name].append(score)

    return gathered_scores

def _if_duplicate(kp_seqs):
    duplicate_flags = []
    phrase_set = set()  # some phrases are duplicate after stemming, like "model" and "models" would be same after stemming, thus we ignore the following ones

    for kp_seq in kp_seqs:
        # check if it is duplicate
        if '_'.join(kp_seq) in phrase_set:
            duplicate_flags.append(True)
        else:
            duplicate_flags.append(False)
        phrase_set.add('_'.join(kp_seq))
    
    return np.asarray(duplicate_flags)
    
data_base_dir = '/Users/memray/project/kp/OpenNMT-kpg/data/keyphrase/KenChan/kp_datasets/separated/'

dataset_names = ['inspec', 'krapivin', 'nus', 'semeval', 'kp20k']
dataset_names = ['inspec', 'krapivin', 'nus', 'semeval', 'kp20k']
seed_names = ['seed1', 'seed2', 'seed3']

topk_range = [5, 10, 'k', 'M']
# 'precision_hard' and 'f_score_hard' mean that precision is calculated with denominator strictly as K (say 5 or 10), won't be lessened even number of preds is smaller
metric_names = ['correct', 'precision', 'recall', 'f_score', 'precision_hard', 'f_score_hard']

def calc_present_scores(output_base_dir, ignore_nopresent_dp):

    merged_score_dict = defaultdict(lambda: [])

    for seed in seed_names:
        print(seed)
        for dataset in dataset_names:
            print(dataset)
            score_dict = {}

            gt_path = os.path.join(data_base_dir, dataset+'.txt') # ground-truth
            op_path = os.path.join(output_base_dir, seed, dataset+'.txt') # model's output
            with open(gt_path, 'r') as gt_file, open(op_path, 'r') as op_file:
                # check number of lines
    #             print(len(gt_file.readlines()))
    #             print(len(output_file.readlines()))
                for line_id, (gt_line, op_line) in enumerate(zip(gt_file.readlines(), op_file.readlines())):
                    if line_id % 1000 == 0:
                        print(line_id)
                    # split phrases and works in each phrase 
                    gt_phrases = [kp.split() for kp in gt_line.split(';')]
                    op_phrases = [kp.split() for kp in op_line.split(';')]

                    # locate <peos> and truncate the absent part
    #                     print(gt_line)
                    peos_idx = gt_phrases.index(['<peos>']) if ['<peos>'] in gt_phrases else None
    #                     print(pe/os_idx)
                    present_gt_phrases = gt_phrases[: peos_idx]
                    absent_gt_phrases = gt_phrases[peos_idx + 1:]

                    ##########################################
                    # ignore the data points that have no present_gt_phrases
                    ##########################################
                    if len(present_gt_phrases) == 0 and ignore_nopresent_dp:
                        continue

    #                 print('-' * 10)
    #                 print('Model output:')
    #                 print(op_line)
                    peos_idx = op_phrases.index(['<peos>']) if ['<peos>'] in op_phrases else None
                    op_phrases = op_phrases[: peos_idx]
    #                 print(op_phrases)

                    # remove duplicates
                    duplicate_flags = _if_duplicate(op_phrases)
    #                 print(duplicate_flags)
                    op_phrases = [p for p, flag in zip(op_phrases, duplicate_flags) if not flag]
                    match_scores_exact = kp_evaluate.compute_match_scores(tgt_seqs=present_gt_phrases, pred_seqs=op_phrases, type='exact', do_stem=True)
    #                 print(match_scores_exact)
                    present_exact_results = kp_evaluate.run_classic_metrics(match_scores_exact, op_phrases, present_gt_phrases, metric_names, topk_range)
    #                 print(present_exact_results)

                    results_names = ['present_exact']
                    results_list = [present_exact_results]

                    # update score_dict, appending new scores (results_list) to it
                    score_dict = _gather_scores(score_dict, results_names, results_list)
                    # add tgt/pred count for computing average performance on non-empty items
                    results_names = ['tgt_num', 'present_tgt_num', 'absent_tgt_num', 'present_pred_num']
                    results_list = [
                                    {'tgt_num': len(present_gt_phrases)+len(absent_gt_phrases)},
                                    {'present_tgt_num': len(present_gt_phrases)},
                                    {'absent_tgt_num': len(absent_gt_phrases)},
                                    {'present_pred_num': len(op_phrases)},
                                   ]
                    score_dict = _gather_scores(score_dict, results_names, results_list)


    #         print(score_dict)
            for k, v in score_dict.items():
                print('%s, num=%d, mean=%f, sum=%f' % (k, len(v), np.average(v), np.sum(v)))
                merged_score_dict['%s-%s' % (dataset, k)].append(np.average(v))

    for k, v in merged_score_dict.items():
        print('%s=%s, num=%d, mean=%f, sum=%f' % (k, str(v), len(v), np.average(v), np.sum(v)))<jupyter_output><empty_output><jupyter_text>catSeqTG-2RF1<jupyter_code>output_base_dir = '/Users/memray/project/kp/OpenNMT-kpg/data/keyphrase/KenChan/kp_outputs/catSeqTG-2RF1/'

calc_present_scores(output_base_dir, ignore_nopresent_dp=True)<jupyter_output>seed1
inspec
0
present_exact_correct@5, num=496, mean=1.328629, sum=659.000000
present_exact_precision@5, num=496, mean=0.420800, sum=208.716667
present_exact_recall@5, num=496, mean=0.225941, sum=112.066963
present_exact_f_score@5, num=496, mean=0.275797, sum=136.795254
present_exact_precision_hard@5, num=496, mean=0.265726, sum=131.800000
present_exact_f_score_hard@5, num=496, mean=0.224572, sum=111.387637
present_exact_correct@10, num=496, mean=1.334677, sum=662.000000
present_exact_precision@10, num=496, mean=0.421424, sum=209.026190
present_exact_recall@10, num=496, mean=0.227406, sum=112.793153
present_exact_f_score@10, num=496, mean=0.276727, sum=137.256792
present_exact_precision_hard@10, num=496, mean=0.133468, sum=66.200000
present_exact_f_score_hard@10, num=496, mean=0.155695, sum=77.224616
present_exact_correct@k, num=496, mean=1.316532, sum=653.000000
present_exact_precision@k, num=496, mean=0.422547, sum=209.583333
present_exact_recall@k, num=496, mean=0.218501, sum=108.3[...]<jupyter_text>KG-KE-KR-M<jupyter_code>output_base_dir = '/Users/memray/project/kp/OpenNMT-kpg/data/keyphrase/KenChan/kp_outputs/KG-KE-KR-M/'

calc_present_scores(output_base_dir, ignore_nopresent_dp=True)<jupyter_output>seed1
inspec
0
present_exact_correct@5, num=496, mean=1.612903, sum=800.000000
present_exact_precision@5, num=496, mean=0.322581, sum=160.000000
present_exact_recall@5, num=496, mean=0.277941, sum=137.858902
present_exact_f_score@5, num=496, mean=0.275473, sum=136.634658
present_exact_precision_hard@5, num=496, mean=0.322581, sum=160.000000
present_exact_f_score_hard@5, num=496, mean=0.275473, sum=136.634658
present_exact_correct@10, num=496, mean=2.635081, sum=1307.000000
present_exact_precision@10, num=496, mean=0.263508, sum=130.700000
present_exact_recall@10, num=496, mean=0.431018, sum=213.784702
present_exact_f_score@10, num=496, mean=0.303370, sum=150.471610
present_exact_precision_hard@10, num=496, mean=0.263508, sum=130.700000
present_exact_f_score_hard@10, num=496, mean=0.303370, sum=150.471610
present_exact_correct@k, num=496, mean=2.318548, sum=1150.000000
present_exact_precision@k, num=496, mean=0.312482, sum=154.990909
present_exact_recall@k, num=496, mean=0.312482, sum=1[...]
{'fraction_non_alphanumeric': 0.11987934222049236, 'fraction_numerical': 0.06470759949401576, 'mean_word_length': 3.112845138055222, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 27, 'https://': 5, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '340575', 'license': 'MIT', 'n_tokens_mistral': 3951, 'n_tokens_neox': 3348, 'n_words': 693, 'path': 'notebook/KenChan_eval.ipynb', 'repo_name': 'memray/OpenNMT-kpg-release', 'size': '10317'}