|
import re |
|
import os |
|
|
|
def convert_text(text): |
|
|
|
text = text.lower() |
|
text = ' '.join(re.split('(\W)', text)) |
|
text = ' '.join(text.split()) |
|
return text |
|
|
|
def eval_meteor_test_webnlg(folder_data, pred_file, dataset): |
|
|
|
dir_path = os.path.dirname(os.path.realpath(__file__)) |
|
folder_data_before = dir_path + "/../utils" |
|
|
|
cmd_string = "java -jar " + folder_data_before + "/meteor-1.5.jar " + pred_file + " " \ |
|
+ folder_data + "/" + dataset + ".target_eval_meteor -l en -norm -r 3 > " + pred_file.replace("txt", "meteor") |
|
|
|
os.system(cmd_string) |
|
|
|
meteor_info = open(pred_file.replace("txt", "meteor"), 'r').readlines()[-1].strip() |
|
|
|
return meteor_info |
|
|
|
|
|
def eval_chrf_test_webnlg(folder_data, pred_file, dataset): |
|
|
|
dir_path = os.path.dirname(os.path.realpath(__file__)) |
|
folder_data_before = dir_path + "/../utils" |
|
|
|
cmd_string = "python " + folder_data_before + "/chrf++.py -H " + pred_file + " -R " \ |
|
+ folder_data + "/" + dataset + ".target_eval_crf > " + pred_file.replace("txt", "chrf") |
|
|
|
os.system(cmd_string) |
|
|
|
chrf_info_1 = open(pred_file.replace("txt", "chrf"), 'r').readlines()[1].strip() |
|
chrf_info_2 = open(pred_file.replace("txt", "chrf"), 'r').readlines()[2].strip() |
|
|
|
return chrf_info_1 + " " + chrf_info_2 |
|
|
|
def eval_bleu(folder_data, pred_file, dataset): |
|
|
|
dir_path = os.path.dirname(os.path.realpath(__file__)) |
|
folder_data_before = dir_path + "/data/" |
|
|
|
cmd_string = "perl " + folder_data_before + "/multi-bleu.perl -lc " + folder_data + "/" + dataset + ".target_eval " \ |
|
+ folder_data + "/" + dataset + ".target2_eval " + folder_data + "/" + dataset + ".target3_eval < " \ |
|
+ pred_file + " > " + pred_file.replace("txt", "bleu") |
|
|
|
os.system(cmd_string) |
|
|
|
try: |
|
bleu_info = open(pred_file.replace("txt", "bleu"), 'r').readlines()[0].strip() |
|
except: |
|
bleu_info = -1 |
|
|
|
return bleu_info |
|
|
|
|
|
def eval_bleu_sents_tok(pred_file, folder_data, dataset): |
|
|
|
dir_path = os.path.dirname(os.path.realpath(__file__)) |
|
folder_data_before = dir_path + "/../utils" |
|
|
|
cmd_string = "perl " + folder_data_before + "/tokenizer.perl -threads 4 -no-escape < " + pred_file + " > " +\ |
|
pred_file + "_tok" |
|
os.system(cmd_string) |
|
|
|
cmd_string = "perl " + folder_data_before + "/multi-bleu.perl -lc " + folder_data + "/" + dataset + ".target.tok"\ |
|
+ " < " + pred_file + "_tok" + " > " + pred_file.replace("txt", "bleu_data") |
|
os.system(cmd_string) |
|
|
|
try: |
|
bleu_info_data = open(pred_file.replace("txt", "bleu_data"), 'r').readlines()[0].strip() |
|
except: |
|
bleu_info_data = 'no data' |
|
|
|
return bleu_info_data |
|
|
|
|
|
def eval_meteor(ref_file, pred_file): |
|
|
|
dir_path = os.path.dirname(os.path.realpath(__file__)) |
|
folder_data_before = dir_path + "/../utils" |
|
|
|
cmd_string = "java -jar " + folder_data_before + "/meteor-1.5.jar " + pred_file + " " \ |
|
+ ref_file + " > " + pred_file.replace("txt", "meteor") |
|
|
|
os.system(cmd_string) |
|
|
|
meteor_info = open(pred_file.replace("txt", "meteor"), 'r').readlines()[-1].strip() |
|
|
|
return meteor_info |
|
|
|
|
|
def eval_chrf(ref_file, pred_file): |
|
|
|
dir_path = os.path.dirname(os.path.realpath(__file__)) |
|
folder_data_before = dir_path + "/../utils" |
|
|
|
cmd_string = "python " + folder_data_before + "/chrf++.py -H " + pred_file + " -R " \ |
|
+ ref_file + " > " + pred_file.replace("txt", "chrf") |
|
|
|
os.system(cmd_string) |
|
|
|
try: |
|
chrf_info_1 = open(pred_file.replace("txt", "chrf"), 'r').readlines()[1].strip() |
|
chrf_info_2 = open(pred_file.replace("txt", "chrf"), 'r').readlines()[2].strip() |
|
chrf_data = chrf_info_1 + " " + chrf_info_2 |
|
except: |
|
chrf_data = "no data" |
|
|
|
|
|
return chrf_data |
