<jupyter_start><jupyter_text>Evaluating model predictions<jupyter_code>mode = "test"
evaluator = MultiWozEvaluator(mode)
# HIER
model_folder = "../running/external/transformer_hier++_001/"
for x in ['greedy', 'beam_2', 'beam_3', 'beam_5']:
    prediction_json = "{}/model_turns_{}_test.json".format(model_folder, x)
    if os.path.isfile(prediction_json):
        print "\nDecoding Method:", x.upper(), '\n-------------------'
        with open(prediction_json, "r") as f:
            _temp_gen = json.load(f)
        generated_data = {}
        for key, value in _temp_gen.items():
            generated_data[key + '.json'] = value

        # PROVIDE HERE YOUR GENERATED DIALOGUES INSTEAD
        # generated_data = human_proc_data
        # for key, value in human_raw_data.items():
        #     human_proc_data[key] = value['sys'] # Array of system utterances

        _, _, _, all_match_success = evaluator.evaluateModel(generated_data, mode=mode)

        # Match and Success stats
        pred_file = prediction_json.replace("model_turns", "stats").replace("json", "tsv")
        all_match_success = pd.DataFrame.from_records(all_match_success)
        all_match_success.to_csv(pred_file, sep="\t", index=False)
        print("Stats ->", pred_file)
    else:
        print "skip", x.upper(), '\n'<jupyter_output>Decoding Method: GREEDY 
-------------------
test Corpus Matches : 88.80%
test Corpus Success : 84.80%
test Corpus BLEU : 0.00%
Total number of dialogues: 1000 
('Stats ->', '../running/external/transformer_hier++_001//stats_greedy_test.tsv')

Decoding Method: BEAM_2 
-------------------
test Corpus Matches : 88.30%
test Corpus Success : 85.10%
test Corpus BLEU : 0.00%
Total number of dialogues: 1000 
('Stats ->', '../running/external/transformer_hier++_001//stats_beam_2_test.tsv')

Decoding Method: BEAM_3 
-------------------
test Corpus Matches : 88.10%
test Corpus Success : 85.20%
test Corpus BLEU : 0.00%
Total number of dialogues: 1000 
('Stats ->', '../running/external/transformer_hier++_001//stats_beam_3_test.tsv')

Decoding Method: BEAM_5 
-------------------
test Corpus Matches : 87.30%
test Corpus Success : 84.60%
test Corpus BLEU : 0.00%
Total number of dialogues: 1000 
('Stats ->', '../running/external/transformer_hier++_001//stats_beam_5_test.tsv')
{'fraction_non_alphanumeric': 0.16921064108155254, 'fraction_numerical': 0.03663323157435674, 'mean_word_length': 3.089126559714795, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '130592', 'license': 'MIT', 'n_tokens_mistral': 837, 'n_tokens_neox': 759, 'n_words': 171, 'path': 'HIER/new_eval/new_eval.ipynb', 'repo_name': 'bsantraigi/HIER', 'size': '2293'}