<jupyter_start><jupyter_text>Main Code: Compute Bleu Score<jupyter_code>import nltk
class Metrics:
    @staticmethod
    def getBleuScore(listOfGroundTruthStrings, predictionStrings):
        """
        listOfGroundTruthStrings - List of Ground Truth strings Ex: [[ref1a, ref1b], [ref2a, ref2b], [ref3a]]
        predictionStrings - Prediction string - Ex: [pred1, pred2, pred3]
        """
        
        listOfGroundTruthWords = [[groundTruthString.split() for groundTruthString in groundTruthStrings] for groundTruthStrings in listOfGroundTruthStrings]
        predictionWords = list(map(str.split, predictionStrings))
        return nltk.translate.bleu_score.corpus_bleu(listOfGroundTruthWords, predictionWords)

def extractGroundTruthsAndPredictions(prediction_groundtruths):
    GroundTruthListKey = 'GroundTruth'
    PredictionKey = 'Prediction'

    listOfGroundTruthStrings = []
    predictionStrings = []

    for val in prediction_groundtruths.values():
        listOfGroundTruthStrings.append(val[GroundTruthListKey])
        predictionStrings.append(val[PredictionKey])     

    return listOfGroundTruthStrings, predictionStrings<jupyter_output><empty_output><jupyter_text>Features_2D - Bleu Score  Load GroundTruth corresponding Predictions file<jupyter_code>import pickle

prediction_groundtruths = pickle.load( open( "./DATA/Predictions/VideoCaptions_2d/video_groundtruth_predictions.pkl", "rb" ) )<jupyter_output><empty_output><jupyter_text>Extract ground truths and prediction string<jupyter_code>listOfGroundTruthStrings, predictionStrings = extractGroundTruthsAndPredictions(prediction_groundtruths)<jupyter_output><empty_output><jupyter_text>Bleu Score: 2D<jupyter_code>Metrics.getBleuScore(listOfGroundTruthStrings, predictionStrings)<jupyter_output><empty_output><jupyter_text>pickle file to support Python2 version for Meteor scoring<jupyter_code>pickle.dump(prediction_groundtruths, open( "./DATA/Predictions/VideoCaptions_2d/video_groundtruth_predictions_p2.pkl", "wb"), protocol=2)<jupyter_output><empty_output><jupyter_text>------------------------------------  Features_2D_MeanPooling - Bleu Score  Load GroundTruth corresponding Predictions file<jupyter_code>import pickle

prediction_groundtruths = pickle.load( open( "./DATA/Predictions/VideoCaptions_2d_mean/video_groundtruth_predictions.pkl", "rb" ) )<jupyter_output><empty_output><jupyter_text>Extract ground truths and prediction string<jupyter_code>listOfGroundTruthStrings, predictionStrings = extractGroundTruthsAndPredictions(prediction_groundtruths)<jupyter_output><empty_output><jupyter_text>Bleu Score: 2D_MeanPooling<jupyter_code>Metrics.getBleuScore(listOfGroundTruthStrings, predictionStrings)<jupyter_output><empty_output><jupyter_text>pickle file to support Python2 version for Meteor scoring<jupyter_code>pickle.dump(prediction_groundtruths, open( "./DATA/Predictions/VideoCaptions_2d_mean/video_groundtruth_predictions_p2.pkl", "wb"), protocol=2)<jupyter_output><empty_output><jupyter_text>-------  Features_3D - Bleu Score  Load GroundTruth corresponding Predictions file<jupyter_code>import pickle

prediction_groundtruths = pickle.load( open( "./DATA/Predictions/VideoCaptions_3d/video_groundtruth_predictions.pkl", "rb" ) )<jupyter_output><empty_output><jupyter_text>Extract ground truths and prediction string<jupyter_code>listOfGroundTruthStrings, predictionStrings = extractGroundTruthsAndPredictions(prediction_groundtruths)<jupyter_output><empty_output><jupyter_text>Bleu Score: 2D_MeanPooling<jupyter_code>Metrics.getBleuScore(listOfGroundTruthStrings, predictionStrings)<jupyter_output><empty_output><jupyter_text>pickle file to support Python2 version for Meteor scoring<jupyter_code>pickle.dump(prediction_groundtruths, open( "./DATA/Predictions/VideoCaptions_3d/video_groundtruth_predictions_p2.pkl", "wb"), protocol=2)<jupyter_output><empty_output><jupyter_text>------  Tried implementing Meteor  Get Meteor related file from this location "http://www.cs.cmu.edu/~alavie/METEOR/download/meteor-1.5.tar.gz"and then extract tar file and copy meteor-1.5.jar file to the same folder that this notebook is present.<jupyter_code>from meteor import Meteor
meteorObj = Meteor()
meteorObj.compute_score({1:['it is a cat', 'it looks like a cat'], 2: ['it is a chair']},{1:['it is cat'], 2:['it is a chair']})<jupyter_output><empty_output><jupyter_text>IGNORE: This part is just for test<jupyter_code>import nltk

hypothesis = 'it may be a cat'.split()
references = ['it is a cat'.split(),\
              'it is cat'.split(),\
              'it seems a cat'.split()\
             ]
#there may be several references
BLEUscore = nltk.translate.bleu_score.sentence_bleu(references, hypothesis)
print(BLEUscore)<jupyter_output>0.6223329772884784<jupyter_text>Test Bleu@N using NLTK<jupyter_code>import nltk 
import nltk

hypothesis = ['It', 'is', 'a', 'cat', 'at', 'room']
references = [['It', 'is', 'a', 'cat', 'inside', 'the', 'room'], 'it looks like a cat is inside the room'.split(), \
              'it seems to be a cat inside the room'.split(),\
              'there is a cat inside the room'.split()\
             ]
#there may be several references
BLEUscore = nltk.translate.bleu_score.sentence_bleu(reference, hypothesis)
print(BLEUscore)
import nltk
hypothesis =  ['it is a cat'.split(),\
               'dog is running'.split()\
              ]
             
references = [
              ['it is a cat'.split(),\
               'it is cat'.split(),\
               'it seems a cat'.split()\
              ],\
              [
                'dog is running'.split(),\
                'dog runs'.split()
              ]
            ]
#there may be several references
BLEUscore = nltk.translate.bleu_score.corpus_bleu(references, hypothesis)
print(BLEUscore)<jupyter_output>0.8408964152537145
{'fraction_non_alphanumeric': 0.11040303134688254, 'fraction_numerical': 0.012228728901136755, 'mean_word_length': 5.106203995793901, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 63, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '16', 'dir': 'jupyter-structured-clean-dedup', 'id': '576031', 'license': 'MIT', 'n_tokens_mistral': 1977, 'n_tokens_neox': 1882, 'n_words': 352, 'path': 'Metrics-Bleu.ipynb', 'repo_name': 'Curious-Geek/Video-Captioning', 'size': '5806'}