natural language toolkit machine translation c 20012023 nltk project uday krishna udaykrishna5gmail com contributor tom aarsen url https www nltk org for license information see license txt takes in pretokenized inputs for hypothesis and reference and returns enumerated word lists for each of them param hypothesis pretokenized hypothesis param reference pretokenized reference preprocess preprocessing method default str lower return enumerated words list matches exact words in hypothesis and reference and returns a word mapping based on the enumerated word id between hypothesis and reference param hypothesis pretokenized hypothesis param reference pretokenized reference return enumerated matched tuples enumerated unmatched hypothesis tuples enumerated unmatched reference tuples matches exact words in hypothesis and reference and returns a word mapping between enumhypothesislist and enumreferencelist based on the enumerated word id param enumhypothesislist enumerated hypothesis list param enumreferencelist enumerated reference list return enumerated matched tuples enumerated unmatched hypothesis tuples enumerated unmatched reference tuples stems each word and matches them in hypothesis and reference and returns a word mapping between enumhypothesislist and enumreferencelist based on the enumerated word id the function also returns a enumerated list of unmatched words for hypothesis and reference param enumhypothesislist enumerated hypothesis list param enumreferencelist enumerated reference list param stemmer nltk stem api stemmeri object default porterstemmer return enumerated matched tuples enumerated unmatched hypothesis tuples enumerated unmatched reference tuples stems each word and matches them in hypothesis and reference and returns a word mapping between hypothesis and reference param hypothesis pretokenized hypothesis param reference pretokenized reference param stemmer nltk stem api stemmeri object default porterstemmer return enumerated matched tuples enumerated unmatched hypothesis tuples enumerated unmatched reference tuples matches each word in reference to a word in hypothesis if any synonym of a hypothesis word is the exact match to the reference word param enumhypothesislist enumerated hypothesis list param enumreferencelist enumerated reference list param wordnet a wordnet corpus reader object default nltk corpus wordnet matches each word in reference to a word in hypothesis if any synonym of a hypothesis word is the exact match to the reference word param hypothesis pretokenized hypothesis param reference pretokenized reference param wordnet a wordnet corpus reader object default nltk corpus wordnet return list of mapped tuples alignsmatches words in the hypothesis to reference by sequentially applying exact match stemmed match and wordnet based synonym match in case there are multiple matches the match which has the least number of crossing is chosen takes enumerated list as input instead of string input param enumhypothesislist enumerated hypothesis list param enumreferencelist enumerated reference list param stemmer nltk stem api stemmeri object default porterstemmer param wordnet a wordnet corpus reader object default nltk corpus wordnet return sorted list of matched tuples unmatched hypothesis list unmatched reference list alignsmatches words in the hypothesis to reference by sequentially applying exact match stemmed match and wordnet based synonym match in case there are multiple matches the match which has the least number of crossing is chosen param hypothesis pretokenized hypothesis param reference pretokenized reference param stemmer nltk stem api stemmeri object default porterstemmer param wordnet a wordnet corpus reader object default nltk corpus wordnet return sorted list of matched tuples unmatched hypothesis list unmatched reference list counts the fewest possible number of chunks such that matched unigrams of each chunk are adjacent to each other this is used to calculate the fragmentation part of the metric param matches list containing a mapping of matched words output of alignwords return number of chunks a sentence is divided into post alignment calculates meteor score for single hypothesis and reference as per meteor an automatic metric for mt evaluation with highlevels of correlation with human judgments by alon lavie and abhaya agarwal in proceedings of acl https www cs cmu edualaviemeteorpdflavieagarwal2007meteor pdf hypothesis1 it is a guide to action which ensures that the military always obeys the commands of the party reference1 it is a guide to action that ensures that the military will forever heed party commands roundsinglemeteorscorereference1 hypothesis1 4 0 6944 if there is no words match during the alignment the method returns the score as 0 we can safely return a zero instead of raising a division by zero error as no match usually implies a bad translation roundsinglemeteorscore this is a cat non matching hypothesis 4 0 0 param reference pretokenized reference param hypothesis pretokenized hypothesis param preprocess preprocessing function default str lower param stemmer nltk stem api stemmeri object default porterstemmer param wordnet a wordnet corpus reader object default nltk corpus wordnet param alpha parameter for controlling relative weights of precision and recall param beta parameter for controlling shape of penalty as a function of as a function of fragmentation param gamma relative weight assigned to fragmentation penalty return the sentencelevel meteor score calculates meteor score for hypothesis with multiple references as described in meteor an automatic metric for mt evaluation with highlevels of correlation with human judgments by alon lavie and abhaya agarwal in proceedings of acl https www cs cmu edualaviemeteorpdflavieagarwal2007meteor pdf in case of multiple references the best score is chosen this method iterates over singlemeteorscore and picks the best pair among all the references for a given hypothesis hypothesis1 it is a guide to action which ensures that the military always obeys the commands of the party hypothesis2 it is to insure the troops forever hearing the activity guidebook that party direct reference1 it is a guide to action that ensures that the military will forever heed party commands reference2 it is the guiding principle which guarantees the military forces always being under the command of the party reference3 it is the practical guide for the army always to heed the directions of the party roundmeteorscorereference1 reference2 reference3 hypothesis1 4 0 6944 if there is no words match during the alignment the method returns the score as 0 we can safely return a zero instead of raising a division by zero error as no match usually implies a bad translation roundmeteorscore this is a cat non matching hypothesis 4 0 0 param references pretokenized reference sentences param hypothesis a pretokenized hypothesis sentence param preprocess preprocessing function default str lower param stemmer nltk stem api stemmeri object default porterstemmer param wordnet a wordnet corpus reader object default nltk corpus wordnet param alpha parameter for controlling relative weights of precision and recall param beta parameter for controlling shape of penalty as a function of as a function of fragmentation param gamma relative weight assigned to fragmentation penalty return the sentencelevel meteor score natural language toolkit machine translation c 2001 2023 nltk project uday krishna udaykrishna5 gmail com contributor tom aarsen url https www nltk org for license information see license txt takes in pre tokenized inputs for hypothesis and reference and returns enumerated word lists for each of them param hypothesis pre tokenized hypothesis param reference pre tokenized reference preprocess preprocessing method default str lower return enumerated words list matches exact words in hypothesis and reference and returns a word mapping based on the enumerated word id between hypothesis and reference param hypothesis pre tokenized hypothesis param reference pre tokenized reference return enumerated matched tuples enumerated unmatched hypothesis tuples enumerated unmatched reference tuples matches exact words in hypothesis and reference and returns a word mapping between enum_hypothesis_list and enum_reference_list based on the enumerated word id param enum_hypothesis_list enumerated hypothesis list param enum_reference_list enumerated reference list return enumerated matched tuples enumerated unmatched hypothesis tuples enumerated unmatched reference tuples stems each word and matches them in hypothesis and reference and returns a word mapping between enum_hypothesis_list and enum_reference_list based on the enumerated word id the function also returns a enumerated list of unmatched words for hypothesis and reference param enum_hypothesis_list enumerated hypothesis list param enum_reference_list enumerated reference list param stemmer nltk stem api stemmeri object default porterstemmer return enumerated matched tuples enumerated unmatched hypothesis tuples enumerated unmatched reference tuples stems each word and matches them in hypothesis and reference and returns a word mapping between hypothesis and reference param hypothesis pre tokenized hypothesis param reference pre tokenized reference param stemmer nltk stem api stemmeri object default porterstemmer return enumerated matched tuples enumerated unmatched hypothesis tuples enumerated unmatched reference tuples matches each word in reference to a word in hypothesis if any synonym of a hypothesis word is the exact match to the reference word param enum_hypothesis_list enumerated hypothesis list param enum_reference_list enumerated reference list param wordnet a wordnet corpus reader object default nltk corpus wordnet matches each word in reference to a word in hypothesis if any synonym of a hypothesis word is the exact match to the reference word param hypothesis pre tokenized hypothesis param reference pre tokenized reference param wordnet a wordnet corpus reader object default nltk corpus wordnet return list of mapped tuples aligns matches words in the hypothesis to reference by sequentially applying exact match stemmed match and wordnet based synonym match in case there are multiple matches the match which has the least number of crossing is chosen takes enumerated list as input instead of string input param enum_hypothesis_list enumerated hypothesis list param enum_reference_list enumerated reference list param stemmer nltk stem api stemmeri object default porterstemmer param wordnet a wordnet corpus reader object default nltk corpus wordnet return sorted list of matched tuples unmatched hypothesis list unmatched reference list aligns matches words in the hypothesis to reference by sequentially applying exact match stemmed match and wordnet based synonym match in case there are multiple matches the match which has the least number of crossing is chosen param hypothesis pre tokenized hypothesis param reference pre tokenized reference param stemmer nltk stem api stemmeri object default porterstemmer param wordnet a wordnet corpus reader object default nltk corpus wordnet return sorted list of matched tuples unmatched hypothesis list unmatched reference list counts the fewest possible number of chunks such that matched unigrams of each chunk are adjacent to each other this is used to calculate the fragmentation part of the metric param matches list containing a mapping of matched words output of align_words return number of chunks a sentence is divided into post alignment calculates meteor score for single hypothesis and reference as per meteor an automatic metric for mt evaluation with highlevels of correlation with human judgments by alon lavie and abhaya agarwal in proceedings of acl https www cs cmu edu alavie meteor pdf lavie agarwal 2007 meteor pdf hypothesis1 it is a guide to action which ensures that the military always obeys the commands of the party reference1 it is a guide to action that ensures that the military will forever heed party commands round single_meteor_score reference1 hypothesis1 4 0 6944 if there is no words match during the alignment the method returns the score as 0 we can safely return a zero instead of raising a division by zero error as no match usually implies a bad translation round single_meteor_score this is a cat non matching hypothesis 4 0 0 param reference pre tokenized reference param hypothesis pre tokenized hypothesis param preprocess preprocessing function default str lower param stemmer nltk stem api stemmeri object default porterstemmer param wordnet a wordnet corpus reader object default nltk corpus wordnet param alpha parameter for controlling relative weights of precision and recall param beta parameter for controlling shape of penalty as a function of as a function of fragmentation param gamma relative weight assigned to fragmentation penalty return the sentence level meteor score calculates meteor score for hypothesis with multiple references as described in meteor an automatic metric for mt evaluation with highlevels of correlation with human judgments by alon lavie and abhaya agarwal in proceedings of acl https www cs cmu edu alavie meteor pdf lavie agarwal 2007 meteor pdf in case of multiple references the best score is chosen this method iterates over single_meteor_score and picks the best pair among all the references for a given hypothesis hypothesis1 it is a guide to action which ensures that the military always obeys the commands of the party hypothesis2 it is to insure the troops forever hearing the activity guidebook that party direct reference1 it is a guide to action that ensures that the military will forever heed party commands reference2 it is the guiding principle which guarantees the military forces always being under the command of the party reference3 it is the practical guide for the army always to heed the directions of the party round meteor_score reference1 reference2 reference3 hypothesis1 4 0 6944 if there is no words match during the alignment the method returns the score as 0 we can safely return a zero instead of raising a division by zero error as no match usually implies a bad translation round meteor_score this is a cat non matching hypothesis 4 0 0 param references pre tokenized reference sentences param hypothesis a pre tokenized hypothesis sentence param preprocess preprocessing function default str lower param stemmer nltk stem api stemmeri object default porterstemmer param wordnet a wordnet corpus reader object default nltk corpus wordnet param alpha parameter for controlling relative weights of precision and recall param beta parameter for controlling shape of penalty as a function of as a function of fragmentation param gamma relative weight assigned to fragmentation penalty return the sentence level meteor score
from itertools import chain, product
from typing import Callable, Iterable, List, Tuple

from nltk.corpus import WordNetCorpusReader, wordnet
from nltk.stem.api import StemmerI
from nltk.stem.porter import PorterStemmer


def _generate_enums(
    hypothesis: Iterable[str],
    reference: Iterable[str],
    preprocess: Callable[[str], str] = str.lower,
) -> Tuple[List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    if isinstance(hypothesis, str):
        raise TypeError(
            f'"hypothesis" expects pre-tokenized hypothesis (Iterable[str]): {hypothesis}'
        )

    if isinstance(reference, str):
        raise TypeError(
            f'"reference" expects pre-tokenized reference (Iterable[str]): {reference}'
        )

    enum_hypothesis_list = list(enumerate(map(preprocess, hypothesis)))
    enum_reference_list = list(enumerate(map(preprocess, reference)))
    return enum_hypothesis_list, enum_reference_list


def exact_match(
    hypothesis: Iterable[str], reference: Iterable[str]
) -> Tuple[List[Tuple[int, int]], List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference)
    return _match_enums(enum_hypothesis_list, enum_reference_list)


def _match_enums(
    enum_hypothesis_list: List[Tuple[int, str]],
    enum_reference_list: List[Tuple[int, str]],
) -> Tuple[List[Tuple[int, int]], List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    word_match = []
    for i in range(len(enum_hypothesis_list))[::-1]:
        for j in range(len(enum_reference_list))[::-1]:
            if enum_hypothesis_list[i][1] == enum_reference_list[j][1]:
                word_match.append(
                    (enum_hypothesis_list[i][0], enum_reference_list[j][0])
                )
                enum_hypothesis_list.pop(i)
                enum_reference_list.pop(j)
                break
    return word_match, enum_hypothesis_list, enum_reference_list


def _enum_stem_match(
    enum_hypothesis_list: List[Tuple[int, str]],
    enum_reference_list: List[Tuple[int, str]],
    stemmer: StemmerI = PorterStemmer(),
) -> Tuple[List[Tuple[int, int]], List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    stemmed_enum_hypothesis_list = [
        (word_pair[0], stemmer.stem(word_pair[1])) for word_pair in enum_hypothesis_list
    ]

    stemmed_enum_reference_list = [
        (word_pair[0], stemmer.stem(word_pair[1])) for word_pair in enum_reference_list
    ]

    return _match_enums(stemmed_enum_hypothesis_list, stemmed_enum_reference_list)


def stem_match(
    hypothesis: Iterable[str],
    reference: Iterable[str],
    stemmer: StemmerI = PorterStemmer(),
) -> Tuple[List[Tuple[int, int]], List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference)
    return _enum_stem_match(enum_hypothesis_list, enum_reference_list, stemmer=stemmer)


def _enum_wordnetsyn_match(
    enum_hypothesis_list: List[Tuple[int, str]],
    enum_reference_list: List[Tuple[int, str]],
    wordnet: WordNetCorpusReader = wordnet,
) -> Tuple[List[Tuple[int, int]], List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    word_match = []
    for i in range(len(enum_hypothesis_list))[::-1]:
        hypothesis_syns = set(
            chain.from_iterable(
                (
                    lemma.name()
                    for lemma in synset.lemmas()
                    if lemma.name().find("_") < 0
                )
                for synset in wordnet.synsets(enum_hypothesis_list[i][1])
            )
        ).union({enum_hypothesis_list[i][1]})
        for j in range(len(enum_reference_list))[::-1]:
            if enum_reference_list[j][1] in hypothesis_syns:
                word_match.append(
                    (enum_hypothesis_list[i][0], enum_reference_list[j][0])
                )
                enum_hypothesis_list.pop(i)
                enum_reference_list.pop(j)
                break
    return word_match, enum_hypothesis_list, enum_reference_list


def wordnetsyn_match(
    hypothesis: Iterable[str],
    reference: Iterable[str],
    wordnet: WordNetCorpusReader = wordnet,
) -> Tuple[List[Tuple[int, int]], List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference)
    return _enum_wordnetsyn_match(
        enum_hypothesis_list, enum_reference_list, wordnet=wordnet
    )


def _enum_align_words(
    enum_hypothesis_list: List[Tuple[int, str]],
    enum_reference_list: List[Tuple[int, str]],
    stemmer: StemmerI = PorterStemmer(),
    wordnet: WordNetCorpusReader = wordnet,
) -> Tuple[List[Tuple[int, int]], List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    exact_matches, enum_hypothesis_list, enum_reference_list = _match_enums(
        enum_hypothesis_list, enum_reference_list
    )

    stem_matches, enum_hypothesis_list, enum_reference_list = _enum_stem_match(
        enum_hypothesis_list, enum_reference_list, stemmer=stemmer
    )

    wns_matches, enum_hypothesis_list, enum_reference_list = _enum_wordnetsyn_match(
        enum_hypothesis_list, enum_reference_list, wordnet=wordnet
    )

    return (
        sorted(
            exact_matches + stem_matches + wns_matches, key=lambda wordpair: wordpair[0]
        ),
        enum_hypothesis_list,
        enum_reference_list,
    )


def align_words(
    hypothesis: Iterable[str],
    reference: Iterable[str],
    stemmer: StemmerI = PorterStemmer(),
    wordnet: WordNetCorpusReader = wordnet,
) -> Tuple[List[Tuple[int, int]], List[Tuple[int, str]], List[Tuple[int, str]]]:
    
    enum_hypothesis_list, enum_reference_list = _generate_enums(hypothesis, reference)
    return _enum_align_words(
        enum_hypothesis_list, enum_reference_list, stemmer=stemmer, wordnet=wordnet
    )


def _count_chunks(matches: List[Tuple[int, int]]) -> int:
    
    i = 0
    chunks = 1
    while i < len(matches) - 1:
        if (matches[i + 1][0] == matches[i][0] + 1) and (
            matches[i + 1][1] == matches[i][1] + 1
        ):
            i += 1
            continue
        i += 1
        chunks += 1
    return chunks


def single_meteor_score(
    reference: Iterable[str],
    hypothesis: Iterable[str],
    preprocess: Callable[[str], str] = str.lower,
    stemmer: StemmerI = PorterStemmer(),
    wordnet: WordNetCorpusReader = wordnet,
    alpha: float = 0.9,
    beta: float = 3.0,
    gamma: float = 0.5,
) -> float:
    
    enum_hypothesis, enum_reference = _generate_enums(
        hypothesis, reference, preprocess=preprocess
    )
    translation_length = len(enum_hypothesis)
    reference_length = len(enum_reference)
    matches, _, _ = _enum_align_words(
        enum_hypothesis, enum_reference, stemmer=stemmer, wordnet=wordnet
    )
    matches_count = len(matches)
    try:
        precision = float(matches_count) / translation_length
        recall = float(matches_count) / reference_length
        fmean = (precision * recall) / (alpha * precision + (1 - alpha) * recall)
        chunk_count = float(_count_chunks(matches))
        frag_frac = chunk_count / matches_count
    except ZeroDivisionError:
        return 0.0
    penalty = gamma * frag_frac**beta
    return (1 - penalty) * fmean


def meteor_score(
    references: Iterable[Iterable[str]],
    hypothesis: Iterable[str],
    preprocess: Callable[[str], str] = str.lower,
    stemmer: StemmerI = PorterStemmer(),
    wordnet: WordNetCorpusReader = wordnet,
    alpha: float = 0.9,
    beta: float = 3.0,
    gamma: float = 0.5,
) -> float:
    
    return max(
        single_meteor_score(
            reference,
            hypothesis,
            preprocess=preprocess,
            stemmer=stemmer,
            wordnet=wordnet,
            alpha=alpha,
            beta=beta,
            gamma=gamma,
        )
        for reference in references
    )