natural language toolkit chrf score c 20012023 nltk project s maja popovic contributors liling tan ale tamchyna memsource url https www nltk org for license information see license txt chrf score implementation import re from collections import counter defaultdict from nltk util import ngrams def sentencechrf reference hypothesis minlen1 maxlen6 beta3 0 ignorewhitespacetrue return corpuschrf reference hypothesis minlen maxlen betabeta ignorewhitespaceignorewhitespace def preprocesssent ignorewhitespace if typesent str turn list of tokens into a string sent joinsent if ignorewhitespace sent re subrs sent return sent def chrfprecisionrecallfscoresupport reference hypothesis n beta3 0 epsilon1e16 refngrams counterngramsreference n hypngrams counterngramshypothesis n calculate the number of ngram matches overlapngrams refngrams hypngrams tp sumoverlapngrams values true positives tpfp sumhypngrams values true positives false positives tpfn sumrefngrams values true positives false negatives try prec tp tpfp precision rec tp tpfn recall factor beta2 fscore 1 factor prec rec factor prec rec except zerodivisionerror prec rec fscore epsilon return prec rec fscore tp def corpuschrf references hypotheses minlen1 maxlen6 beta3 0 ignorewhitespacetrue assert lenreferences len hypotheses the number of hypotheses and their references should be the same numsents lenhypotheses keep fscores for each ngram order separate ngramfscores defaultdictlambda list iterate through each hypothesis and their corresponding references for reference hypothesis in zipreferences hypotheses preprocess both reference and hypothesis reference preprocessreference ignorewhitespace hypothesis preprocesshypothesis ignorewhitespace calculate fscores for each sentence and for each ngram order separately for n in rangeminlen maxlen 1 compute the precision recall fscore and support prec rec fscore tp chrfprecisionrecallfscoresupport reference hypothesis n betabeta ngramfscoresn appendfscore how many ngram sizes numngramsizes lenngramfscores sum of fscores over all sentences for each ngram order totalscores sumfscores for n fscores in ngramfscores items macroaverage over ngram orders and over all sentences return sumtotalscores numngramsizes numsents natural language toolkit chrf score c 2001 2023 nltk project s maja popovic contributors liling tan aleš tamchyna memsource url https www nltk org for license information see license txt chrf score implementation calculates the sentence level chrf character n gram f score described in maja popovic 2015 chrf character n gram f score for automatic mt evaluation in proceedings of the 10th workshop on machine translation https www statmt org wmt15 pdf wmt49 pdf maja popovic 2016 chrf deconstructed β parameters and n gram weights in proceedings of the 1st conference on machine translation https www statmt org wmt16 pdf w16 2341 pdf this implementation of chrf only supports a single reference at the moment for details not reported in the paper consult maja popovic s original implementation https github com m popovic chrf the code should output results equivalent to running chrf with the following options nw 0 b 3 an example from the original bleu paper https www aclweb org anthology p02 1040 pdf ref1 str it is a guide to action that ensures that the military will forever heed party commands split hyp1 str it is a guide to action which ensures that the military always obeys the commands of the party split hyp2 str it is to insure the troops forever hearing the activity guidebook that party direct split sentence_chrf ref1 hyp1 doctest ellipsis 0 6349 sentence_chrf ref1 hyp2 doctest ellipsis 0 3330 the infamous the the the example ref the cat is on the mat split hyp the the the the the the the split sentence_chrf ref hyp doctest ellipsis 0 1468 an example to show that this function allows users to use strings instead of tokens i e list str as inputs ref1 str it is a guide to action that ensures that the military will forever heed party commands hyp1 str it is a guide to action which ensures that the military always obeys the commands of the party sentence_chrf ref1 hyp1 doctest ellipsis 0 6349 type ref1 type hyp1 str true sentence_chrf ref1 split hyp1 split doctest ellipsis 0 6349 to skip the unigrams and only use 2 to 3 grams sentence_chrf ref1 hyp1 min_len 2 max_len 3 doctest ellipsis 0 6617 param references reference sentence type references list str str param hypothesis a hypothesis sentence type hypothesis list str str param min_len the minimum order of n gram this function should extract type min_len int param max_len the maximum order of n gram this function should extract type max_len int param beta the parameter to assign more importance to recall over precision type beta float param ignore_whitespace ignore whitespace characters in scoring type ignore_whitespace bool return the sentence level chrf score rtype float turn list of tokens into a string this function computes the precision recall and fscore from the ngram overlaps it returns the support which is the true positive score by underspecifying the input type the function will be agnostic as to how it computes the ngrams and simply take the whichever element in the list it could be either token or character param reference the reference sentence type reference list param hypothesis the hypothesis sentence type hypothesis list param n extract up to the n th order ngrams type n int param beta the parameter to assign more importance to recall over precision type beta float param epsilon the fallback value if the hypothesis or reference is empty type epsilon float return returns the precision recall and f score and support true positive rtype tuple float calculate the number of ngram matches true positives true positives false positives true positives false negatives precision recall calculates the corpus level chrf character n gram f score it is the macro averaged value of the sentence segment level chrf score this implementation of chrf only supports a single reference at the moment ref1 str it is a guide to action that ensures that the military will forever heed party commands split ref2 str it is the guiding principle which guarantees the military forces always being under the command of the party split hyp1 str it is a guide to action which ensures that the military always obeys the commands of the party split hyp2 str it is to insure the troops forever hearing the activity guidebook that party direct corpus_chrf ref1 ref2 ref1 ref2 hyp1 hyp2 hyp2 hyp1 doctest ellipsis 0 3910 param references a corpus of list of reference sentences w r t hypotheses type references list list str param hypotheses a list of hypothesis sentences type hypotheses list list str param min_len the minimum order of n gram this function should extract type min_len int param max_len the maximum order of n gram this function should extract type max_len int param beta the parameter to assign more importance to recall over precision type beta float param ignore_whitespace ignore whitespace characters in scoring type ignore_whitespace bool return the sentence level chrf score rtype float keep f scores for each n gram order separate iterate through each hypothesis and their corresponding references preprocess both reference and hypothesis calculate f scores for each sentence and for each n gram order separately compute the precision recall fscore and support how many n gram sizes sum of f scores over all sentences for each n gram order macro average over n gram orders and over all sentences
import re
from collections import Counter, defaultdict

from nltk.util import ngrams


def sentence_chrf(
    reference, hypothesis, min_len=1, max_len=6, beta=3.0, ignore_whitespace=True
):
    
    return corpus_chrf(
        [reference],
        [hypothesis],
        min_len,
        max_len,
        beta=beta,
        ignore_whitespace=ignore_whitespace,
    )


def _preprocess(sent, ignore_whitespace):
    if type(sent) != str:
        
        sent = " ".join(sent)

    if ignore_whitespace:
        sent = re.sub(r"\s+", "", sent)
    return sent


def chrf_precision_recall_fscore_support(
    reference, hypothesis, n, beta=3.0, epsilon=1e-16
):
    
    ref_ngrams = Counter(ngrams(reference, n))
    hyp_ngrams = Counter(ngrams(hypothesis, n))

    
    overlap_ngrams = ref_ngrams & hyp_ngrams
    tp = sum(overlap_ngrams.values())  
    tpfp = sum(hyp_ngrams.values())  
    tpfn = sum(ref_ngrams.values())  

    try:
        prec = tp / tpfp  
        rec = tp / tpfn  
        factor = beta**2
        fscore = (1 + factor) * (prec * rec) / (factor * prec + rec)
    except ZeroDivisionError:
        prec = rec = fscore = epsilon
    return prec, rec, fscore, tp


def corpus_chrf(
    references, hypotheses, min_len=1, max_len=6, beta=3.0, ignore_whitespace=True
):
    

    assert len(references) == len(
        hypotheses
    ), "The number of hypotheses and their references should be the same"
    num_sents = len(hypotheses)

    
    ngram_fscores = defaultdict(lambda: list())

    
    for reference, hypothesis in zip(references, hypotheses):

        
        reference = _preprocess(reference, ignore_whitespace)
        hypothesis = _preprocess(hypothesis, ignore_whitespace)

        
        
        for n in range(min_len, max_len + 1):
            
            prec, rec, fscore, tp = chrf_precision_recall_fscore_support(
                reference, hypothesis, n, beta=beta
            )
            ngram_fscores[n].append(fscore)

    
    num_ngram_sizes = len(ngram_fscores)

    
    total_scores = [sum(fscores) for n, fscores in ngram_fscores.items()]

    
    return (sum(total_scores) / num_ngram_sizes) / num_sents