class containing unit tests for nltk metrics agreement disagreement simple test based on https github comfoolswoodkrippendorffsalpharawmasterkrippendorff pdf same simple test with 1 rating removed removal of that rating should not matter kapha ignores items with only 1 rating more advanced test based on http www agreestat comresearchpapersonkrippendorffalpha pdf same more advanced example but with 1 rating removed again removal of that 1 rating should not matter class containing unit tests for nltk metrics agreement disagreement simple test based on https github com foolswood krippendorffs_alpha raw master krippendorff pdf same simple test with 1 rating removed removal of that rating should not matter k apha ignores items with only 1 rating more advanced test based on http www agreestat com research_papers onkrippendorffalpha pdf same more advanced example but with 1 rating removed again removal of that 1 rating should not matter
import unittest

from nltk.metrics.agreement import AnnotationTask


class TestDisagreement(unittest.TestCase):

    

    def test_easy(self):
        
        data = [
            ("coder1", "dress1", "YES"),
            ("coder2", "dress1", "NO"),
            ("coder3", "dress1", "NO"),
            ("coder1", "dress2", "YES"),
            ("coder2", "dress2", "NO"),
            ("coder3", "dress3", "NO"),
        ]
        annotation_task = AnnotationTask(data)
        self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)

    def test_easy2(self):
        
        data = [
            ("coder1", "dress1", "YES"),
            ("coder2", "dress1", "NO"),
            ("coder3", "dress1", "NO"),
            ("coder1", "dress2", "YES"),
            ("coder2", "dress2", "NO"),
        ]
        annotation_task = AnnotationTask(data)
        self.assertAlmostEqual(annotation_task.alpha(), -0.3333333)

    def test_advanced(self):
        
        data = [
            ("A", "1", "1"),
            ("B", "1", "1"),
            ("D", "1", "1"),
            ("A", "2", "2"),
            ("B", "2", "2"),
            ("C", "2", "3"),
            ("D", "2", "2"),
            ("A", "3", "3"),
            ("B", "3", "3"),
            ("C", "3", "3"),
            ("D", "3", "3"),
            ("A", "4", "3"),
            ("B", "4", "3"),
            ("C", "4", "3"),
            ("D", "4", "3"),
            ("A", "5", "2"),
            ("B", "5", "2"),
            ("C", "5", "2"),
            ("D", "5", "2"),
            ("A", "6", "1"),
            ("B", "6", "2"),
            ("C", "6", "3"),
            ("D", "6", "4"),
            ("A", "7", "4"),
            ("B", "7", "4"),
            ("C", "7", "4"),
            ("D", "7", "4"),
            ("A", "8", "1"),
            ("B", "8", "1"),
            ("C", "8", "2"),
            ("D", "8", "1"),
            ("A", "9", "2"),
            ("B", "9", "2"),
            ("C", "9", "2"),
            ("D", "9", "2"),
            ("B", "10", "5"),
            ("C", "10", "5"),
            ("D", "10", "5"),
            ("C", "11", "1"),
            ("D", "11", "1"),
            ("C", "12", "3"),
        ]
        annotation_task = AnnotationTask(data)
        self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)

    def test_advanced2(self):
        
        data = [
            ("A", "1", "1"),
            ("B", "1", "1"),
            ("D", "1", "1"),
            ("A", "2", "2"),
            ("B", "2", "2"),
            ("C", "2", "3"),
            ("D", "2", "2"),
            ("A", "3", "3"),
            ("B", "3", "3"),
            ("C", "3", "3"),
            ("D", "3", "3"),
            ("A", "4", "3"),
            ("B", "4", "3"),
            ("C", "4", "3"),
            ("D", "4", "3"),
            ("A", "5", "2"),
            ("B", "5", "2"),
            ("C", "5", "2"),
            ("D", "5", "2"),
            ("A", "6", "1"),
            ("B", "6", "2"),
            ("C", "6", "3"),
            ("D", "6", "4"),
            ("A", "7", "4"),
            ("B", "7", "4"),
            ("C", "7", "4"),
            ("D", "7", "4"),
            ("A", "8", "1"),
            ("B", "8", "1"),
            ("C", "8", "2"),
            ("D", "8", "1"),
            ("A", "9", "2"),
            ("B", "9", "2"),
            ("C", "9", "2"),
            ("D", "9", "2"),
            ("B", "10", "5"),
            ("C", "10", "5"),
            ("D", "10", "5"),
            ("C", "11", "1"),
            ("D", "11", "1"),
            ("C", "12", "3"),
        ]
        annotation_task = AnnotationTask(data)
        self.assertAlmostEqual(annotation_task.alpha(), 0.743421052632)