@ Override public double calculateObservedDisagreement ( ) { ensureDistanceFunction ( ) ; double result = 0.0 ; double maxDistance = 1.0 ; for ( ICodingAnnotationItem item : study . getItems ( ) ) { Map < Object , Integer > annotationsPerCategory = CodingAnnotationStudy . countTotalAnnotationsPerCategory ( item ) ; for ( Entry < Object , Integer > category1 : annotationsPerCategory . entrySet ( ) ) for ( Entry < Object , Integer > category2 : annotationsPerCategory . entrySet ( ) ) { if ( category1 . getValue ( ) == null ) continue ; if ( category2 . getValue ( ) == null ) continue ; double distance = distanceFunction . measureDistance ( study , category1 . getKey ( ) , category2 . getKey ( ) ) ; result += category1 . getValue ( ) * category2 . getValue ( ) * distance ; if ( distance > maxDistance ) maxDistance = distance ; } } result /= ( double ) ( maxDistance * study . getItemCount ( ) * study . getRaterCount ( ) * ( study . getRaterCount ( ) - 1 ) ) ; return result ; }
Calculates the observed inter - rater agreement for the annotation study that was passed to the class constructor and the currently assigned distance function .