@ Override public double calculateExpectedAgreement ( ) { Map < Object , int [ ] > annotationsPerCategory = CodingAnnotationStudy . countAnnotationsPerCategory ( study ) ; BigDecimal result = BigDecimal . ZERO ; for ( Object category : study . getCategories ( ) ) { int [ ] annotations = annotationsPerCategory . get ( category ) ; if ( annotations != null ) { BigDecimal prod = BigDecimal . ONE ; for ( int rater = 0 ; rater < study . getRaterCount ( ) ; rater ++ ) prod = prod . multiply ( new BigDecimal ( annotations [ rater ] ) ) ; result = result . add ( prod ) ; } } result = result . divide ( new BigDecimal ( study . getItemCount ( ) ) . pow ( 2 ) , MathContext . DECIMAL128 ) ; return result . doubleValue ( ) ; }
Calculates the expected inter - rater agreement that assumes a different probability distribution for all raters .