int match ( CharsetDetector det , int [ ] commonChars ) { @ SuppressWarnings ( "unused" ) int singleByteCharCount = 0 ; //TODO Do we really need this? int doubleByteCharCount = 0 ; int commonCharCount = 0 ; int badCharCount = 0 ; int totalCharCount = 0 ; int confidence = 0 ; iteratedChar iter = new iteratedChar ( ) ; detectBlock : { for ( iter . reset ( ) ; nextChar ( iter , det ) ; ) { totalCharCount ++ ; if ( iter . error ) { badCharCount ++ ; } else { long cv = iter . charValue & 0xFFFFFFFF L ; if ( cv <= 0xff ) { singleByteCharCount ++ ; } else { doubleByteCharCount ++ ; if ( commonChars != null ) { // NOTE: This assumes that there are no 4-byte common chars. if ( Arrays . binarySearch ( commonChars , ( int ) cv ) >= 0 ) { commonCharCount ++ ; } } } } if ( badCharCount >= 2 && badCharCount * 5 >= doubleByteCharCount ) { // Bail out early if the byte data is not matching the encoding scheme. break detectBlock ; } } if ( doubleByteCharCount <= 10 && badCharCount == 0 ) { // Not many multi-byte chars. if ( doubleByteCharCount == 0 && totalCharCount < 10 ) { // There weren't any multibyte sequences, and there was a low density of non-ASCII single bytes. // We don't have enough data to have any confidence. // Statistical analysis of single byte non-ASCII charcters would probably help here. confidence = 0 ; } else { // ASCII or ISO file? It's probably not our encoding, // but is not incompatible with our encoding, so don't give it a zero. confidence = 10 ; } break detectBlock ; } // // No match if there are too many characters that don't fit the encoding scheme. // (should we have zero tolerance for these?) // if ( doubleByteCharCount < 20 * badCharCount ) { confidence = 0 ; break detectBlock ; } if ( commonChars == null ) { // We have no statistics on frequently occuring characters. // Assess confidence purely on having a reasonable number of // multi-byte characters (the more the better confidence = 30 + doubleByteCharCount - 20 * badCharCount ; if ( confidence > 100 ) { confidence = 100 ; } } else { // // Frequency of occurence statistics exist. // double maxVal = Math . log ( ( float ) doubleByteCharCount / 4 ) ; double scaleFactor = 90.0 / maxVal ; confidence = ( int ) ( Math . log ( commonCharCount + 1 ) * scaleFactor + 10 ) ; confidence = Math . min ( confidence , 100 ) ; } } // end of detectBlock: return confidence ; }
Test the match of this charset with the input text data which is obtained via the CharsetDetector object .