public static String cleanBlockText ( String text ) { // only keep text longer that 5 if ( text == null || text . length ( ) < 6 ) return null ; // only keep text longer that 5 if ( text . length ( ) < 6 ) { return null ; } // keep only 1st line int firstReturn = text . indexOf ( "\n" ) ; if ( firstReturn > - 1 ) { text = text . substring ( 0 , firstReturn ) ; } // remove double spaces text = text . replaceAll ( " +" , " " ) . trim ( ) ; // remove numbers at beg, tabs text = text . replaceAll ( "^\\d*" , "" ) . replaceAll ( "\t" , " " ) . trim ( ) ; // then, remove dot at beg if ( text . length ( ) > 0 && text . charAt ( 0 ) == ' ' ) { text = text . substring ( 1 ) . trim ( ) ; } // check that at least 50% of chars are letters double letters = 0 ; for ( int i = 0 ; i < text . length ( ) ; i ++ ) { if ( Character . isLetter ( text . charAt ( i ) ) ) letters ++ ; } if ( ( letters / ( text . length ( ) + 0d ) ) < 0.5d ) { return null ; } // stemming String stemmed = "" ; for ( String token : text . trim ( ) . split ( " " ) ) { stemmed += stemmer . stem ( token ) + " " ; } // trim at 30 chars String snippet = StringUtils . snippetize ( stemmed . trim ( ) , 30 ) ; return snippet . trim ( ) ; }
Used to identify block titles like Introduction References