public static String [ ] tokenize ( String input , boolean lowercase ) { String [ ] tokens = input . split ( "\\W" ) ; // lower case for ( int i = 0 ; i < tokens . length && lowercase ; i ++ ) { tokens [ i ] = tokens [ i ] . toLowerCase ( ) ; } return tokens ; }
tokenize based on simple regular expression