public static List < String > wonderfulTokenizer ( String tokenString , String [ ] delimiters ) { List < String > tokens = new ArrayList < String > ( ) ; String currentToken = "" ; for ( int i = 0 ; i < tokenString . length ( ) ; i ++ ) { String _currItem = String . valueOf ( tokenString . charAt ( i ) ) ; if ( _currItem . trim ( ) . isEmpty ( ) ) continue ; String delimiterFound = "" ; for ( String d : delimiters ) { if ( d . startsWith ( _currItem ) ) { if ( d . length ( ) > 1 ) { if ( d . equals ( tokenString . substring ( i , i + d . length ( ) ) ) ) { delimiterFound = d ; i = i + ( d . length ( ) - 1 ) ; } } else delimiterFound = d ; } if ( ! delimiterFound . isEmpty ( ) ) break ; } if ( ! delimiterFound . isEmpty ( ) ) { if ( ! currentToken . isEmpty ( ) ) { tokens . add ( currentToken ) ; currentToken = "" ; } tokens . add ( delimiterFound ) ; } else currentToken += _currItem ; } if ( ! currentToken . isEmpty ( ) ) tokens . add ( currentToken ) ; return tokens ; }
Tokenize string based on rules