public static List < String > tokenize ( String arguments , boolean stringify ) { LinkedList < String > argList = new LinkedList <> ( ) ; StringBuilder currArg = new StringBuilder ( ) ; boolean escaped = false ; int state = NO_TOKEN_STATE ; // start in the NO_TOKEN_STATE int len = arguments . length ( ) ; // Loop over each character in the string for ( int i = 0 ; i < len ; i ++ ) { char c = arguments . charAt ( i ) ; if ( escaped ) { // Escaped state: just append the next character to the current arg. escaped = false ; currArg . append ( c ) ; } else { switch ( state ) { case SINGLE_QUOTE_STATE : if ( c == ' ' ) { // Seen the close quote; continue this arg until whitespace is seen state = NORMAL_TOKEN_STATE ; } else { currArg . append ( c ) ; } break ; case DOUBLE_QUOTE_STATE : if ( c == ' ' ) { // Seen the close quote; continue this arg until whitespace is seen state = NORMAL_TOKEN_STATE ; } else if ( c == ' ' ) { // Look ahead, and only escape quotes or backslashes i ++ ; char next = arguments . charAt ( i ) ; if ( next == ' ' || next == ' ' ) { currArg . append ( next ) ; } else { currArg . append ( c ) ; currArg . append ( next ) ; } } else { currArg . append ( c ) ; } break ; // case NORMAL_TOKEN_STATE: // if (Character.isWhitespace(c)) { // // Whitespace ends the token; start a new one // argList.add(currArg.toString()); // currArg = new StringBuffer(); // state = NO_TOKEN_STATE; // } // else if (c == '\\') { // // Backslash in a normal token: escape the next character // escaped = true; // } // else if (c == '\'') { // state = SINGLE_QUOTE_STATE; // } // else if (c == '"') { // state = DOUBLE_QUOTE_STATE; // } // else { // currArg.append(c); // } // break; case NO_TOKEN_STATE : case NORMAL_TOKEN_STATE : switch ( c ) { case ' ' : escaped = true ; state = NORMAL_TOKEN_STATE ; break ; case ' ' : state = SINGLE_QUOTE_STATE ; break ; case ' ' : state = DOUBLE_QUOTE_STATE ; break ; default : if ( ! Character . isWhitespace ( c ) ) { currArg . append ( c ) ; state = NORMAL_TOKEN_STATE ; } else if ( state == NORMAL_TOKEN_STATE ) { // Whitespace ends the token; start a new one argList . add ( currArg . toString ( ) ) ; currArg = new StringBuilder ( ) ; state = NO_TOKEN_STATE ; } } break ; default : throw new IllegalStateException ( "ArgumentTokenizer state " + state + " is invalid!" ) ; } } } // If we're still escaped, put in the backslash if ( escaped ) { currArg . append ( ' ' ) ; argList . add ( currArg . toString ( ) ) ; } // Close the last argument if we haven't yet else if ( state != NO_TOKEN_STATE ) { argList . add ( currArg . toString ( ) ) ; } // Format each argument if we've been told to stringify them if ( stringify ) { for ( int i = 0 ; i < argList . size ( ) ; i ++ ) { argList . set ( i , "\"" + _escapeQuotesAndBackslashes ( argList . get ( i ) ) + "\"" ) ; } } return argList ; }
Tokenizes the given String into String tokens .