public static Token [ ] parse ( String text , boolean optimize ) { if ( text == null ) { return null ; } if ( text . length ( ) == 0 ) { Token t = new Token ( text ) ; return new Token [ ] { t } ; } Token [ ] tokens = null ; List < Token > tokenList = Tokenizer . tokenize ( text , optimize ) ; if ( ! tokenList . isEmpty ( ) ) { tokens = tokenList . toArray ( new Token [ 0 ] ) ; if ( optimize ) { tokens = Tokenizer . optimize ( tokens ) ; } } return tokens ; }
Returns an array of tokens that contains tokenized string .