public synchronized List < Token > tokens ( String text ) { requireValidHandle ( ) ; List < Token > allTokens = new ArrayList < Token > ( ) ; int lastStart = 0 ; for ( int i = indexOfSpecialUnknown ( text , 0 ) ; i != - 1 ; i = indexOfSpecialUnknown ( text , i + 1 ) ) { allTokens . addAll ( tokensNonNull ( text . substring ( lastStart , i ) , lastStart ) ) ; allTokens . add ( new Token ( TokenType . UNKNOWN , Character . toString ( text . charAt ( i ) ) , i ) ) ; lastStart = i + 1 ; } allTokens . addAll ( tokensNonNull ( text . substring ( lastStart ) , lastStart ) ) ; return allTokens ; }
Split the given natural language text into a list of Token objects .