public List < String > tokenize ( String text ) { List < String > tokens = new ArrayList < String > ( ) ; text = text . toLowerCase ( ) ; char [ ] array = text . toCharArray ( ) ; //convert all apostrophes to 0x27 for ( int i = 0 ; i < array . length ; i ++ ) { if ( isApostrofe ( array [ i ] ) ) array [ i ] = 0x27 ; } int pos = 0 ; //term cycle while ( pos < array . length ) { //scan to the start of the term while ( pos < array . length && ! Character . isLetterOrDigit ( array [ pos ] ) ) pos ++ ; int start = pos ; if ( start == array . length ) break ; //scan to the end of the term while ( pos < array . length && isLetterOrDigitOrApostrofe ( array [ pos ] ) ) pos ++ ; int newpos = pos ; while ( newpos > start && isApostrofe ( array [ newpos - 1 ] ) ) newpos -- ; if ( newpos > start ) tokens . add ( new String ( array , start , newpos - start ) ) ; } return tokens ; }
get list of tokens from the text for indexing