@ Override public List < String > tokenize ( final String text ) { final List < String > l = new ArrayList <> ( ) ; final StringTokenizer st = new StringTokenizer ( text , nlTokenizingChars , true ) ; while ( st . hasMoreElements ( ) ) { String token = st . nextToken ( ) ; String origToken = token ; if ( token . length ( ) > 1 ) { if ( startsWithQuote ( token ) && endsWithQuote ( token ) && token . length ( ) > 2 ) { l . add ( token . substring ( 0 , 1 ) ) ; l . add ( token . substring ( 1 , token . length ( ) - 1 ) ) ; l . add ( token . substring ( token . length ( ) - 1 , token . length ( ) ) ) ; } else if ( endsWithQuote ( token ) ) { int cnt = 0 ; while ( endsWithQuote ( token ) ) { token = token . substring ( 0 , token . length ( ) - 1 ) ; cnt ++ ; } l . add ( token ) ; for ( int i = origToken . length ( ) - cnt ; i < origToken . length ( ) ; i ++ ) { l . add ( origToken . substring ( i , i + 1 ) ) ; } } else if ( startsWithQuote ( token ) ) { while ( startsWithQuote ( token ) ) { l . add ( token . substring ( 0 , 1 ) ) ; token = token . substring ( 1 , token . length ( ) ) ; } l . add ( token ) ; } else { l . add ( token ) ; } } else { l . add ( token ) ; } } return joinEMailsAndUrls ( l ) ; }
Tokenizes just like WordTokenizer with the exception for words such as oma s that contain an apostrophe in their middle .