public final String [ ] getTokensWithMultiWords ( final String [ ] tokens ) { final Span [ ] multiWordSpans = multiWordsToSpans ( tokens ) ; final List < String > tokenList = new ArrayList < String > ( Arrays . asList ( tokens ) ) ; int counter = 0 ; for ( final Span mwSpan : multiWordSpans ) { final int fromIndex = mwSpan . getStart ( ) - counter ; final int toIndex = mwSpan . getEnd ( ) - counter ; // System.err.println(fromIndex + " " + toIndex); // add to the counter the length of the sublist removed // to allow the fromIndex and toIndex to match wrt to the tokenList // indexes counter = counter + tokenList . subList ( fromIndex , toIndex ) . size ( ) - 1 ; // create the multiword joining the sublist final String multiWord = Joiner . on ( "#" ) . join ( tokenList . subList ( fromIndex , toIndex ) ) ; // remove the sublist containing the tokens to be replaced in the span tokenList . subList ( fromIndex , toIndex ) . clear ( ) ; // add the multiword containing the tokens in one Span tokenList . add ( fromIndex , multiWord ) ; } return tokenList . toArray ( new String [ tokenList . size ( ) ] ) ; }
Get input text and join the multiwords found in the dictionary object .