public final Span [ ] multiWordsToSpans ( final String [ ] tokens ) { final List < Span > multiWordsFound = new LinkedList < Span > ( ) ; for ( int offsetFrom = 0 ; offsetFrom < tokens . length ; offsetFrom ++ ) { Span multiwordFound = null ; String tokensSearching [ ] = new String [ ] { } ; for ( int offsetTo = offsetFrom ; offsetTo < tokens . length ; offsetTo ++ ) { final int lengthSearching = offsetTo - offsetFrom + 1 ; if ( lengthSearching > getMaxTokenCount ( ) ) { break ; } else { tokensSearching = new String [ lengthSearching ] ; System . arraycopy ( tokens , offsetFrom , tokensSearching , 0 , lengthSearching ) ; final String entryForSearch = StringUtils . getStringFromTokens ( tokensSearching ) ; final String entryValue = dictionary . get ( entryForSearch . toLowerCase ( ) ) ; if ( entryValue != null ) { multiwordFound = new Span ( offsetFrom , offsetTo + 1 , entryValue ) ; } } } if ( multiwordFound != null ) { multiWordsFound . add ( multiwordFound ) ; offsetFrom += multiwordFound . length ( ) - 1 ; } } return multiWordsFound . toArray ( new Span [ multiWordsFound . size ( ) ] ) ; }
Detects multiword expressions ignoring case .