@ Override protected TokenStreamComponents createComponents ( String fieldName ) { //final Tokenizer source = new AlphaNumericTokenizer(); final Tokenizer source = new WhitespaceTokenizer ( ) ; TokenStream stream = source ; stream = new UrlTokenizingFilter ( stream ) ; stream = new AlphaNumericFilter ( stream ) ; stream = new WordDelimiterGraphFilter ( stream , WordDelimiterGraphFilter . GENERATE_WORD_PARTS //| WordDelimiterGraphFilter.GENERATE_NUMBER_PARTS | WordDelimiterGraphFilter . PRESERVE_ORIGINAL | WordDelimiterGraphFilter . SPLIT_ON_CASE_CHANGE | WordDelimiterGraphFilter . SPLIT_ON_NUMERICS | WordDelimiterGraphFilter . STEM_ENGLISH_POSSESSIVE , null ) ; stream = new LowerCaseFilter ( stream ) ; stream = new StopFilter ( stream , stopWords ) ; concatenatingFilter = new TokenPairConcatenatingFilter ( stream ) ; return new TokenStreamComponents ( source , concatenatingFilter ) ; }
Creates a the TokenStreamComponents used to analyze the stream .