private MinusOneFieldAndOptimizationResult runWithScrollingAndAddFilter ( String firstFieldName , String secondFieldName ) throws SqlParseException { SearchResponse scrollResp = ElasticUtils . scrollOneTimeWithHits ( this . client , this . builder . getFirstSearchRequest ( ) , builder . getOriginalSelect ( true ) , this . maxDocsToFetchOnEachScrollShard ) ; Set < Object > results = new HashSet <> ( ) ; int currentNumOfResults = 0 ; SearchHit [ ] hits = scrollResp . getHits ( ) . getHits ( ) ; SearchHit someHit = null ; if ( hits . length != 0 ) { //we need some hit for creating InnerResults. someHit = hits [ 0 ] ; } int totalDocsFetchedFromFirstTable = 0 ; int totalDocsFetchedFromSecondTable = 0 ; Where originalWhereSecondTable = this . builder . getOriginalSelect ( false ) . getWhere ( ) ; while ( hits . length != 0 ) { totalDocsFetchedFromFirstTable += hits . length ; Set < Object > currentSetFromResults = new HashSet <> ( ) ; fillSetFromHits ( firstFieldName , hits , currentSetFromResults ) ; //fetch from second Select secondQuerySelect = this . builder . getOriginalSelect ( false ) ; Where where = createWhereWithOrigianlAndTermsFilter ( secondFieldName , originalWhereSecondTable , currentSetFromResults ) ; secondQuerySelect . setWhere ( where ) ; DefaultQueryAction queryAction = new DefaultQueryAction ( this . client , secondQuerySelect ) ; queryAction . explain ( ) ; if ( totalDocsFetchedFromSecondTable > this . maxDocsToFetchOnSecondTable ) { break ; } SearchResponse responseForSecondTable = ElasticUtils . scrollOneTimeWithHits ( this . client , queryAction . getRequestBuilder ( ) , secondQuerySelect , this . maxDocsToFetchOnEachScrollShard ) ; SearchHits secondQuerySearchHits = responseForSecondTable . getHits ( ) ; SearchHit [ ] secondQueryHits = secondQuerySearchHits . getHits ( ) ; while ( secondQueryHits . length > 0 ) { totalDocsFetchedFromSecondTable += secondQueryHits . length ; removeValuesFromSetAccordingToHits ( secondFieldName , currentSetFromResults , secondQueryHits ) ; if ( totalDocsFetchedFromSecondTable > this . maxDocsToFetchOnSecondTable ) { break ; } responseForSecondTable = client . prepareSearchScroll ( responseForSecondTable . getScrollId ( ) ) . setScroll ( new TimeValue ( 600000 ) ) . execute ( ) . actionGet ( ) ; secondQueryHits = responseForSecondTable . getHits ( ) . getHits ( ) ; } results . addAll ( currentSetFromResults ) ; if ( totalDocsFetchedFromFirstTable > this . maxDocsToFetchOnFirstTable ) { System . out . println ( "too many results for first table, stoping at:" + totalDocsFetchedFromFirstTable ) ; break ; } scrollResp = client . prepareSearchScroll ( scrollResp . getScrollId ( ) ) . setScroll ( new TimeValue ( 600000 ) ) . execute ( ) . actionGet ( ) ; hits = scrollResp . getHits ( ) . getHits ( ) ; } return new MinusOneFieldAndOptimizationResult ( results , someHit ) ; }
1 . 1 . 2 add all results left from miniset to bigset