private boolean shouldRemoveDataPullUpperBounds ( ) { if ( ! this . workUnitState . getPropAsBoolean ( ConfigurationKeys . SOURCE_QUERYBASED_ALLOW_REMOVE_UPPER_BOUNDS , true ) ) { return false ; } // Only consider the last work unit if ( ! partition . isLastPartition ( ) ) { return false ; } // Don't remove if user specifies one or is recorded in previous run if ( partition . getHasUserSpecifiedHighWatermark ( ) || this . workUnitState . getProp ( ConfigurationKeys . WORK_UNIT_STATE_ACTUAL_HIGH_WATER_MARK_KEY ) != null ) { return false ; } return true ; }
Check if it s appropriate to remove data pull upper bounds in the last work unit fetching as much data as possible from the source . As between the time when data query was created and that was executed there might be some new data generated in the source . Removing the upper bounds will help us grab the new data .