boolean processRecords ( Configuration conf , Connection hbaseConnection , String cluster , int batchSize , int threadCount , String processFileSubstring ) throws IOException , InterruptedException , ClassNotFoundException , ExecutionException , RowKeyParseException { List < ProcessRecord > processRecords = getProcessRecords ( conf , hbaseConnection , cluster , processFileSubstring ) ; // Bail out early if needed if ( ( processRecords == null ) || ( processRecords . size ( ) == 0 ) ) { return true ; } // Grab the min and the max jobId from all processing records. MinMaxJobFileTracker minMaxJobFileTracker = new MinMaxJobFileTracker ( ) ; for ( ProcessRecord processRecord : processRecords ) { minMaxJobFileTracker . track ( processRecord . getMinJobId ( ) ) ; minMaxJobFileTracker . track ( processRecord . getMaxJobId ( ) ) ; } List < JobRunner > jobRunners = getJobRunners ( conf , hbaseConnection , cluster , false , batchSize , minMaxJobFileTracker . getMinJobId ( ) , minMaxJobFileTracker . getMaxJobId ( ) ) ; boolean success = runJobs ( threadCount , jobRunners ) ; if ( success ) { updateProcessRecords ( conf , hbaseConnection , processRecords ) ; } return success ; }
Pick up the ranges of jobs to process from ProcessRecords . Skip raw rows that have already been processed .