@ Override protected final ParseWriter parseChunk ( int chunkId , ParseReader din , ParseWriter dout ) { _cidx = chunkId ; // only do something if within file size and the orc file is not empty List < StripeInformation > stripesInfo = ( ( OrcParseSetup ) this . _setup ) . getStripes ( ) ; if ( stripesInfo . size ( ) == 0 ) { dout . addError ( new ParseWriter . ParseErr ( "Orc Parser: Empty file." , chunkId , 0L , - 2L ) ) ; return dout ; // empty file } OrcParseSetup setup = ( OrcParseSetup ) this . _setup ; StripeInformation thisStripe = stripesInfo . get ( chunkId ) ; // get one stripe // write one stripe of data to H2O frame String [ ] orcTypes = setup . getColumnTypesString ( ) ; boolean [ ] toInclude = setup . getToInclude ( ) ; try { RecordReader perStripe = orcFileReader . rows ( thisStripe . getOffset ( ) , thisStripe . getDataLength ( ) , setup . getToInclude ( ) , null , setup . getColumnNames ( ) ) ; VectorizedRowBatch batch = null ; long rows = 0 ; long rowCount = thisStripe . getNumberOfRows ( ) ; while ( rows != rowCount ) { batch = perStripe . nextBatch ( batch ) ; // read orc file stripes in vectorizedRowBatch long currentBatchRow = batch . count ( ) ; int nrows = ( int ) currentBatchRow ; if ( currentBatchRow != nrows ) throw new IllegalArgumentException ( "got batch with too many records, does not fit in int" ) ; ColumnVector [ ] dataVectors = batch . cols ; int colIndex = 0 ; for ( int col = 0 ; col < batch . numCols ; ++ col ) { // read one column at a time; if ( toInclude [ col + 1 ] ) { // only write a column if we actually want it if ( _setup . getColumnTypes ( ) [ colIndex ] != Vec . T_BAD ) write1column ( dataVectors [ col ] , orcTypes [ colIndex ] , colIndex , nrows , dout ) ; else dout . addNAs ( col , nrows ) ; colIndex ++ ; } } rows += currentBatchRow ; // record number of rows of data actually read } byte [ ] col_types = _setup . getColumnTypes ( ) ; for ( int i = 0 ; i < col_types . length ; ++ i ) { if ( col_types [ i ] == Vec . T_BAD ) dout . addNAs ( i , ( int ) rowCount ) ; } perStripe . close ( ) ; } catch ( IOException ioe ) { throw new RuntimeException ( ioe ) ; } return dout ; }
This method calculates the number of stripes that will be read for each chunk . Since only single threading is supported in reading each stripe we will never split one stripe over different chunks .