DataInputStream computeMetadata ( Configuration conf , InputStream dataStream ) throws IOException { ByteArrayOutputStream mdOutBase = new ByteArrayOutputStream ( 1024 * 1024 ) ; DataOutputStream mdOut = new DataOutputStream ( mdOutBase ) ; // First, write out the version. mdOut . writeShort ( FSDataset . FORMAT_VERSION_NON_INLINECHECKSUM ) ; // Create a summer and write out its header. int bytesPerChecksum = conf . getInt ( "io.bytes.per.checksum" , 512 ) ; DataChecksum sum = DataChecksum . newDataChecksum ( DataChecksum . CHECKSUM_CRC32 , bytesPerChecksum ) ; sum . writeHeader ( mdOut ) ; // Buffer to read in a chunk of data. byte [ ] buf = new byte [ bytesPerChecksum ] ; // Buffer to store the checksum bytes. byte [ ] chk = new byte [ sum . getChecksumSize ( ) ] ; // Read data till we reach the end of the input stream. int bytesSinceFlush = 0 ; while ( true ) { // Read some bytes. int bytesRead = dataStream . read ( buf , bytesSinceFlush , bytesPerChecksum - bytesSinceFlush ) ; if ( bytesRead == - 1 ) { if ( bytesSinceFlush > 0 ) { boolean reset = true ; sum . writeValue ( chk , 0 , reset ) ; // This also resets the sum. // Write the checksum to the stream. mdOut . write ( chk , 0 , chk . length ) ; bytesSinceFlush = 0 ; } break ; } // Update the checksum. sum . update ( buf , bytesSinceFlush , bytesRead ) ; bytesSinceFlush += bytesRead ; // Flush the checksum if necessary. if ( bytesSinceFlush == bytesPerChecksum ) { boolean reset = true ; sum . writeValue ( chk , 0 , reset ) ; // This also resets the sum. // Write the checksum to the stream. mdOut . write ( chk , 0 , chk . length ) ; bytesSinceFlush = 0 ; } } byte [ ] mdBytes = mdOutBase . toByteArray ( ) ; return new DataInputStream ( new ByteArrayInputStream ( mdBytes ) ) ; }
Reads data from the data stream provided and computes metadata .