private void handleWholeFileDataFormat ( S3ObjectSummary s3ObjectSummary , String recordId ) throws StageException { S3Object partialS3ObjectForMetadata ; //partialObject with fetchSize 1 byte. //This is mostly used for extracting metadata and such. partialS3ObjectForMetadata = AmazonS3Util . getObjectRange ( s3Client , s3ConfigBean . s3Config . bucket , s3ObjectSummary . getKey ( ) , 1 , s3ConfigBean . sseConfig . useCustomerSSEKey , s3ConfigBean . sseConfig . customerKey , s3ConfigBean . sseConfig . customerKeyMd5 ) ; S3FileRef . Builder s3FileRefBuilder = new S3FileRef . Builder ( ) . s3Client ( s3Client ) . s3ObjectSummary ( s3ObjectSummary ) . useSSE ( s3ConfigBean . sseConfig . useCustomerSSEKey ) . customerKey ( s3ConfigBean . sseConfig . customerKey ) . customerKeyMd5 ( s3ConfigBean . sseConfig . customerKeyMd5 ) . bufferSize ( ( int ) dataParser . suggestedWholeFileBufferSize ( ) ) . createMetrics ( true ) . totalSizeInBytes ( s3ObjectSummary . getSize ( ) ) . rateLimit ( dataParser . wholeFileRateLimit ( ) ) ; if ( dataParser . isWholeFileChecksumRequired ( ) ) { s3FileRefBuilder . verifyChecksum ( true ) . checksumAlgorithm ( HashingUtil . HashType . MD5 ) //128 bit hex encoded md5 checksum. . checksum ( partialS3ObjectForMetadata . getObjectMetadata ( ) . getETag ( ) ) ; } Map < String , Object > metadata = AmazonS3Util . getMetaData ( partialS3ObjectForMetadata ) ; metadata . put ( S3Constants . BUCKET , s3ObjectSummary . getBucketName ( ) ) ; metadata . put ( S3Constants . OBJECT_KEY , s3ObjectSummary . getKey ( ) ) ; metadata . put ( S3Constants . OWNER , s3ObjectSummary . getOwner ( ) ) ; metadata . put ( S3Constants . SIZE , s3ObjectSummary . getSize ( ) ) ; metadata . put ( HeaderAttributeConstants . FILE_NAME , s3ObjectSummary . getKey ( ) ) ; metadata . remove ( S3Constants . CONTENT_LENGTH ) ; parser = dataParser . getParser ( recordId , metadata , s3FileRefBuilder . build ( ) ) ; //Object is assigned so that setHeaders() function can use this to get metadata //information about the object object = partialS3ObjectForMetadata ; }
as the record is just the metadata along with file ref .