public RDD < Cells > createS3RDD ( ExtractorConfig < Cells > config ) { Serializable bucket = config . getValues ( ) . get ( ExtractorConstants . S3_BUCKET ) ; Serializable path = config . getValues ( ) . get ( ExtractorConstants . FS_FILE_PATH ) ; final TextFileDataTable textFileDataTable = UtilFS . createTextFileMetaDataFromConfig ( config , this ) ; String filePath = path . toString ( ) ; if ( config . getExtractorImplClassName ( ) . equals ( ExtractorConstants . S3 ) ) { filePath = ExtractorConstants . S3_PREFIX + bucket . toString ( ) + path . toString ( ) ; } Configuration hadoopConf = this . sc ( ) . hadoopConfiguration ( ) ; hadoopConf . set ( "fs.s3n.impl" , "org.apache.hadoop.fs.s3native.NativeS3FileSystem" ) ; hadoopConf . set ( "fs.s3n.awsAccessKeyId" , config . getString ( ExtractorConstants . S3_ACCESS_KEY_ID ) ) ; hadoopConf . set ( "fs.s3n.awsSecretAccessKey" , config . getString ( ExtractorConstants . S3_SECRET_ACCESS_KEY ) ) ; return createRDDFromFilePath ( filePath , textFileDataTable ) ; }
Returns a Cells RDD from S3 fileSystem .