private void writeControlFile ( FileSystem fs , Path outputPath , Path checksumFile , String name ) throws IOException { SequenceFile . Writer write = null ; try { Path parentDir = new Path ( rtc . input , "filelists" ) ; if ( ! fs . exists ( parentDir ) ) { fs . mkdirs ( parentDir ) ; } Path controlFile = new Path ( parentDir , name ) ; write = SequenceFile . createWriter ( fs , fs . getConf ( ) , controlFile , Text . class , Text . class , CompressionType . NONE ) ; write . append ( new Text ( outputPath . toString ( ) ) , new Text ( checksumFile . toString ( ) ) ) ; } finally { if ( write != null ) write . close ( ) ; write = null ; } }
This is used for verification Each mapper writes one control file control file only contains the base directory written by this mapper and the checksum file path so that we could create a Read mapper which scanned the files under the base directory and verify the checksum of files with the information given in the checksum file .