private boolean setup ( ) throws IOException { estimateSavings ( ) ; final String randomId = getRandomId ( ) ; JobClient jClient = new JobClient ( jobconf ) ; Path jobdir = new Path ( jClient . getSystemDir ( ) , NAME + "_" + randomId ) ; LOG . info ( JOB_DIR_LABEL + "=" + jobdir ) ; jobconf . set ( JOB_DIR_LABEL , jobdir . toString ( ) ) ; Path log = new Path ( jobdir , "_logs" ) ; // The control file should have small size blocks. This helps // in spreading out the load from mappers that will be spawned. jobconf . setInt ( "dfs.blocks.size" , OP_LIST_BLOCK_SIZE ) ; FileOutputFormat . setOutputPath ( jobconf , log ) ; LOG . info ( "log=" + log ) ; // create operation list FileSystem fs = jobdir . getFileSystem ( jobconf ) ; Path opList = new Path ( jobdir , "_" + OP_LIST_LABEL ) ; jobconf . set ( OP_LIST_LABEL , opList . toString ( ) ) ; int opCount = 0 , synCount = 0 ; SequenceFile . Writer opWriter = null ; try { opWriter = SequenceFile . createWriter ( fs , jobconf , opList , Text . class , PolicyInfo . class , SequenceFile . CompressionType . NONE ) ; for ( RaidPolicyPathPair p : raidPolicyPathPairList ) { // If a large set of files are Raided for the first time, files // in the same directory that tend to have the same size will end up // with the same map. This shuffle mixes things up, allowing a better // mix of files. java . util . Collections . shuffle ( p . srcPaths ) ; for ( EncodingCandidate ec : p . srcPaths ) { opWriter . append ( new Text ( ec . toString ( ) ) , p . policy ) ; opCount ++ ; if ( ++ synCount > SYNC_FILE_MAX ) { opWriter . sync ( ) ; synCount = 0 ; } } } } finally { if ( opWriter != null ) { opWriter . close ( ) ; } fs . setReplication ( opList , OP_LIST_REPLICATION ) ; // increase replication for control file } raidPolicyPathPairList . clear ( ) ; jobconf . setInt ( OP_COUNT_LABEL , opCount ) ; LOG . info ( "Number of files=" + opCount ) ; jobconf . setNumMapTasks ( getMapCount ( opCount ) ) ; LOG . info ( "jobName= " + jobName + " numMapTasks=" + jobconf . getNumMapTasks ( ) ) ; return opCount != 0 ; }
set up input file which has the list of input files .