private void genFiles ( ) throws IOException { // // BufferedReader in = new BufferedReader(new FileReader(new File(inDir, // StructureGenerator.FILE_STRUCTURE_FILE_NAME))); // String line; // while ((line = in.readLine()) != null) { // String[] tokens = line.split(" "); // if (tokens.length != 2) { // throw new IOException("Expect at most 2 tokens per line: " // + line); // } // String fileName = root + tokens[0]; // long fileSize = (long) (BLOCK_SIZE * Double.parseDouble(tokens[1])); // genFile(new Path(fileName), fileSize); // } config = new Configuration ( getConf ( ) ) ; config . setInt ( "dfs.replication" , 3 ) ; config . set ( "dfs.rootdir" , root . toString ( ) ) ; JobConf job = new JobConf ( config , DataGenerator . class ) ; job . setJobName ( "data-genarator" ) ; FileOutputFormat . setOutputPath ( job , new Path ( "data-generator-result" ) ) ; // create the input for the map-reduce job Path inputPath = new Path ( ROOT + "load_input" ) ; fs . mkdirs ( inputPath ) ; fs . copyFromLocalFile ( new Path ( inDir + "/" + StructureGenerator . FILE_STRUCTURE_FILE_NAME ) , inputPath ) ; FileInputFormat . setInputPaths ( job , new Path ( ROOT + "load_input" ) ) ; job . setInputFormat ( TextInputFormat . class ) ; job . setOutputKeyClass ( Text . class ) ; job . setOutputValueClass ( Text . class ) ; job . setMapperClass ( CreateFiles . class ) ; job . setNumMapTasks ( nFiles / 10 ) ; job . setNumReduceTasks ( 0 ) ; JobClient . runJob ( job ) ; }
Read file structure file under the input directory . Create each file under the specified root . The file names are relative to the root .