public static void copyToHDFS ( String localFile , String hdfsFile ) throws IOException { String hadoopConfPath ; if ( Environment . getProperty ( "hadoop.conf.path" ) == null ) hadoopConfPath = "../../../conf" ; else hadoopConfPath = Environment . getProperty ( "hadoop.conf.path" ) ; // Read the configuration for the Hadoop environment Configuration hadoopConf = new Configuration ( ) ; hadoopConf . addResource ( new Path ( hadoopConfPath + "/hadoop-default.xml" ) ) ; hadoopConf . addResource ( new Path ( hadoopConfPath + "/hadoop-site.xml" ) ) ; // System.out.println(hadoopConf.get("hadoop.tmp.dir")); // System.out.println(hadoopConf.get("fs.default.name")); FileSystem fs = FileSystem . get ( hadoopConf ) ; // HadoopDFS deals with Path Path inFile = new Path ( "file://" + localFile ) ; Path outFile = new Path ( hadoopConf . get ( "fs.default.name" ) + hdfsFile ) ; // Read from and write to new file Environment . logInfo ( "Uploading to HDFS (file " + outFile + ") ..." ) ; fs . copyFromLocalFile ( false , inFile , outFile ) ; }
Copy a local file to HDFS