@ SuppressWarnings ( "deprecation" ) private void addLocalFiles ( Path jobFileDir , String jobFileList , Configuration conf ) throws IOException { DistributedCache . createSymlink ( conf ) ; for ( String jobFile : SPLITTER . split ( jobFileList ) ) { Path srcJobFile = new Path ( jobFile ) ; // DistributedCache requires absolute path, so we need to use makeQualified. Path destJobFile = new Path ( this . fs . makeQualified ( jobFileDir ) , srcJobFile . getName ( ) ) ; // Copy the file from local file system to HDFS this . fs . copyFromLocalFile ( srcJobFile , destJobFile ) ; // Create a URI that is in the form path#symlink URI destFileUri = URI . create ( destJobFile . toUri ( ) . getPath ( ) + "#" + destJobFile . getName ( ) ) ; LOG . info ( String . format ( "Adding %s to DistributedCache" , destFileUri ) ) ; // Finally add the file to DistributedCache with a symlink named after the file name DistributedCache . addCacheFile ( destFileUri , conf ) ; } }
Add local non - jar files the job depends on to DistributedCache .