private void deleteNonShuffleFiles ( String [ ] dirs ) { FilenameFilter filter = new FilenameFilter ( ) { @ Override public boolean accept ( File dir , String name ) { // Don't delete shuffle data or shuffle index files. return ! name . endsWith ( ".index" ) && ! name . endsWith ( ".data" ) ; } } ; for ( String localDir : dirs ) { try { JavaUtils . deleteRecursively ( new File ( localDir ) , filter ) ; logger . debug ( "Successfully cleaned up non-shuffle files in directory: {}" , localDir ) ; } catch ( Exception e ) { logger . error ( "Failed to delete non-shuffle files in directory: " + localDir , e ) ; } } }
Synchronously deletes non - shuffle files in each directory recursively . Should be executed in its own thread as this may take a long time .