public static void merge ( File mergeDir , List < File > sortedFiles , File mergedFile , Comparator < String > comparator , int mergeFactor ) { LinkedList < File > mergeFiles = new LinkedList < File > ( sortedFiles ) ; // merge all files LinkedList < BatchFile > batch = new LinkedList < BatchFile > ( ) ; try { while ( mergeFiles . size ( ) > 0 ) { // create batch batch . clear ( ) ; for ( int i = 0 ; i < mergeFactor && mergeFiles . size ( ) > 0 ; i ++ ) { batch . add ( new BatchFile ( mergeFiles . remove ( ) ) ) ; } // create aggregation file File aggFile ; if ( mergeFiles . size ( ) > 0 ) { // create new aggregate file aggFile = File . createTempFile ( "merge-" , ".part" , mergeDir ) ; mergeFiles . addLast ( aggFile ) ; } else { // create final file aggFile = mergedFile ; } BufferedWriter writer = new BufferedWriter ( new OutputStreamWriter ( new FileOutputStream ( aggFile . getAbsoluteFile ( ) , false ) , DataUtilDefaults . charSet ) ) ; // process batch String [ ] buffer = new String [ batch . size ( ) ] ; Arrays . fill ( buffer , null ) ; boolean [ ] inUse = new boolean [ batch . size ( ) ] ; Arrays . fill ( inUse , true ) ; boolean inUseFlag = true ; while ( inUseFlag ) { // load comparison buffer int index = - 1 ; String selected = null ; for ( int i = 0 ; i < batch . size ( ) ; i ++ ) { if ( inUse [ i ] ) { if ( buffer [ i ] == null ) { // need more data buffer [ i ] = batch . get ( i ) . getRow ( ) ; if ( buffer [ i ] == null ) { inUse [ i ] = false ; } } if ( buffer [ i ] != null ) { if ( index == - 1 ) { // set item index = i ; selected = buffer [ i ] ; } else if ( comparator . compare ( buffer [ i ] , selected ) < 0 ) { // replace item index = i ; selected = buffer [ i ] ; } } } } if ( index >= 0 ) { // select item and write to new aggregate file writer . write ( buffer [ index ] + DataUtilDefaults . lineTerminator ) ; buffer [ index ] = null ; inUseFlag = true ; } else { inUseFlag = false ; } } // no more data writer . flush ( ) ; writer . close ( ) ; } } catch ( IOException e ) { throw new DataUtilException ( e ) ; } }
Merge multiple files into a single file . Multi - pass approach where the merge factor controls how many passes that are required . If a merge factor of X is specified then X files will be opened concurrently for merging .