protected PriorityQueue < LuceneIndexInputSplit > findSplits ( Configuration conf ) throws IOException { PriorityQueue < LuceneIndexInputSplit > splits = new PriorityQueue < LuceneIndexInputSplit > ( ) ; List < Path > indexDirs = Lists . newLinkedList ( ) ; // find all indexes nested under all the input paths // (which happen to be directories themselves) for ( Path path : inputPaths ) { HdfsUtils . collectPaths ( path , path . getFileSystem ( conf ) , indexDirPathFilter , indexDirs ) ; } // compute the size of each index // and create a single split per index for ( Path indexDir : indexDirs ) { long size = HdfsUtils . getDirectorySize ( indexDir , indexDir . getFileSystem ( conf ) ) ; splits . add ( new LuceneIndexInputSplit ( Lists . newLinkedList ( Arrays . asList ( indexDir ) ) , size ) ) ; } return splits ; }
Finds and creates all the index splits based on the input paths set in conf