public static IndexSummary downsample ( IndexSummary existing , int newSamplingLevel , int minIndexInterval , IPartitioner partitioner ) { // To downsample the old index summary, we'll go through (potentially) several rounds of downsampling. // Conceptually, each round starts at position X and then removes every Nth item. The value of X follows // a particular pattern to evenly space out the items that we remove. The value of N decreases by one each // round. int currentSamplingLevel = existing . getSamplingLevel ( ) ; assert currentSamplingLevel > newSamplingLevel ; assert minIndexInterval == existing . getMinIndexInterval ( ) ; // calculate starting indexes for downsampling rounds int [ ] startPoints = Downsampling . getStartPoints ( currentSamplingLevel , newSamplingLevel ) ; // calculate new off-heap size int newKeyCount = existing . size ( ) ; long newEntriesLength = existing . getEntriesLength ( ) ; for ( int start : startPoints ) { for ( int j = start ; j < existing . size ( ) ; j += currentSamplingLevel ) { newKeyCount -- ; long length = existing . getEndInSummary ( j ) - existing . getPositionInSummary ( j ) ; newEntriesLength -= length ; } } Memory oldEntries = existing . getEntries ( ) ; Memory newOffsets = Memory . allocate ( newKeyCount * 4 ) ; Memory newEntries = Memory . allocate ( newEntriesLength ) ; // Copy old entries to our new Memory. int i = 0 ; int newEntriesOffset = 0 ; outer : for ( int oldSummaryIndex = 0 ; oldSummaryIndex < existing . size ( ) ; oldSummaryIndex ++ ) { // to determine if we can skip this entry, go through the starting points for our downsampling rounds // and see if the entry's index is covered by that round for ( int start : startPoints ) { if ( ( oldSummaryIndex - start ) % currentSamplingLevel == 0 ) continue outer ; } // write the position of the actual entry in the index summary (4 bytes) newOffsets . setInt ( i * 4 , newEntriesOffset ) ; i ++ ; long start = existing . getPositionInSummary ( oldSummaryIndex ) ; long length = existing . getEndInSummary ( oldSummaryIndex ) - start ; newEntries . put ( newEntriesOffset , oldEntries , start , length ) ; newEntriesOffset += length ; } assert newEntriesOffset == newEntriesLength ; return new IndexSummary ( partitioner , newOffsets , newKeyCount , newEntries , newEntriesLength , existing . getMaxNumberOfEntries ( ) , minIndexInterval , newSamplingLevel ) ; }
Downsamples an existing index summary to a new sampling level .