private void tryDenseMode ( ) { if ( numSpillFiles != 0 ) { return ; } long minKey = Long . MAX_VALUE ; long maxKey = Long . MIN_VALUE ; long recordCount = 0 ; for ( LongHashPartition p : this . partitionsBeingBuilt ) { long partitionRecords = p . getBuildSideRecordCount ( ) ; recordCount += partitionRecords ; if ( partitionRecords > 0 ) { if ( p . getMinKey ( ) < minKey ) { minKey = p . getMinKey ( ) ; } if ( p . getMaxKey ( ) > maxKey ) { maxKey = p . getMaxKey ( ) ; } } } if ( buildSpillRetBufferNumbers != 0 ) { throw new RuntimeException ( "buildSpillRetBufferNumbers should be 0: " + buildSpillRetBufferNumbers ) ; } long range = maxKey - minKey + 1 ; if ( range <= recordCount * 4 || range <= segmentSize / 8 ) { // try to request memory. int buffers = ( int ) Math . ceil ( ( ( double ) ( range * 8 ) ) / segmentSize ) ; // TODO MemoryManager needs to support flexible larger segment, so that the index area of the // build side is placed on a segment to avoid the overhead of addressing. MemorySegment [ ] denseBuckets = new MemorySegment [ buffers ] ; for ( int i = 0 ; i < buffers ; i ++ ) { MemorySegment seg = getNextBuffer ( ) ; if ( seg == null ) { returnAll ( Arrays . asList ( denseBuckets ) ) ; return ; } denseBuckets [ i ] = seg ; for ( int j = 0 ; j < segmentSize ; j += 8 ) { seg . putLong ( j , INVALID_ADDRESS ) ; } } denseMode = true ; LOG . info ( "LongHybridHashTable: Use dense mode!" ) ; this . minKey = minKey ; this . maxKey = maxKey ; buildSpillReturnBuffers . drainTo ( availableMemory ) ; ArrayList < MemorySegment > dataBuffers = new ArrayList <> ( ) ; long addressOffset = 0 ; for ( LongHashPartition p : this . partitionsBeingBuilt ) { p . iteratorToDenseBucket ( denseBuckets , addressOffset , minKey ) ; p . updateDenseAddressOffset ( addressOffset ) ; dataBuffers . addAll ( Arrays . asList ( p . getPartitionBuffers ( ) ) ) ; addressOffset += ( p . getPartitionBuffers ( ) . length << segmentSizeBits ) ; returnAll ( Arrays . asList ( p . getBuckets ( ) ) ) ; } this . denseBuckets = denseBuckets ; this . densePartition = new LongHashPartition ( this , buildSideSerializer , dataBuffers . toArray ( new MemorySegment [ dataBuffers . size ( ) ] ) ) ; freeCurrent ( ) ; } }
After build end try to use dense mode .