private void growPointerArrayIfNecessary ( ) throws IOException { assert ( inMemSorter != null ) ; if ( ! inMemSorter . hasSpaceForAnotherRecord ( ) ) { long used = inMemSorter . getMemoryUsage ( ) ; LongArray array ; try { // could trigger spilling array = allocateArray ( used / 8 * 2 ) ; } catch ( TooLargePageException e ) { // The pointer array is too big to fix in a single page, spill. spill ( ) ; return ; } catch ( SparkOutOfMemoryError e ) { // should have trigger spilling if ( ! inMemSorter . hasSpaceForAnotherRecord ( ) ) { logger . error ( "Unable to grow the pointer array" ) ; throw e ; } return ; } // check if spilling is triggered or not if ( inMemSorter . hasSpaceForAnotherRecord ( ) ) { freeArray ( array ) ; } else { inMemSorter . expandPointerArray ( array ) ; } } }
Checks whether there is enough space to insert an additional record in to the sort pointer array and grows the array if additional space is required . If the required space cannot be obtained then the in - memory data will be spilled to disk .