public void cumSumWithinPartition ( ) { // Accumulator to get the max of the cumulative sum in each partition final Accumulator < Counter < Integer > > maxPerPartitionAcc = sc . accumulator ( new Counter < Integer > ( ) , new MaxPerPartitionAccumulator ( ) ) ; // Partition mapping to fold within partition foldWithinPartitionRDD = sentenceCountRDD . mapPartitionsWithIndex ( new FoldWithinPartitionFunction ( maxPerPartitionAcc ) , true ) . cache ( ) ; actionForMapPartition ( foldWithinPartitionRDD ) ; // Broadcast the counter (partition index : sum of count) to all workers broadcastedMaxPerPartitionCounter = sc . broadcast ( maxPerPartitionAcc . value ( ) ) ; }
Do cum sum within the partition