public static Map < Writable , Long > sampleMostFrequentFromColumn ( int nMostFrequent , String columnName , Schema schema , JavaRDD < List < Writable > > data ) { int columnIdx = schema . getIndexOfColumn ( columnName ) ; JavaPairRDD < Writable , Long > keyedByWritable = data . mapToPair ( new ColumnToKeyPairTransform ( columnIdx ) ) ; JavaPairRDD < Writable , Long > reducedByWritable = keyedByWritable . reduceByKey ( new SumLongsFunction2 ( ) ) ; List < Tuple2 < Writable , Long > > list = reducedByWritable . takeOrdered ( nMostFrequent , new Tuple2Comparator < Writable > ( false ) ) ; List < Tuple2 < Writable , Long > > sorted = new ArrayList <> ( list ) ; Collections . sort ( sorted , new Tuple2Comparator < Writable > ( false ) ) ; Map < Writable , Long > map = new LinkedHashMap <> ( ) ; for ( Tuple2 < Writable , Long > t2 : sorted ) { map . put ( t2 . _1 ( ) , t2 . _2 ( ) ) ; } return map ; }
Sample the N most frequently occurring values in the specified column