def test_disk_shuffle_check_actual_compression():

    def generate_raw_partd_file(compression):
        df1 = pd.DataFrame({'a': list(range(10000))})
        df1['b'] = (df1['a'] * 123).astype(str)
        with dask.config.set({'dataframe.shuffle-compression': compression}):
            p1 = maybe_buffered_partd(buffer=False, tempdir=None)()
            p1.append({'x': df1})
            filename = (p1.partd.partd.filename('x') if compression else p1.partd.filename('x'))
            with open(filename, 'rb') as f:
                return f.read()
    uncompressed_data = generate_raw_partd_file(compression=None)
    compressed_data = generate_raw_partd_file(compression='BZ2')
    assert (len(uncompressed_data) > len(compressed_data))
def test_disk_shuffle_check_actual_compression():

    def generate_raw_partd_file(compression):
        df1 = pd.DataFrame({'a': list(range(10000))})
        df1['b'] = (df1['a'] * 123).astype(str)
        with dask.config.set({'dataframe.shuffle-compression': compression}):
            p1 = maybe_buffered_partd(buffer=False, tempdir=None)()
            p1.append({'x': df1})
            filename = (p1.partd.partd.filename('x') if compression else p1.partd.filename('x'))
            with open(filename, 'rb') as f:
                return f.read()
    uncompressed_data = generate_raw_partd_file(compression=None)
    compressed_data = generate_raw_partd_file(compression='BZ2')
    assert (len(uncompressed_data) > len(compressed_data))