<jupyter_start><jupyter_text>Setup<jupyter_code>import numpy as np
import numcodecs
numcodecs.__version__<jupyter_output><empty_output><jupyter_text>Unicode string benchmarks<jupyter_code>import fastparquet


class FastParquetCodec(numcodecs.abc.Codec):
    """Hacked codec using fastparquet utf8 encoding, for benchmarking purposes only."""
    
    codec_id = 'xxx-fastparquet'
    
    def encode(self, buf):
        buf = np.asanyarray(buf)
        n = buf.size
        ba = fastparquet.speedups.array_encode_utf8(buf)
        enc = fastparquet.speedups.pack_byte_array(ba.tolist())
        return n, enc  # hack for now, return n
    
    def decode(self, data, out=None):
        n, enc = data
        ba = fastparquet.speedups.unpack_byte_array(enc, n)
        dec = fastparquet.speedups.array_decode_utf8(np.array(ba, dtype=object))
        if out is not None:
            out[:] = dec
            return out
        return dec
zstd1 = numcodecs.Zstd(1)
zstd5 = numcodecs.Zstd(5)
zstd9 = numcodecs.Zstd(9)


def benchmark_codec(codec, a):
    print(codec)
    print('encode')
    %timeit codec.encode(a)
    enc = codec.encode(a)
    print('decode')
    %timeit codec.decode(enc)
    if isinstance(codec, FastParquetCodec):
        enc = enc[1]  # hack
    print('size         : {:,}'.format(len(enc)))
    print('size (zstd 1): {:,}'.format(len(zstd1.encode(enc))))
    print('size (zstd 5): {:,}'.format(len(zstd5.encode(enc))))
    print('size (zstd 9): {:,}'.format(len(zstd9.encode(enc))))

from numcodecs.tests.common import greetings
msgpack_codec = numcodecs.MsgPack()
json_codec = numcodecs.JSON()
pickle_codec = numcodecs.Pickle()
cat_codec = numcodecs.Categorize(greetings, dtype=object, astype='u1')
vlen_codec = numcodecs.VLenUTF8()
fp_codec = FastParquetCodec()<jupyter_output><empty_output><jupyter_text>Greetings benchmark<jupyter_code>np.random.seed(42)
data = np.random.choice(greetings, size=1000000).astype(object)
data
%time enc = vlen_codec.encode(data)
%time dec = vlen_codec.decode(enc)
benchmark_codec(msgpack_codec, data)
benchmark_codec(json_codec, data)
benchmark_codec(pickle_codec, data)
benchmark_codec(cat_codec, data)
benchmark_codec(vlen_codec, data)
benchmark_codec(fp_codec, data)<jupyter_output>FastParquetCodec()
encode
107 ms ± 497 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
decode
223 ms ± 4.44 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
size         : 21,830,271
size (zstd 1): 1,762,809
size (zstd 5): 1,546,612
size (zstd 9): 1,358,813<jupyter_text>Lorem benchmark<jupyter_code>from faker import Faker
fake = Faker()
data2 = np.array(' '.join(fake.sentences(nb=200000)).split(), dtype=object)
len(data2), data2[:10]
benchmark_codec(msgpack_codec, data2)
benchmark_codec(json_codec, data2)
benchmark_codec(pickle_codec, data2)
benchmark_codec(vlen_codec, data2)
benchmark_codec(fp_codec, data2)<jupyter_output>FastParquetCodec()
encode
80.4 ms ± 841 µs per loop (mean ± std. dev. of 7 runs, 10 loops each)
decode
148 ms ± 1.08 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
size         : 11,680,588
size (zstd 1): 2,956,144
size (zstd 5): 2,859,941
size (zstd 9): 2,555,154<jupyter_text>Byte strings benchmark<jupyter_code>vlen_bytes_codec = numcodecs.VLenBytes()
np.random.seed(42)
greetings_bytes = [g.encode('utf-8') for g in greetings]
data3 = np.random.choice(greetings_bytes, size=1000000).astype(object)
data3
vlen_bytes_codec.decode(vlen_bytes_codec.encode(data3))
benchmark_codec(pickle_codec, data3)
benchmark_codec(vlen_bytes_codec, data3)<jupyter_output>VLenBytes()
encode
14.2 ms ± 76.6 µs per loop (mean ± std. dev. of 7 runs, 100 loops each)
decode
45.5 ms ± 2.38 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
size         : 21,830,275
size (zstd 1): 1,762,784
size (zstd 5): 1,546,616
size (zstd 9): 1,358,840<jupyter_text>Array benchmarks<jupyter_code>np.random.seed(42)
data4 = np.array([np.random.randint(0, 100, size=np.random.randint(0, 20)).astype('i4')
                  for i in range(100000)], dtype=object)
data4
vlen_arr_codec = numcodecs.VLenArray('<i4')
vlen_arr_codec.decode(vlen_arr_codec.encode(data4))
benchmark_codec(vlen_arr_codec, data4)
benchmark_codec(pickle_codec, data4)<jupyter_output>Pickle(protocol=4)
encode
311 ms ± 6.38 ms per loop (mean ± std. dev. of 7 runs, 1 loop each)
decode
126 ms ± 1.36 ms per loop (mean ± std. dev. of 7 runs, 10 loops each)
size         : 6,791,830
size (zstd 1): 1,716,068
size (zstd 5): 1,566,619
size (zstd 9): 1,488,721
{'fraction_non_alphanumeric': 0.12962138084632516, 'fraction_numerical': 0.0621380846325167, 'mean_word_length': 4.692015209125476, 'pattern_counts': {'":': 0, '<': 22, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '234627', 'license': 'MIT', 'n_tokens_mistral': 1892, 'n_tokens_neox': 1689, 'n_words': 406, 'path': 'notebooks/benchmark_vlen.ipynb', 'repo_name': 'tushar-deepsource/numcodecs', 'size': '4490'}