<commit_before>from numpy import arange, squeeze, vstack, repeat, asarray, ones

import pytest

from bolt.common import allclose
from bolt import array


def test_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.shape == x.shape

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc)
    assert b.shape == x.shape


def test_size(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.size == x.size


def test_split(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.split == 1

    b = array(x, sc, axes=(0, 1))
    assert b.split == 2


def test_mask(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.mask == (1, 0, 0)

    b = array(x, sc, axes=(0, 1))
    assert b.mask == (1, 1, 0)

    b = array(x, sc, axes=(0, 1, 2))
    assert b.mask == (1, 1, 1)


def test_value_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.values.shape == (3,)

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.values.shape == (3, 4)


def test_key_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.keys.shape == (2,)

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0, 1))
    assert b.keys.shape == (2, 3)


def test_reshape_keys(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    c = b.keys.reshape((3, 2))
    assert c.keys.shape == (3, 2)
    assert allclose(c.toarray(), x.reshape((3, 2, 4)))

    b = array(x, sc, axes=(0,))
    c = b.keys.reshape((2, 1))
    assert allclose(c.toarray(), x.reshape((2, 1, 3, 4)))

    b = array(x, sc, axes=(0,))
    c = b.keys.reshape((2,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.keys.reshape((2, 3))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.keys.reshape((2, 3, 4))


def test_reshape_values(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    c = b.values.reshape((4, 3))
    assert c.values.shape == (4, 3)
    assert allclose(c.toarray(), x.reshape((2, 4, 3)))

    b = array(x, sc, axes=(0, 1))
    c = b.values.reshape((1, 4))
    assert allclose(c.toarray(), x.reshape((2, 3, 1, 4)))

    b = array(x, sc, axes=(0, 1))
    c = b.values.reshape((4,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0,))
    c = b.values.reshape((3, 4))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.values.reshape((2, 3, 4))


def test_transpose_keys(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    c = b.keys.transpose((1, 0))
    assert c.keys.shape == (3, 2)
    assert allclose(c.toarray(), x.transpose((1, 0, 2)))

    b = array(x, sc, axes=(0,))
    c = b.keys.transpose((0,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.keys.transpose((0, 1))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.keys.transpose((0, 2))

    with pytest.raises(ValueError):
        b.keys.transpose((1, 1))

    with pytest.raises(ValueError):
        b.keys.transpose((0,))


def test_transpose_values(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    c = b.values.transpose((1, 0))
    assert c.values.shape == (4, 3)
    assert allclose(c.toarray(), x.transpose((0, 2, 1)))

    b = array(x, sc, axes=(0,))
    c = b.values.transpose((0, 1))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.values.transpose((0,))
    assert allclose(c.toarray(), x.reshape((2, 3, 4)))

    b = array(x, sc, axes=(0,))
    with pytest.raises(ValueError):
        b.values.transpose((0, 2))

    with pytest.raises(ValueError):
        b.values.transpose((1, 1))

    with pytest.raises(ValueError):
        b.values.transpose((0,))

"""
Blockable interface tests
"""

def _2D_blockable_preamble(sc, num_partitions=2):
    dims = (10, 10)
    arr = vstack([[x]*dims[1] for x in arange(dims[0])])
    barr = barray(arr, sc, split=1)
    barr = BoltArraySpark(barr._rdd.partitionBy(num_partitions),
            shape=barr.shape, split=barr.split)
    return barr


def _3D_blockable_preamble(sc, num_partitions=2):
    dims = (10, 10, 10)
    area = dims[0] * dims[1]
    arr = asarray([repeat(x,area).reshape(dims[0], dims[1]) for x in range(dims[2])])
    barr = barray(arr, sc, split=1)
    barr = BoltArraySpark(barr._rdd.partitionBy(num_partitions),
            shape=barr.shape, split=barr.split)
    return barr


def test_block_2D(sc):

    barr = _2D_blockable_preamble(sc)

    # Without block_size
    blocked = barr.blocked()
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (5, 10)
    assert blocked._barray.shape == (10, 10)

    # With block_size
    blocked = barr.blocked(block_size=2)
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (2, 10)

    # Invalid block_size
    blocked = barr.blocked(block_size=0)
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (5, 10)

    # Unblocking
    unblocked = blocked.unblock()
    arr = unblocked.toarray()
    assert arr.shape == (10, 10)
    assert allclose(arr, barr.toarray())


def test_block_3D(sc):

    barr = _3D_blockable_preamble(sc)

    # With block_size
    blocked = barr.blocked(block_size=2)
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (2, 10, 10)

    # Invalid block_size
    blocked = barr.blocked(block_size=0)
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (5, 10, 10)

    # Unblocking
    unblocked = blocked.unblock()
    arr = unblocked.toarray()
    assert arr.shape == (10, 10, 10)
    assert allclose(arr, barr.toarray())


def test_blocked_map(sc):

    barr = _2D_blockable_preamble(sc)

    map_func1 = lambda x: x * 2
    map_func2 = lambda x: ones(10)

    funcs = [map_func1, map_func2]

    for func in funcs:
        blocked = barr.blocked()
        blocked_map = blocked.map(func)
        normal_map = barr.map(func)
        unblocked = blocked_map.unblock()
        assert normal_map.shape == unblocked.shape
        assert normal_map.split == unblocked.split
        assert allclose(normal_map.toarray(), unblocked.toarray())


def test_blocked_reduce(sc):

    from numpy import max

    barr = _2D_blockable_preamble(sc)

    reduce_func1 = lambda x,y: max(x, y)

    funcs = [reduce_func1]

    for func in funcs:
        blocked = barr.blocked()
        blocked_reduce = blocked.reduce(func)
        normal_reduce = barr.reduce(func)
        unblocked = blocked_map.unblock()
        assert normal_map.shape == unblocked.shape
        assert normal_map.split == unblocked.split
        assert allclose(normal_map.toarray(), unblocked.toarray())


def test_getitem_slice(sc):

    x = arange(2*3).reshape((2, 3))

    b = array(x, sc, axes=(0,))
    assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
    assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
    assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
    assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
    assert allclose(b[:2, :2].toarray(), x[:2, :2])
    assert allclose(b[1:, 1:].toarray(), x[1:, 1:])

    b = array(x, sc, axes=(0, 1))
    assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
    assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
    assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
    assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
    assert allclose(b[:2, :2].toarray(), x[:2, :2])
    assert allclose(b[1:, 1:].toarray(), x[1:, 1:])

def test_getitem_slice_ragged(sc):

    x = arange(10*10*3).reshape((10, 10, 3))

    b = array(x, sc, axes=(0,1))
    assert allclose(b[0:5:2, 0:2].toarray(), x[0:5:2, 0:2])
    assert allclose(b[0:5:3, 0:2].toarray(), x[0:5:3, 0:2])
    assert allclose(b[0:9:3, 0:2].toarray(), x[0:9:3, 0:2])

def test_getitem_int(sc):

    x = arange(2*3).reshape((2, 3))

    b = array(x, sc, axes=(0,))
    assert allclose(b[0, 0].toarray(), x[0, 0])
    assert allclose(b[0, 1].toarray(), x[0, 1])
    assert allclose(b[0, 0:1].toarray(), x[0, 0:1])
    assert allclose(b[1, 2].toarray(), x[1, 2])
    assert allclose(b[[1], [2]].toarray(), x[[1], [2]])

    b = array(x, sc, axes=(0, 1))
    assert allclose(b[0, 0].toarray(), x[0, 0])
    assert allclose(b[0, 1].toarray(), x[0, 1])
    assert allclose(b[0, 0:1].toarray(), x[0, 0:1])
    assert allclose(b[1, 2].toarray(), x[1, 2])
    assert allclose(b[[1], [2]].toarray(), x[[1], [2]])

def test_getitem_list(sc):

    x = arange(3*3*4).reshape((3, 3, 4))

    b = array(x, sc, axes=(0,))
    assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
    assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
    assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])

    b = array(x, sc, axes=(0,1))
    assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
    assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
    assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])

def test_getitem_list_array(sc):

    x = arange(3*3*4).reshape((3, 3, 4))

    rows = [[0, 0], [1, 1]]
    cols = [[0, 2], [0, 2]]
    dept = [[0, 3], [0, 3]]

    b = array(x, sc, axes=(0,))
    assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])

    b = array(x, sc, axes=(0,1))
    assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])
<commit_msg>Updated blockable unit tests<commit_after>from numpy import arange, squeeze, vstack, repeat, asarray, ones

import pytest

from bolt.common import allclose
from bolt import array


def test_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.shape == x.shape

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc)
    assert b.shape == x.shape


def test_size(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.size == x.size


def test_split(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.split == 1

    b = array(x, sc, axes=(0, 1))
    assert b.split == 2


def test_mask(sc):

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.mask == (1, 0, 0)

    b = array(x, sc, axes=(0, 1))
    assert b.mask == (1, 1, 0)

    b = array(x, sc, axes=(0, 1, 2))
    assert b.mask == (1, 1, 1)


def test_value_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.values.shape == (3,)

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0,))
    assert b.values.shape == (3, 4)


def test_key_shape(sc):

    x = arange(2*3).reshape((2, 3))
    b = array(x, sc)
    assert b.keys.shape == (2,)

    x = arange(2*3*4).reshape((2, 3, 4))
    b = array(x, sc, axes=(0, 1))
    assert b.keys.shape == (2, 3)


def test_reshape_keys(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    c = b.keys.reshape((3, 2))
    assert c.keys.shape == (3, 2)
    assert allclose(c.toarray(), x.reshape((3, 2, 4)))

    b = array(x, sc, axes=(0,))
    c = b.keys.reshape((2, 1))
    assert allclose(c.toarray(), x.reshape((2, 1, 3, 4)))

    b = array(x, sc, axes=(0,))
    c = b.keys.reshape((2,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.keys.reshape((2, 3))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.keys.reshape((2, 3, 4))


def test_reshape_values(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    c = b.values.reshape((4, 3))
    assert c.values.shape == (4, 3)
    assert allclose(c.toarray(), x.reshape((2, 4, 3)))

    b = array(x, sc, axes=(0, 1))
    c = b.values.reshape((1, 4))
    assert allclose(c.toarray(), x.reshape((2, 3, 1, 4)))

    b = array(x, sc, axes=(0, 1))
    c = b.values.reshape((4,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0,))
    c = b.values.reshape((3, 4))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.values.reshape((2, 3, 4))


def test_transpose_keys(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0, 1))
    c = b.keys.transpose((1, 0))
    assert c.keys.shape == (3, 2)
    assert allclose(c.toarray(), x.transpose((1, 0, 2)))

    b = array(x, sc, axes=(0,))
    c = b.keys.transpose((0,))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.keys.transpose((0, 1))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    with pytest.raises(ValueError):
        b.keys.transpose((0, 2))

    with pytest.raises(ValueError):
        b.keys.transpose((1, 1))

    with pytest.raises(ValueError):
        b.keys.transpose((0,))


def test_transpose_values(sc):

    x = arange(2*3*4).reshape((2, 3, 4))

    b = array(x, sc, axes=(0,))
    c = b.values.transpose((1, 0))
    assert c.values.shape == (4, 3)
    assert allclose(c.toarray(), x.transpose((0, 2, 1)))

    b = array(x, sc, axes=(0,))
    c = b.values.transpose((0, 1))
    assert allclose(c.toarray(), x)

    b = array(x, sc, axes=(0, 1))
    c = b.values.transpose((0,))
    assert allclose(c.toarray(), x.reshape((2, 3, 4)))

    b = array(x, sc, axes=(0,))
    with pytest.raises(ValueError):
        b.values.transpose((0, 2))

    with pytest.raises(ValueError):
        b.values.transpose((1, 1))

    with pytest.raises(ValueError):
        b.values.transpose((0,))

"""
Blockable interface tests
"""

def _2D_blockable_preamble(sc, num_partitions=2):
    dims = (10, 10)
    arr = vstack([[x]*dims[1] for x in arange(dims[0])])
    barr = array(arr, sc, split=1)
    barr = BoltArraySpark(barr._rdd.partitionBy(num_partitions),
            shape=barr.shape, split=barr.split)
    return barr


def _3D_blockable_preamble(sc, num_partitions=2):
    dims = (10, 10, 10)
    area = dims[0] * dims[1]
    arr = asarray([repeat(x,area).reshape(dims[0], dims[1]) for x in range(dims[2])])
    barr = array(arr, sc, split=1)
    barr = BoltArraySpark(barr._rdd.partitionBy(num_partitions),
            shape=barr.shape, split=barr.split)
    return barr


def test_block_2D(sc):

    barr = _2D_blockable_preamble(sc)

    # Without block_size
    blocked = barr.blocked()
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (5, 10)
    assert blocked._barray.shape == (10, 10)

    # With block_size
    blocked = barr.blocked(block_size=2)
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (2, 10)

    # Invalid block_size
    blocked = barr.blocked(block_size=0)
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (5, 10)

    # Unblocking
    unblocked = blocked.unblock()
    arr = unblocked.toarray()
    assert arr.shape == (10, 10)
    assert allclose(arr, barr.toarray())


def test_block_3D(sc):

    barr = _3D_blockable_preamble(sc)

    # With block_size
    blocked = barr.blocked(block_size=2)
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (2, 10, 10)

    # Invalid block_size
    blocked = barr.blocked(block_size=0)
    first_partition = blocked._barray._rdd.first()[1]
    assert first_partition.shape == (5, 10, 10)

    # Unblocking
    unblocked = blocked.unblock()
    arr = unblocked.toarray()
    assert arr.shape == (10, 10, 10)
    assert allclose(arr, barr.toarray())


def test_blocked_map(sc):

    barr = _2D_blockable_preamble(sc)

    map_func1 = lambda x: x * 2
    map_func2 = lambda x: ones(10)

    funcs = [map_func1, map_func2]

    for func in funcs:
        blocked = barr.blocked()
        blocked_map = blocked.map(func)
        normal_map = barr.map(func)
        unblocked = blocked_map.unblock()
        assert normal_map.shape == unblocked.shape
        assert normal_map.split == unblocked.split
        assert allclose(normal_map.toarray(), unblocked.toarray())


def test_blocked_reduce(sc):

    from numpy import max

    barr = _2D_blockable_preamble(sc)

    reduce_func1 = lambda x,y: max(x, y)

    funcs = [reduce_func1]

    for func in funcs:
        blocked = barr.blocked()
        blocked_reduce = blocked.reduce(func)
        normal_reduce = barr.reduce(func)
        unblocked = blocked_map.unblock()
        assert normal_map.shape == unblocked.shape
        assert normal_map.split == unblocked.split
        assert allclose(normal_map.toarray(), unblocked.toarray())


def test_getitem_slice(sc):

    x = arange(2*3).reshape((2, 3))

    b = array(x, sc, axes=(0,))
    assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
    assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
    assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
    assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
    assert allclose(b[:2, :2].toarray(), x[:2, :2])
    assert allclose(b[1:, 1:].toarray(), x[1:, 1:])

    b = array(x, sc, axes=(0, 1))
    assert allclose(b[0:1, 0:1].toarray(), x[0:1, 0:1])
    assert allclose(b[0:2, 0:2].toarray(), x[0:2, 0:2])
    assert allclose(b[0:2, 0:3].toarray(), x[0:2, 0:3])
    assert allclose(b[0:2, 0:3:2].toarray(), x[0:2, 0:3:2])
    assert allclose(b[:2, :2].toarray(), x[:2, :2])
    assert allclose(b[1:, 1:].toarray(), x[1:, 1:])

def test_getitem_slice_ragged(sc):

    x = arange(10*10*3).reshape((10, 10, 3))

    b = array(x, sc, axes=(0,1))
    assert allclose(b[0:5:2, 0:2].toarray(), x[0:5:2, 0:2])
    assert allclose(b[0:5:3, 0:2].toarray(), x[0:5:3, 0:2])
    assert allclose(b[0:9:3, 0:2].toarray(), x[0:9:3, 0:2])

def test_getitem_int(sc):

    x = arange(2*3).reshape((2, 3))

    b = array(x, sc, axes=(0,))
    assert allclose(b[0, 0].toarray(), x[0, 0])
    assert allclose(b[0, 1].toarray(), x[0, 1])
    assert allclose(b[0, 0:1].toarray(), x[0, 0:1])
    assert allclose(b[1, 2].toarray(), x[1, 2])
    assert allclose(b[[1], [2]].toarray(), x[[1], [2]])

    b = array(x, sc, axes=(0, 1))
    assert allclose(b[0, 0].toarray(), x[0, 0])
    assert allclose(b[0, 1].toarray(), x[0, 1])
    assert allclose(b[0, 0:1].toarray(), x[0, 0:1])
    assert allclose(b[1, 2].toarray(), x[1, 2])
    assert allclose(b[[1], [2]].toarray(), x[[1], [2]])

def test_getitem_list(sc):

    x = arange(3*3*4).reshape((3, 3, 4))

    b = array(x, sc, axes=(0,))
    assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
    assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
    assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])

    b = array(x, sc, axes=(0,1))
    assert allclose(b[[0, 1], [0, 1], [0, 2]].toarray(), x[[0, 1], [0, 1], [0, 2]])
    assert allclose(b[[0, 1], [0, 2], [0, 3]].toarray(), x[[0, 1], [0, 2], [0, 3]])
    assert allclose(b[[0, 1, 2], [0, 2, 1], [0, 3, 1]].toarray(), x[[0, 1, 2], [0, 2, 1], [0, 3, 1]])

def test_getitem_list_array(sc):

    x = arange(3*3*4).reshape((3, 3, 4))

    rows = [[0, 0], [1, 1]]
    cols = [[0, 2], [0, 2]]
    dept = [[0, 3], [0, 3]]

    b = array(x, sc, axes=(0,))
    assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])

    b = array(x, sc, axes=(0,1))
    assert allclose(b[rows, cols, dept].toarray(), x[rows, cols, dept])

{'fraction_non_alphanumeric': 0.20614809954065924, 'fraction_numerical': 0.05845237494321336, 'mean_word_length': 3.112079701120797, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1326641', 'n_tokens_mistral': 8971, 'n_tokens_neox': 8328, 'n_words': 1865}