<commit_before>"""Tests for file serializers"""
import io
try:
    from unittest import mock
except ImportError:
    import mock

import numpy as np
import pytest

import omf


@pytest.mark.parametrize('arr', [
    [1, 2, 3],
    np.array(['a', 'b', 'c']),
])
def test_bad_array_serializer(arr):
    """Test array serializer failures"""
    with pytest.raises(ValueError):
        omf.serializers.array_serializer(arr, {})


@pytest.mark.parametrize(('arr', 'dtype'), [
    (np.array([1., 2., 3.]), '<f8'),
    (np.array([np.nan, 2., 3.]), '<f8'),
    (np.array([1, 2, 3]), '<i8'),
])
def test_array_serializer(arr, dtype):
    """Test array serializer creates correct output index"""
    binary_dict = {}
    output = omf.serializers.array_serializer(arr, binary_dict)
    assert output['start'] == 0
    assert output['dtype'] == dtype
    assert output['length'] == 24
    assert output['array'] in binary_dict

def test_array_serializer_none():
    """Test array serializer when value is None"""
    output = omf.serializers.array_serializer(None, {})
    assert output is None


def test_bad_shape():
    """Test bad shape for array deserializer"""
    with pytest.raises(TypeError):
        omf.serializers.array_deserializer(['*', '*'])


@mock.patch('omf.serializers.np.frombuffer')
@mock.patch('omf.serializers.zlib.decompress')
@pytest.mark.parametrize(('dtype', 'shape'), [
    ('int', [3, 3]),
    ('<i8', [5]),
    ('<i8', [10]),
    ('<i8', [4, '*']),
])
def test_bad_deserialize(mock_decompress, mock_frombuffer, dtype, shape):
    """Test expected errors during deserialization"""
    mock_decompress.return_value = None
    mock_frombuffer.return_value = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9])
    deserializer = omf.serializers.array_deserializer(shape)
    index = {
        'dtype': dtype,
        'start': 0,
        'length': 0,
        'array': 'a',
    }
    with pytest.raises(AssertionError):
        deserializer(index, {'a': None})


@pytest.mark.parametrize('arr', [
    np.random.randint(100, size=[4, 5, 6]),
    np.random.randint(100, size=[3]),
    np.random.rand(4, 5, 6),
    np.array([0.5, 1., np.nan, np.nan, 2.5]),
])<commit_msg>Fix serializer tests for removed zlib import<commit_after>"""Tests for file serializers"""
import io
try:
    from unittest import mock
except ImportError:
    import mock

import numpy as np
import pytest

import omf


@pytest.mark.parametrize('arr', [
    [1, 2, 3],
    np.array(['a', 'b', 'c']),
])
def test_bad_array_serializer(arr):
    """Test array serializer failures"""
    with pytest.raises(ValueError):
        omf.serializers.array_serializer(arr, {})


@pytest.mark.parametrize(('arr', 'dtype'), [
    (np.array([1., 2., 3.]), '<f8'),
    (np.array([np.nan, 2., 3.]), '<f8'),
    (np.array([1, 2, 3]), '<i8'),
])
def test_array_serializer(arr, dtype):
    """Test array serializer creates correct output index"""
    binary_dict = {}
    output = omf.serializers.array_serializer(arr, binary_dict)
    assert output['start'] == 0
    assert output['dtype'] == dtype
    assert output['length'] == 24
    assert output['array'] in binary_dict

def test_array_serializer_none():
    """Test array serializer when value is None"""
    output = omf.serializers.array_serializer(None, {})
    assert output is None


def test_bad_shape():
    """Test bad shape for array deserializer"""
    with pytest.raises(TypeError):
        omf.serializers.array_deserializer(['*', '*'])


@mock.patch('omf.serializers.np.frombuffer')
@pytest.mark.parametrize(('dtype', 'shape'), [
    ('int', [3, 3]),
    ('<i8', [5]),
    ('<i8', [10]),
    ('<i8', [4, '*']),
])
def test_bad_deserialize(mock_frombuffer, dtype, shape):
    """Test expected errors during deserialization"""
    mock_frombuffer.return_value = np.array([1, 2, 3, 4, 5, 6, 7, 8, 9])
    deserializer = omf.serializers.array_deserializer(shape)
    index = {
        'dtype': dtype,
        'start': 0,
        'length': 0,
        'array': 'a',
    }
    with pytest.raises(AssertionError):
        deserializer(index, {'a': None})


@pytest.mark.parametrize('arr', [
    np.random.randint(100, size=[4, 5, 6]),
    np.random.randint(100, size=[3]),
    np.random.rand(4, 5, 6),
    np.array([0.5, 1., np.nan, np.nan, 2.5]),
])
{'fraction_non_alphanumeric': 0.18455036393519605, 'fraction_numerical': 0.02582765907490021, 'mean_word_length': 3.635473340587595, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5293271', 'n_tokens_mistral': 1578, 'n_tokens_neox': 1521, 'n_words': 346}