<commit_before>from stash import PickleSerializer
from stash.lib import six


def test_basic():
    serializer = PickleSerializer()

    assert serializer.loads(serializer.dumps(1234)) == 1234
    assert serializer.loads(serializer.dumps('1234')) == '1234'


def test_unicode():
    serializer = PickleSerializer()

    assert serializer.loads(serializer.dumps(six.u('\xee'))) == six.u('\xee')
    assert serializer.loads(serializer.dumps(six.u('\xae'))) == six.u('\xae')


def test_utf8():
    serializer = PickleSerializer()

    assert serializer.loads(serializer.dumps('\xc3\xae')) == '\xc3\xae'
    assert serializer.loads(serializer.dumps('\xc2\xae')) == '\xc2\xae'<commit_msg>Added extra tests for [stash/serializers/s_pickle.py]<commit_after>from stash import PickleSerializer
from stash.lib import six

#
# Pickle v0
#

def test_0_basic():
    serializer = PickleSerializer(protocol=0)

    assert serializer.loads(serializer.dumps(1234)) == 1234
    assert serializer.loads(serializer.dumps('1234')) == '1234'


def test_0_unicode():
    serializer = PickleSerializer(protocol=0)

    assert serializer.loads(serializer.dumps(six.u('\xee'))) == six.u('\xee')
    assert serializer.loads(serializer.dumps(six.u('\xae'))) == six.u('\xae')


def test_0_utf8():
    serializer = PickleSerializer(protocol=0)

    assert serializer.loads(serializer.dumps('\xc3\xae')) == '\xc3\xae'
    assert serializer.loads(serializer.dumps('\xc2\xae')) == '\xc2\xae'


def test_0_escape():
    serializer = PickleSerializer(protocol=0)

    assert serializer.loads(serializer.dumps('\\use')) == '\\use'

#
# Pickle v1
#

def test_1_basic():
    serializer = PickleSerializer(protocol=1)

    assert serializer.loads(serializer.dumps(1234)) == 1234
    assert serializer.loads(serializer.dumps('1234')) == '1234'


def test_1_unicode():
    serializer = PickleSerializer(protocol=1)

    assert serializer.loads(serializer.dumps(six.u('\xee'))) == six.u('\xee')
    assert serializer.loads(serializer.dumps(six.u('\xae'))) == six.u('\xae')


def test_1_utf8():
    serializer = PickleSerializer(protocol=1)

    assert serializer.loads(serializer.dumps('\xc3\xae')) == '\xc3\xae'
    assert serializer.loads(serializer.dumps('\xc2\xae')) == '\xc2\xae'


def test_1_escape():
    serializer = PickleSerializer(protocol=1)

    assert serializer.loads(serializer.dumps('\\use')) == '\\use'

#
# Pickle v2
#

def test_2_basic():
    serializer = PickleSerializer(protocol=2)

    assert serializer.loads(serializer.dumps(1234)) == 1234
    assert serializer.loads(serializer.dumps('1234')) == '1234'


def test_2_unicode():
    serializer = PickleSerializer(protocol=2)

    assert serializer.loads(serializer.dumps(six.u('\xee'))) == six.u('\xee')
    assert serializer.loads(serializer.dumps(six.u('\xae'))) == six.u('\xae')


def test_2_utf8():
    serializer = PickleSerializer(protocol=2)

    assert serializer.loads(serializer.dumps('\xc3\xae')) == '\xc3\xae'
    assert serializer.loads(serializer.dumps('\xc2\xae')) == '\xc2\xae'


def test_2_escape():
    serializer = PickleSerializer(protocol=2)

    assert serializer.loads(serializer.dumps('\\use')) == '\\use'
{'fraction_non_alphanumeric': 0.17077798861480076, 'fraction_numerical': 0.03510436432637571, 'mean_word_length': 4.598230088495575, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4208290', 'n_tokens_mistral': 1177, 'n_tokens_neox': 1148, 'n_words': 166}