<commit_before># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from __future__ import print_function

import json
import pytest
import sys

# I feel a little bad that I have absolutely no idea
# what these extractions mean. I've taken them from
# jpf.go.jp (japan foundation) and BBC arabic

unicode_basic = {
    'test_string': u'国際交流基',
    'test_num': 12334.0,
    'test_arr': [u'言葉を超えた共感', '123', u'また', 'four'],
    'test_obj': {'this': u'سيتبعها الرئيس الأمريكي المنتخب،', u'غرينتش': 1000},
    'test_empty_arr': [],
    'test_empty_obj': {}
}

ascii_basic = {
    "test_string": "abc123",
    "test_num": 12334.0,
    "test_arr": ["abc", "123", "four", "five"],
    "test_obj": {"this": "is a very basic", "object": 1000},
    "test_empty_arr": [],
    "test_empty_obj": {}
}


@pytest.fixture(params=[ascii_basic, unicode_basic], ids=["ascii", "unicode"])
def testdata(request):
    return json.dumps(request.param), request.param

@pytest.fixture(
    params=['json_cherry_pick.py_jcp', 'json_cherry_pick.c_jcp'],
    ids=['PyJCP', 'CJCP']
)
def jcp(request):
    mod = __import__(request.param)
    for submod in request.param.split('.')[1:]:
        mod = getattr(mod, submod)
    return mod

class TestExtraction(object):

    def test_extract_string(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_string') == expected['test_string']

    def test_extract_num(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_num') == expected['test_num']

    def test_extract_arr(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_arr') == expected['test_arr']

    def test_pluck_list(self, testdata, jcp):
        base, expected = testdata
        assert jcp.pluck_list(base, 'test_arr', 2) == expected['test_arr'][2]

    def test_pluck_end_of_list(self, testdata, jcp):
        base, expected = testdata
        idx = len(expected['test_arr']) - 1
        assert jcp.pluck_list(base, 'test_arr', idx) == expected['test_arr'][idx]

    def test_extract_obj(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_obj') == expected['test_obj']

    def test_extract_tokenized(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_obj.this') == expected['test_obj']['this']

    def test_extract_empty_arr(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_empty_arr') == expected['test_empty_arr']

    def test_extract_empty_obj(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_empty_obj') == expected['test_empty_obj']

    def test_extract_unicode_key(self, jcp):
        base, expected = json.dumps(unicode_basic), unicode_basic
        s = u'غرينتش'.encode("utf-8") if sys.version_info < (3, 0) else u'غرينتش'
        print(json.dumps(u'غرينتش'))
        assert jcp.extract(base, s) == expected['test_obj'][u'غرينتش']


class TestExistence(object):
    
    def test_key_exists(self, testdata, jcp):
        base, expected = testdata
        for key in expected.keys():
            assert jcp.key_exists(base, key)

    def test_key_not_exist(self, testdata, jcp):
        base = testdata[0]
        assert not jcp.key_exists(base, 'invalid_key')

    def test_key_not_exist_utf8(self, testdata, jcp):
        base = testdata[0]
        s = u'また'.encode('utf-8') if sys.version_info < (3, 0) else u'また'
        assert not jcp.key_exists(base, s)

    def test_subkey_exists(self, testdata, jcp):
        base, expected = testdata
        for key in expected['test_obj'].keys():
            s = '.'.join(['test_obj', key]).encode('utf-8')
            assert jcp.key_exists(base, s)

    def test_subkey_out_of_scope(self, testdata, jcp):
        base = testdata[0]
        assert not jcp.key_exists(base, 'test_arr.this')

        <commit_msg>Py2->3 Unicode is a pita<commit_after># -*- coding: utf-8 -*-
from __future__ import unicode_literals
from __future__ import print_function

import json
import pytest
import sys

# I feel a little bad that I have absolutely no idea
# what these extractions mean. I've taken them from
# jpf.go.jp (japan foundation) and BBC arabic

unicode_basic = {
    'test_string': u'国際交流基',
    'test_num': 12334.0,
    'test_arr': [u'言葉を超えた共感', '123', u'また', 'four'],
    'test_obj': {'this': u'سيتبعها الرئيس الأمريكي المنتخب،', u'غرينتش': 1000},
    'test_empty_arr': [],
    'test_empty_obj': {}
}

ascii_basic = {
    "test_string": "abc123",
    "test_num": 12334.0,
    "test_arr": ["abc", "123", "four", "five"],
    "test_obj": {"this": "is a very basic", "object": 1000},
    "test_empty_arr": [],
    "test_empty_obj": {}
}


@pytest.fixture(params=[ascii_basic, unicode_basic], ids=["ascii", "unicode"])
def testdata(request):
    return json.dumps(request.param), request.param

@pytest.fixture(
    params=['json_cherry_pick.py_jcp', 'json_cherry_pick.c_jcp'],
    ids=['PyJCP', 'CJCP']
)
def jcp(request):
    mod = __import__(request.param)
    for submod in request.param.split('.')[1:]:
        mod = getattr(mod, submod)
    return mod

class TestExtraction(object):

    def test_extract_string(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_string') == expected['test_string']

    def test_extract_num(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_num') == expected['test_num']

    def test_extract_arr(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_arr') == expected['test_arr']

    def test_pluck_list(self, testdata, jcp):
        base, expected = testdata
        assert jcp.pluck_list(base, 'test_arr', 2) == expected['test_arr'][2]

    def test_pluck_end_of_list(self, testdata, jcp):
        base, expected = testdata
        idx = len(expected['test_arr']) - 1
        assert jcp.pluck_list(base, 'test_arr', idx) == expected['test_arr'][idx]

    def test_extract_obj(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_obj') == expected['test_obj']

    def test_extract_tokenized(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_obj.this') == expected['test_obj']['this']

    def test_extract_empty_arr(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_empty_arr') == expected['test_empty_arr']

    def test_extract_empty_obj(self, testdata, jcp):
        base, expected = testdata
        assert jcp.extract(base, 'test_empty_obj') == expected['test_empty_obj']

    def test_extract_unicode_key(self, jcp):
        base, expected = json.dumps(unicode_basic), unicode_basic
        s = u'غرينتش'.encode("utf-8") if sys.version_info < (3, 0) else u'غرينتش'
        print(json.dumps(u'غرينتش'))
        assert jcp.extract(base, s) == expected['test_obj'][u'غرينتش']


class TestExistence(object):
    
    def test_key_exists(self, testdata, jcp):
        base, expected = testdata
        for key in expected.keys():
            assert jcp.key_exists(base, key)

    def test_key_not_exist(self, testdata, jcp):
        base = testdata[0]
        assert not jcp.key_exists(base, 'invalid_key')

    def test_key_not_exist_utf8(self, testdata, jcp):
        base = testdata[0]
        s = u'また'.encode('utf-8') if sys.version_info < (3, 0) else u'また'
        assert not jcp.key_exists(base, s)

    def test_subkey_exists(self, testdata, jcp):
        base, expected = testdata
        for key in expected['test_obj'].keys():
            s = '.'.join(['test_obj', key])
            s = s.encode('utf-8') if sys.version_info < (3, 0) else s
            assert jcp.key_exists(base, s)

    def test_subkey_out_of_scope(self, testdata, jcp):
        base = testdata[0]
        assert not jcp.key_exists(base, 'test_arr.this')

        
{'fraction_non_alphanumeric': 0.1406620861961274, 'fraction_numerical': 0.011742660836976889, 'mean_word_length': 3.40131940626718, 'pattern_counts': {'":': 16, '<': 8, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2643316', 'n_tokens_mistral': 2921, 'n_tokens_neox': 2814, 'n_words': 624}