<commit_before># -*- coding:utf-8 -*-
import unittest
from cStringIO import StringIO
from decimal import Decimal
import threading
from importlib import import_module

from ijson.common import JSONError, IncompleteJSONError, ObjectBuilder, parse, items
from ijson.backends.python import basic_parse

JSON = r'''
{
  "docs": [
    {
      "string": "\u0441\u0442\u0440\u043e\u043a\u0430",
      "null": null,
      "boolean": false,
      "integer": 0,
      "double": 0.5,
      "long": 10000000000,
      "decimal": 10000000000.5
    },
    {
      "meta": [[1], {}]
    },
    {
      "meta": {"key": "value"}
    },
    {
      "meta": null
    }
  ]
}
'''
SCALAR_JSON = '0'
EMPTY_JSON = ''
INVALID_JSON = '{"key": "value",}'
INCOMPLETE_JSON = '"test'
STRINGS_JSON = r'''
{
    "str1": "",
    "str2": "\"",
    "str3": "\\",
    "str4": "\\\\"
}
'''

class Parse(object):
    '''
    Base class for parsing tests that is used to create test cases for each
    available backends.
    '''
    def test_basic_parse(self):
        events = list(self.backend.basic_parse(StringIO(JSON)))
        reference = [
            ('start_map', None),
                ('map_key', 'docs'),
                ('start_array', None),
                    ('start_map', None),
                        ('map_key', 'string'),
                        ('string', u'строка'),
                        ('map_key', 'null'),
                        ('null', None),
                        ('map_key', 'boolean'),
                        ('boolean', False),
                        ('map_key', 'integer'),
                        ('number', 0),
                        ('map_key', 'double'),
                        ('number', Decimal('0.5')),
                        ('map_key', 'long'),
                        ('number', 10000000000),
                        ('map_key', 'decimal'),
                        ('number', Decimal('10000000000.5')),
                    ('end_map', None),
                    ('start_map', None),
                        ('map_key', 'meta'),
                        ('start_array', None),
                            ('start_array', None),
                                ('number', 1),
                            ('end_array', None),
                            ('start_map', None),
                            ('end_map', None),
                        ('end_array', None),
                    ('end_map', None),
                    ('start_map', None),
                        ('map_key', 'meta'),
                        ('start_map', None),
                            ('map_key', 'key'),
                            ('string', 'value'),
                        ('end_map', None),
                    ('end_map', None),
                    ('start_map', None),
                        ('map_key', 'meta'),
                        ('null', None),
                    ('end_map', None),
                ('end_array', None),
            ('end_map', None),
        ]
        for e, r in zip(events, reference):
            self.assertEqual(e, r)

    def test_basic_parse_threaded(self):
        thread = threading.Thread(target=self.test_basic_parse)
        thread.start()
        thread.join()

    def test_scalar(self):
        events = list(self.backend.basic_parse(StringIO(SCALAR_JSON)))
        self.assertEqual(events, [('number', 0)])

    def test_strings(self):
        events = list(self.backend.basic_parse(StringIO(STRINGS_JSON)))
        strings = [value for event, value in events if event == 'string']
        self.assertEqual(strings, ['', '"', '\\', '\\\\'])

    def test_empty(self):
        self.assertRaises(
            IncompleteJSONError,
            lambda: list(self.backend.basic_parse(StringIO(EMPTY_JSON))),
        )

    def test_incomplete(self):
        self.assertRaises(
            IncompleteJSONError,
            lambda: list(self.backend.basic_parse(StringIO(INCOMPLETE_JSON))),
        )

    def test_invalid(self):
        self.assertRaises(
            JSONError,
            lambda: list(self.backend.basic_parse(StringIO(INVALID_JSON))),
        )

    def test_lazy(self):
        # shouldn't fail since iterator is not exhausted
        self.backend.basic_parse(StringIO(INVALID_JSON))
        self.assertTrue(True)

# Generating real TestCase classes for each importable backend
for name in ['python', 'yajl', 'yajl2']:
    try:
        classname = '%sParse' % name.capitalize()
        locals()[classname] = type(
            classname,
            (unittest.TestCase, Parse),
            {'backend': import_module('ijson.backends.%s' % name)},
        )
    except ImportError:
        pass

class Common(unittest.TestCase):
    def test_object_builder(self):
        builder = ObjectBuilder()
        for event, value in basic_parse(StringIO(JSON)):
            builder.event(event, value)
        self.assertEqual(builder.value, {
            'docs': [
                {
                   'string': u'строка',
                   'null': None,
                   'boolean': False,
                   'integer': 0,
                   'double': Decimal('0.5'),
                   'long': 10000000000,
                   'decimal': Decimal('10000000000.5'),
                },
                {
                    'meta': [[1], {}],
                },
                {
                    'meta': {'key': 'value'},
                },
                {
                    'meta': None,
                },
            ],
        })

    def test_scalar_builder(self):
        builder = ObjectBuilder()
        for event, value in basic_parse(StringIO(SCALAR_JSON)):
            builder.event(event, value)
        self.assertEqual(builder.value, 0)

    def test_parse(self):
        events = parse(basic_parse(StringIO(JSON)))
        events = [value
            for prefix, event, value in events
            if prefix == 'docs.item.meta.item.item'
        ]
        self.assertEqual(events, [1])

    def test_items(self):
        meta = list(items(parse(basic_parse(StringIO(JSON))), 'docs.item.meta'))
        self.assertEqual(meta, [
            [[1], {}],
            {'key': 'value'},
            None,
        ])


if __name__ == '__main__':
    unittest.main()
<commit_msg>Docstring for Common<commit_after># -*- coding:utf-8 -*-
import unittest
from cStringIO import StringIO
from decimal import Decimal
import threading
from importlib import import_module

from ijson.common import JSONError, IncompleteJSONError, ObjectBuilder, parse, items
from ijson.backends.python import basic_parse

JSON = r'''
{
  "docs": [
    {
      "string": "\u0441\u0442\u0440\u043e\u043a\u0430",
      "null": null,
      "boolean": false,
      "integer": 0,
      "double": 0.5,
      "long": 10000000000,
      "decimal": 10000000000.5
    },
    {
      "meta": [[1], {}]
    },
    {
      "meta": {"key": "value"}
    },
    {
      "meta": null
    }
  ]
}
'''
SCALAR_JSON = '0'
EMPTY_JSON = ''
INVALID_JSON = '{"key": "value",}'
INCOMPLETE_JSON = '"test'
STRINGS_JSON = r'''
{
    "str1": "",
    "str2": "\"",
    "str3": "\\",
    "str4": "\\\\"
}
'''

class Parse(object):
    '''
    Base class for parsing tests that is used to create test cases for each
    available backends.
    '''
    def test_basic_parse(self):
        events = list(self.backend.basic_parse(StringIO(JSON)))
        reference = [
            ('start_map', None),
                ('map_key', 'docs'),
                ('start_array', None),
                    ('start_map', None),
                        ('map_key', 'string'),
                        ('string', u'строка'),
                        ('map_key', 'null'),
                        ('null', None),
                        ('map_key', 'boolean'),
                        ('boolean', False),
                        ('map_key', 'integer'),
                        ('number', 0),
                        ('map_key', 'double'),
                        ('number', Decimal('0.5')),
                        ('map_key', 'long'),
                        ('number', 10000000000),
                        ('map_key', 'decimal'),
                        ('number', Decimal('10000000000.5')),
                    ('end_map', None),
                    ('start_map', None),
                        ('map_key', 'meta'),
                        ('start_array', None),
                            ('start_array', None),
                                ('number', 1),
                            ('end_array', None),
                            ('start_map', None),
                            ('end_map', None),
                        ('end_array', None),
                    ('end_map', None),
                    ('start_map', None),
                        ('map_key', 'meta'),
                        ('start_map', None),
                            ('map_key', 'key'),
                            ('string', 'value'),
                        ('end_map', None),
                    ('end_map', None),
                    ('start_map', None),
                        ('map_key', 'meta'),
                        ('null', None),
                    ('end_map', None),
                ('end_array', None),
            ('end_map', None),
        ]
        for e, r in zip(events, reference):
            self.assertEqual(e, r)

    def test_basic_parse_threaded(self):
        thread = threading.Thread(target=self.test_basic_parse)
        thread.start()
        thread.join()

    def test_scalar(self):
        events = list(self.backend.basic_parse(StringIO(SCALAR_JSON)))
        self.assertEqual(events, [('number', 0)])

    def test_strings(self):
        events = list(self.backend.basic_parse(StringIO(STRINGS_JSON)))
        strings = [value for event, value in events if event == 'string']
        self.assertEqual(strings, ['', '"', '\\', '\\\\'])

    def test_empty(self):
        self.assertRaises(
            IncompleteJSONError,
            lambda: list(self.backend.basic_parse(StringIO(EMPTY_JSON))),
        )

    def test_incomplete(self):
        self.assertRaises(
            IncompleteJSONError,
            lambda: list(self.backend.basic_parse(StringIO(INCOMPLETE_JSON))),
        )

    def test_invalid(self):
        self.assertRaises(
            JSONError,
            lambda: list(self.backend.basic_parse(StringIO(INVALID_JSON))),
        )

    def test_lazy(self):
        # shouldn't fail since iterator is not exhausted
        self.backend.basic_parse(StringIO(INVALID_JSON))
        self.assertTrue(True)

# Generating real TestCase classes for each importable backend
for name in ['python', 'yajl', 'yajl2']:
    try:
        classname = '%sParse' % name.capitalize()
        locals()[classname] = type(
            classname,
            (unittest.TestCase, Parse),
            {'backend': import_module('ijson.backends.%s' % name)},
        )
    except ImportError:
        pass

class Common(unittest.TestCase):
    '''
    Backend independent tests. They all use basic_parse imported explicitly from
    the python backend to generate parsing events.
    '''
    def test_object_builder(self):
        builder = ObjectBuilder()
        for event, value in basic_parse(StringIO(JSON)):
            builder.event(event, value)
        self.assertEqual(builder.value, {
            'docs': [
                {
                   'string': u'строка',
                   'null': None,
                   'boolean': False,
                   'integer': 0,
                   'double': Decimal('0.5'),
                   'long': 10000000000,
                   'decimal': Decimal('10000000000.5'),
                },
                {
                    'meta': [[1], {}],
                },
                {
                    'meta': {'key': 'value'},
                },
                {
                    'meta': None,
                },
            ],
        })

    def test_scalar_builder(self):
        builder = ObjectBuilder()
        for event, value in basic_parse(StringIO(SCALAR_JSON)):
            builder.event(event, value)
        self.assertEqual(builder.value, 0)

    def test_parse(self):
        events = parse(basic_parse(StringIO(JSON)))
        events = [value
            for prefix, event, value in events
            if prefix == 'docs.item.meta.item.item'
        ]
        self.assertEqual(events, [1])

    def test_items(self):
        meta = list(items(parse(basic_parse(StringIO(JSON))), 'docs.item.meta'))
        self.assertEqual(meta, [
            [[1], {}],
            {'key': 'value'},
            None,
        ])


if __name__ == '__main__':
    unittest.main()

{'fraction_non_alphanumeric': 0.14241633040588655, 'fraction_numerical': 0.018039401851412295, 'mean_word_length': 1.4755189972581277, 'pattern_counts': {'":': 34, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 12, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '151860', 'n_tokens_mistral': 3695, 'n_tokens_neox': 3433, 'n_words': 717}