<commit_before># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import unittest
from collections import Iterable
from io import StringIO
import mock
import socket

from duckduckpy.core import api
from duckduckpy.core import Hook
from duckduckpy.core import query
from duckduckpy.core import secure_query
from duckduckpy.core import url_assembler
import duckduckpy.exception as exc
from duckduckpy.utils import camel_to_snake_case


class TestHook(unittest.TestCase):
    def test_non_existent_hook(self):
        self.assertTrue(Hook(1) is None)

    def test_hook_instance_returned(self):
        hook = Hook(Hook.containers[0])
        self.assertTrue(isinstance(hook, Hook))

    def test_containers_exist(self):
        self.assertTrue(isinstance(Hook.containers, Iterable))
        self.assertTrue(Hook.containers > 0)

    def test_no_object_found(self):
        obj = {
            'URL': 'www.test.url.com',
            'SomeThing': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        expected = {
            'URL': 'www.test.url.com',
            'SomeThing': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        hook = Hook('dict')
        actual_dict = hook(obj)
        self.assertEqual(actual_dict, expected)


class TestHookExceptions(unittest.TestCase):
    def test_non_existent_hook_verbose(self):
        self.assertRaises(exc.DuckDuckDeserializeError, Hook, 1, verbose=True)

    def test_no_object_found_verbose(self):
        obj = {
            'URL': 'www.test.url.com',
            'SomeThing': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        hook = Hook('dict', verbose=True)
        self.assertRaises(exc.DuckDuckDeserializeError, hook, obj)


class TestHookDictSerializer(unittest.TestCase):
    def setUp(self):
        self.hook = Hook('dict')
        self.assertEqual(self.hook._container, 'dict')

    def test_deserialize_icon(self):
        obj = {'URL': 'www.some.url.com', 'Width': 10, 'Height': 10}
        expected = {'url': 'www.some.url.com', 'width': 10, 'height': 10}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_result(self):
        obj = {
            'FirstURL': 'www.test.url.com',
            'Icon': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        expected = {
            'first_url': 'www.test.url.com',
            'icon': 'icon',
            'result': 20,
            'text': 'Example of a text'}
        actual_dict = self.hook(obj)
        self.assertEqual(actual_dict, expected)

    def test_deserialize_content(self):
        obj = {
            'data_type': 'test_type',
            'label': 'test_label',
            'sort_order': 'order',
            'value': False,
            'wiki_order': 0}
        expected = {
            'data_type': 'test_type',
            'label': 'test_label',
            'sort_order': 'order',
            'value': False,
            'wiki_order': 0}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_meta(self):
        obj = {'data_type': 'test_type', 'label': True, 'value': 10}
        expected = {'data_type': 'test_type', 'label': True, 'value': 10}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_infobox(self):
        obj = {'content': 'test content', 'meta': True}
        expected = {'content': 'test content', 'meta': True}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_response(self):
        obj = {
            'Redirect': 'YES',
            'Definition': 'def',
            'ImageWidth': 20,
            'Infobox': True,
            'RelatedTopics': ['Topics', 'list'],
            'ImageHeight': 45,
            'Heading': 'HEADING',
            'Answer': 'answer',
            'AbstractText': 'test text',
            'Type': 'yes',
            'ImageIsLogo': 'logo',
            'DefinitionSource': 'test source',
            'AbstractURL': 'http://test.abstract.url.com',
            'Abstract': True,
            'DefinitionURL': 'http://test.definition.url.com',
            'Results': [True, 'results', 'list'],
            'Entity': 'test entity',
            'AnswerType': 'True answer',
            'AbstractSource': 'test abstract source',
            'Image': 'test image'}
        expected = {
            'redirect': 'YES',
            'definition': 'def',
            'image_width': 20,
            'infobox': True,
            'related_topics': ['Topics', 'list'],
            'image_height': 45,
            'heading': 'HEADING',
            'answer': 'answer',
            'abstract_text': 'test text',
            'type': 'yes',
            'image_is_logo': 'logo',
            'definition_source': 'test source',
            'abstract_url': 'http://test.abstract.url.com',
            'abstract': True,
            'definition_url': 'http://test.definition.url.com',
            'results': [True, 'results', 'list'],
            'entity': 'test entity',
            'answer_type': 'True answer',
            'abstract_source': 'test abstract source',
            'image': 'test image'}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)


class TestHookNamedTupleSerializer(unittest.TestCase):
    def setUp(self):
        self.hook = Hook('namedtuple')
        self.assertEqual(self.hook._container, 'namedtuple')

    def test_deserialize_icon(self):
        obj = {'URL': 'www.some.url.com', 'Width': 10, 'Height': 10}
        expected_dict = {'url': 'www.some.url.com', 'width': 10, 'height': 10}
        expected = api.Icon(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_result(self):
        obj = {
            'FirstURL': 'www.test.url.com',
            'Icon': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        expected_dict = {
            'first_url': 'www.test.url.com',
            'icon': 'icon',
            'result': 20,
            'text': 'Example of a text'}
        expected = api.Result(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_content(self):
        obj = {
            'data_type': 'test_type',
            'label': 'test_label',
            'sort_order': 'order',
            'value': False,
            'wiki_order': 0}
        expected_dict = {
            'data_type': 'test_type',
            'label': 'test_label',
            'sort_order': 'order',
            'value': False,
            'wiki_order': 0}
        expected = api.Content(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_meta(self):
        obj = {'data_type': 'test_type', 'label': True, 'value': 10}
        expected_dict = {'data_type': 'test_type', 'label': True, 'value': 10}
        expected = api.Meta(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_infobox(self):
        obj = {'content': 'test content', 'meta': True}
        expected_dict = {'content': 'test content', 'meta': True}
        expected = api.Infobox(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_response(self):
        obj = {
            'Redirect': 'YES',
            'Definition': 'def',
            'ImageWidth': 20,
            'Infobox': True,
            'RelatedTopics': ['Topics', 'list'],
            'ImageHeight': 45,
            'Heading': 'HEADING',
            'Answer': 'answer',
            'AbstractText': 'test text',
            'Type': 'yes',
            'ImageIsLogo': 'logo',
            'DefinitionSource': 'test source',
            'AbstractURL': 'http://test.abstract.url.com',
            'Abstract': True,
            'DefinitionURL': 'http://test.definition.url.com',
            'Results': [True, 'results', 'list'],
            'Entity': 'test entity',
            'AnswerType': 'True answer',
            'AbstractSource': 'test abstract source',
            'Image': 'test image'}
        expected_dict = {
            'redirect': 'YES',
            'definition': 'def',
            'image_width': 20,
            'infobox': True,
            'related_topics': ['Topics', 'list'],
            'image_height': 45,
            'heading': 'HEADING',
            'answer': 'answer',
            'abstract_text': 'test text',
            'type': 'yes',
            'image_is_logo': 'logo',
            'definition_source': 'test source',
            'abstract_url': 'http://test.abstract.url.com',
            'abstract': True,
            'definition_url': 'http://test.definition.url.com',
            'results': [True, 'results', 'list'],
            'entity': 'test entity',
            'answer_type': 'True answer',
            'abstract_source': 'test abstract source',
            'image': 'test image'}
        expected = api.Response(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)


class TestCamelToSnakeCase(unittest.TestCase):
    def assertConverted(self, camel_case_string, snake_case_string):
        self.assertEqual(snake_case_string,
                         camel_to_snake_case(camel_case_string))

    def test_conversion_one_word(self):
        self.assertConverted('Camel', 'camel')

    def test_conversion_two_words(self):
        self.assertConverted('CamelCase', 'camel_case')

    def test_conversion_three_words(self):
        self.assertConverted('CamelCaseToo', 'camel_case_too')

    def test_conversion_three_mixed(self):
        self.assertConverted('CamelCase_Underscore', 'camel_case__underscore')

    def test_conversion_with_numbers(self):
        self.assertConverted('Camel2CaseToo3', 'camel2_case_too3')

    def test_conversion_equal(self):
        self.assertConverted('camel_case', 'camel_case')

    def test_conversion_mixed_case(self):
        self.assertConverted('getHTTPResponseCode', 'get_http_response_code')

    def test_conversion_mixed_case_with_numbers(self):
        self.assertConverted('get2HTTPResponseCode', 'get2_http_response_code')

    def test_conversion_uppercase_one_after_another(self):
        self.assertConverted('HTTPResponseCode', 'http_response_code')


class TestURLAssembler(unittest.TestCase):
    def test_simple(self):
        expected = "/?q=test+query&format=json"
        self.assertEqual(url_assembler("test query"), expected)

    def test_cyrillic(self):
        expected = (
            "/?q=%D0%A1%D0%BB%D0%B0%D0%B2%D0%B0+%D0%"
            "A3%D0%BA%D1%80%D0%B0%D1%97%D0%BD%D1%96&format=json")
        url = url_assembler('Слава Україні')
        self.assertEqual(url, expected)

    def test_no_redirect(self):
        expected = "/?q=test+query&no_redirect=1&format=json"
        url = url_assembler("test query", no_redirect=True)
        self.assertEqual(url, expected)

    def test_no_html(self):
        expected = "/?q=test+query&no_html=1&format=json"
        url = url_assembler("test query", no_html=True)
        self.assertEqual(url, expected)

    def test_skip_disambig(self):
        expected = "/?q=test+query&skip_disambig=1&format=json"
        url = url_assembler("test query", skip_disambig=True)
        self.assertEqual(url, expected)

    def test_all_options_are_on(self):
        expected = ("/?q=test+query&no_html=1&"
                    "no_redirect=1&skip_disambig=1&format=json")
        url = url_assembler("test query",
                            no_redirect=True, no_html=True, skip_disambig=True)
        self.assertEqual(url, expected)


class TestQuery(unittest.TestCase):
    origin = """{
   \"DefinitionSource\" : \"\",
   \"Heading\" : \"Python\",
   \"ImageWidth\" : 0,
   \"RelatedTopics\" : [
      {
         \"Result\" : \"<a href=\\"https://duckduckgo.com/Python_(programmin\",
         \"Icon\" : {
            \"URL\" : \"https://duckduckgo.com/i/7eec482b.png\",
            \"Height\" : \"\",
            \"Width\" : \"\"
         },
         \"FirstURL\" : \"https://duckduckgo.com/Python_(programming\",
         \"Text\" : \"Python (programming language)A widely used general\"
      },
      {
         \"Result\" : \"<a href=\\"https://duckduckgo.com/Monty_Python\\">Mo\",
         \"Icon\" : {
            \"URL\" : \"https://duckduckgo.com/i/4eec9e83.jpg\",
            \"Height\" : \"\",
            \"Width\" : \"\"
         },
         \"FirstURL\" : \"https://duckduckgo.com/Monty_Python\",
         \"Text\" : \"Monty PythonA British surreal comedy group who\"
      },
      {
         \"Result\" : \"<a href=\\"https://duckduckgo.com/Colt_Python\\">Co\",
         \"Icon\" : {
            \"URL\" : \"https://duckduckgo.com/i/7e29c05b.jpg\",<commit_msg>Patch httplib.HTTPConnection.request<commit_after># -*- coding: utf-8 -*-

from __future__ import unicode_literals
import unittest
from collections import Iterable
from io import StringIO
import mock
import socket

from duckduckpy.core import api
from duckduckpy.core import Hook
from duckduckpy.core import query
from duckduckpy.core import secure_query
from duckduckpy.core import url_assembler
import duckduckpy.exception as exc
from duckduckpy.utils import camel_to_snake_case


class TestHook(unittest.TestCase):
    def test_non_existent_hook(self):
        self.assertTrue(Hook(1) is None)

    def test_hook_instance_returned(self):
        hook = Hook(Hook.containers[0])
        self.assertTrue(isinstance(hook, Hook))

    def test_containers_exist(self):
        self.assertTrue(isinstance(Hook.containers, Iterable))
        self.assertTrue(Hook.containers > 0)

    def test_no_object_found(self):
        obj = {
            'URL': 'www.test.url.com',
            'SomeThing': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        expected = {
            'URL': 'www.test.url.com',
            'SomeThing': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        hook = Hook('dict')
        actual_dict = hook(obj)
        self.assertEqual(actual_dict, expected)


class TestHookExceptions(unittest.TestCase):
    def test_non_existent_hook_verbose(self):
        self.assertRaises(exc.DuckDuckDeserializeError, Hook, 1, verbose=True)

    def test_no_object_found_verbose(self):
        obj = {
            'URL': 'www.test.url.com',
            'SomeThing': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        hook = Hook('dict', verbose=True)
        self.assertRaises(exc.DuckDuckDeserializeError, hook, obj)


class TestHookDictSerializer(unittest.TestCase):
    def setUp(self):
        self.hook = Hook('dict')
        self.assertEqual(self.hook._container, 'dict')

    def test_deserialize_icon(self):
        obj = {'URL': 'www.some.url.com', 'Width': 10, 'Height': 10}
        expected = {'url': 'www.some.url.com', 'width': 10, 'height': 10}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_result(self):
        obj = {
            'FirstURL': 'www.test.url.com',
            'Icon': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        expected = {
            'first_url': 'www.test.url.com',
            'icon': 'icon',
            'result': 20,
            'text': 'Example of a text'}
        actual_dict = self.hook(obj)
        self.assertEqual(actual_dict, expected)

    def test_deserialize_content(self):
        obj = {
            'data_type': 'test_type',
            'label': 'test_label',
            'sort_order': 'order',
            'value': False,
            'wiki_order': 0}
        expected = {
            'data_type': 'test_type',
            'label': 'test_label',
            'sort_order': 'order',
            'value': False,
            'wiki_order': 0}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_meta(self):
        obj = {'data_type': 'test_type', 'label': True, 'value': 10}
        expected = {'data_type': 'test_type', 'label': True, 'value': 10}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_infobox(self):
        obj = {'content': 'test content', 'meta': True}
        expected = {'content': 'test content', 'meta': True}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_response(self):
        obj = {
            'Redirect': 'YES',
            'Definition': 'def',
            'ImageWidth': 20,
            'Infobox': True,
            'RelatedTopics': ['Topics', 'list'],
            'ImageHeight': 45,
            'Heading': 'HEADING',
            'Answer': 'answer',
            'AbstractText': 'test text',
            'Type': 'yes',
            'ImageIsLogo': 'logo',
            'DefinitionSource': 'test source',
            'AbstractURL': 'http://test.abstract.url.com',
            'Abstract': True,
            'DefinitionURL': 'http://test.definition.url.com',
            'Results': [True, 'results', 'list'],
            'Entity': 'test entity',
            'AnswerType': 'True answer',
            'AbstractSource': 'test abstract source',
            'Image': 'test image'}
        expected = {
            'redirect': 'YES',
            'definition': 'def',
            'image_width': 20,
            'infobox': True,
            'related_topics': ['Topics', 'list'],
            'image_height': 45,
            'heading': 'HEADING',
            'answer': 'answer',
            'abstract_text': 'test text',
            'type': 'yes',
            'image_is_logo': 'logo',
            'definition_source': 'test source',
            'abstract_url': 'http://test.abstract.url.com',
            'abstract': True,
            'definition_url': 'http://test.definition.url.com',
            'results': [True, 'results', 'list'],
            'entity': 'test entity',
            'answer_type': 'True answer',
            'abstract_source': 'test abstract source',
            'image': 'test image'}
        actual = self.hook(obj)
        self.assertEqual(actual, expected)


class TestHookNamedTupleSerializer(unittest.TestCase):
    def setUp(self):
        self.hook = Hook('namedtuple')
        self.assertEqual(self.hook._container, 'namedtuple')

    def test_deserialize_icon(self):
        obj = {'URL': 'www.some.url.com', 'Width': 10, 'Height': 10}
        expected_dict = {'url': 'www.some.url.com', 'width': 10, 'height': 10}
        expected = api.Icon(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_result(self):
        obj = {
            'FirstURL': 'www.test.url.com',
            'Icon': 'icon',
            'Result': 20,
            'Text': 'Example of a text'}
        expected_dict = {
            'first_url': 'www.test.url.com',
            'icon': 'icon',
            'result': 20,
            'text': 'Example of a text'}
        expected = api.Result(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_content(self):
        obj = {
            'data_type': 'test_type',
            'label': 'test_label',
            'sort_order': 'order',
            'value': False,
            'wiki_order': 0}
        expected_dict = {
            'data_type': 'test_type',
            'label': 'test_label',
            'sort_order': 'order',
            'value': False,
            'wiki_order': 0}
        expected = api.Content(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_meta(self):
        obj = {'data_type': 'test_type', 'label': True, 'value': 10}
        expected_dict = {'data_type': 'test_type', 'label': True, 'value': 10}
        expected = api.Meta(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_infobox(self):
        obj = {'content': 'test content', 'meta': True}
        expected_dict = {'content': 'test content', 'meta': True}
        expected = api.Infobox(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)

    def test_deserialize_response(self):
        obj = {
            'Redirect': 'YES',
            'Definition': 'def',
            'ImageWidth': 20,
            'Infobox': True,
            'RelatedTopics': ['Topics', 'list'],
            'ImageHeight': 45,
            'Heading': 'HEADING',
            'Answer': 'answer',
            'AbstractText': 'test text',
            'Type': 'yes',
            'ImageIsLogo': 'logo',
            'DefinitionSource': 'test source',
            'AbstractURL': 'http://test.abstract.url.com',
            'Abstract': True,
            'DefinitionURL': 'http://test.definition.url.<EMAIL>',
            'Results': [True, 'results', 'list'],
            'Entity': 'test entity',
            'AnswerType': 'True answer',
            'AbstractSource': 'test abstract source',
            'Image': 'test image'}
        expected_dict = {
            'redirect': 'YES',
            'definition': 'def',
            'image_width': 20,
            'infobox': True,
            'related_topics': ['Topics', 'list'],
            'image_height': 45,
            'heading': 'HEADING',
            'answer': 'answer',
            'abstract_text': 'test text',
            'type': 'yes',
            'image_is_logo': 'logo',
            'definition_source': 'test source',
            'abstract_url': 'http://test.abstract.url.com',
            'abstract': True,
            'definition_url': 'http://test.definition.url.com',
            'results': [True, 'results', 'list'],
            'entity': 'test entity',
            'answer_type': 'True answer',
            'abstract_source': 'test abstract source',
            'image': 'test image'}
        expected = api.Response(**expected_dict)
        actual = self.hook(obj)
        self.assertEqual(actual, expected)


class TestCamelToSnakeCase(unittest.TestCase):
    def assertConverted(self, camel_case_string, snake_case_string):
        self.assertEqual(snake_case_string,
                         camel_to_snake_case(camel_case_string))

    def test_conversion_one_word(self):
        self.assertConverted('Camel', 'camel')

    def test_conversion_two_words(self):
        self.assertConverted('CamelCase', 'camel_case')

    def test_conversion_three_words(self):
        self.assertConverted('CamelCaseToo', 'camel_case_too')

    def test_conversion_three_mixed(self):
        self.assertConverted('CamelCase_Underscore', 'camel_case__underscore')

    def test_conversion_with_numbers(self):
        self.assertConverted('Camel2CaseToo3', 'camel2_case_too3')

    def test_conversion_equal(self):
        self.assertConverted('camel_case', 'camel_case')

    def test_conversion_mixed_case(self):
        self.assertConverted('getHTTPResponseCode', 'get_http_response_code')

    def test_conversion_mixed_case_with_numbers(self):
        self.assertConverted('get2HTTPResponseCode', 'get2_http_response_code')

    def test_conversion_uppercase_one_after_another(self):
        self.assertConverted('HTTPResponseCode', 'http_response_code')


class TestURLAssembler(unittest.TestCase):
    def test_simple(self):
        expected = "/?q=test+query&format=json"
        self.assertEqual(url_assembler("test query"), expected)

    def test_cyrillic(self):
        expected = (
            "/?q=%D0%A1%D0%BB%D0%B0%D0%B2%D0%B0+%D0%"
            "A3%D0%BA%D1%80%D0%B0%D1%97%D0%BD%D1%96&format=json")
        url = url_assembler('Слава Україні')
        self.assertEqual(url, expected)

    def test_no_redirect(self):
        expected = "/?q=test+query&no_redirect=1&format=json"
        url = url_assembler("test query", no_redirect=True)
        self.assertEqual(url, expected)

    def test_no_html(self):
        expected = "/?q=test+query&no_html=1&format=json"
        url = url_assembler("test query", no_html=True)
        self.assertEqual(url, expected)

    def test_skip_disambig(self):
        expected = "/?q=test+query&skip_disambig=1&format=json"
        url = url_assembler("test query", skip_disambig=True)
        self.assertEqual(url, expected)

    def test_all_options_are_on(self):
        expected = ("/?q=test+query&no_html=1&"
                    "no_redirect=1&skip_disambig=1&format=json")
        url = url_assembler("test query",
                            no_redirect=True, no_html=True, skip_disambig=True)
        self.assertEqual(url, expected)


@mock.patch('httplib.HTTPConnection.request')
class TestQuery(unittest.TestCase):
    origin = """{
   \"DefinitionSource\" : \"\",
   \"Heading\" : \"Python\",
   \"ImageWidth\" : 0,
   \"RelatedTopics\" : [
      {
         \"Result\" : \"<a href=\\"https://duckduckgo.com/Python_(programmin\",
         \"Icon\" : {
            \"URL\" : \"https://duckduckgo.com/i/7eec482b.png\",
            \"Height\" : \"\",
            \"Width\" : \"\"
         },
         \"FirstURL\" : \"https://duckduckgo.com/Python_(programming\",
         \"Text\" : \"Python (programming language)A widely used general\"
      },
      {
         \"Result\" : \"<a href=\\"https://duckduckgo.com/Monty_Python\\">Mo\",
         \"Icon\" : {
            \"URL\" : \"https://duckduckgo.com/i/4eec9e83.jpg\",
            \"Height\" : \"\",
            \"Width\" : \"\"
         },
         \"FirstURL\" : \"https://duckduckgo.com/Monty_Python\",
         \"Text\" : \"Monty PythonA British surreal comedy group who\"
      },
      {
         \"Result\" : \"<a href=\\"https://duckduckgo.com/Colt_Python\\">Co\",
         \"Icon\" : {
            \"URL\" : \"https://duckduckgo.com/i/7e29c05b.jpg\",
{'fraction_non_alphanumeric': 0.14914002077802146, 'fraction_numerical': 0.008695986763630767, 'mean_word_length': 2.2309796121332672, 'pattern_counts': {'":': 0, '<': 10, '<?xml version=': 0, '>': 10, 'https://': 16, 'lorem ipsum': 0, 'www.': 22, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2902428', 'n_tokens_mistral': 8045, 'n_tokens_neox': 7767, 'n_words': 1521}