<commit_before>
from contextlib import closing
import pickle
import unittest
import json

from docido_sdk.toolbox.file_ext import (
    FileProperties,
    iterator_to_file,
)
from docido_sdk.toolbox.http_ext import delayed_request


class TestStreamFromRequest(unittest.TestCase):
    def test_pickle(self):
        s = delayed_request('http://google.com')
        json.dumps(repr(s))
        pickle.dumps(s)

    def test_fetch_google(self):
        with delayed_request('http://google.com').open() as istr:
            istr = iterator_to_file(iter(istr))
            all_content = content = istr.read(50)
            while len(content) == 50:
                content = istr.read(50)
                all_content += content
        self.assertTrue(all_content.endswith('</body></html>'))

        with delayed_request('http://google.com').open() as istr:
            istr = iterator_to_file(iter(istr))
            all_content_at_once = istr.read()
        self.assertTrue(all_content_at_once.endswith('</body></html>'))

    def test_iterate(self):
        content = ['foo', 'bar']
        result = []
        with closing(iterator_to_file(iter(content))) as istr:
            for word in istr:
                result.append(word)
        self.assertEqual(content, result)


class TestFileProperties(unittest.TestCase):
    def test_file_type(self):
        self.assertEquals(FileProperties.file_type('foo.png'), 'image')<commit_msg>add new tests on CustomJSONEncoder<commit_after>
from contextlib import closing
import pickle
import unittest
import json

from docido_sdk.toolbox.file_ext import (
    FileProperties,
    iterator_to_file,
)
from docido_sdk.toolbox.http_ext import delayed_request
from docido_sdk.index.test import CustomJSONEncoder


class TestStreamFromRequest(unittest.TestCase):
    def test_pickle(self):
        s = delayed_request('http://google.com')
        json.dumps(repr(s))
        pickle.dumps(s)

    def test_json(self):
        s = delayed_request('http://google.com', param=dict(test='test'))
        self.assertEqual(
            json.dumps('Test'),
            json.dumps('Test', cls=CustomJSONEncoder)
        )
        self.assertEqual(
            '"{\\"param\\": {\\"test\\": \\"test\\"}}"',
            json.dumps(s, cls=CustomJSONEncoder)
        )

    def test_fetch_google(self):
        with delayed_request('http://google.com').open() as istr:
            istr = iterator_to_file(iter(istr))
            all_content = content = istr.read(50)
            while len(content) == 50:
                content = istr.read(50)
                all_content += content
        self.assertTrue(all_content.endswith('</body></html>'))

        with delayed_request('http://google.com').open() as istr:
            istr = iterator_to_file(iter(istr))
            all_content_at_once = istr.read()
        self.assertTrue(all_content_at_once.endswith('</body></html>'))

    def test_iterate(self):
        content = ['foo', 'bar']
        result = []
        with closing(iterator_to_file(iter(content))) as istr:
            for word in istr:
                result.append(word)
        self.assertEqual(content, result)


class TestFileProperties(unittest.TestCase):
    def test_file_type(self):
        self.assertEquals(FileProperties.file_type('foo.png'), 'image')
{'fraction_non_alphanumeric': 0.12397943755669791, 'fraction_numerical': 0.003628666465074085, 'mean_word_length': 2.708520179372197, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6850768', 'n_tokens_mistral': 1008, 'n_tokens_neox': 985, 'n_words': 180}