<commit_before>#!/usr/bin/env python3

from mimic import Mimic
import unittest

class TestMimic(unittest.TestCase):

    key_index = 0
    value_index = 1
    expected_result_index = 2

    test_datas = [
        ['crown',0,'William'],
        ['play',3,'croquet'],
    ]


    def setUp(self):
        pass


    def test_mimic_dict(self):
        self.mimic = Mimic('alice.txt')

        for test_data in self.test_datas:
            result = self.mimic.mimic_dict()[test_data[self.key_index]][test_data[self.value_index]]
            expected_result = test_data[self.expected_result_index]
            self.assertEqual(expected_result, result,
                             'mimic_dict()[{}][{}] expected {} but got {}'.format(test_data[self.key_index],
                                                                              test_data[self.value_index],
                                                                              expected_result,
                                                                              result))


    def test_mimic_dict_keys(self):
        self.mimic = Mimic('alice.txt')

        self.assertTrue('mustard' in self.mimic.mimic_dict())
        self.assertTrue('we' in self.mimic.mimic_dict())


if __name__ == "__main__": unittest.main()<commit_msg>In test-mimic.py test_mimic_dict_keys add case dictionary contains key 'we\'ve'. Currently this test fails.<commit_after>#!/usr/bin/env python3

from mimic import Mimic
import unittest

class TestMimic(unittest.TestCase):

    key_index = 0
    value_index = 1
    expected_result_index = 2

    test_datas = [
        ['crown',0,'William'],
        ['play',3,'croquet'],
    ]


    def setUp(self):
        pass


    def test_mimic_dict(self):
        self.mimic = Mimic('alice.txt')

        for test_data in self.test_datas:
            result = self.mimic.mimic_dict()[test_data[self.key_index]][test_data[self.value_index]]
            expected_result = test_data[self.expected_result_index]
            self.assertEqual(expected_result, result,
                             'mimic_dict()[{}][{}] expected {} but got {}'.format(test_data[self.key_index],
                                                                              test_data[self.value_index],
                                                                              expected_result,
                                                                              result))


    def test_mimic_dict_keys(self):
        self.mimic = Mimic('alice.txt')

        self.assertTrue('mustard' in self.mimic.mimic_dict())
        self.assertTrue('we' in self.mimic.mimic_dict())

        # test contracted words have been escaped
        self.assertTrue('we\'ve' in self.mimic.mimic_dict())


if __name__ == "__main__": unittest.main()
{'fraction_non_alphanumeric': 0.10627674750356633, 'fraction_numerical': 0.0042796005706134095, 'mean_word_length': 1.6587677725118484, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2533771', 'n_tokens_mistral': 819, 'n_tokens_neox': 783, 'n_words': 136}