<commit_before>from lexos.helpers.general_functions import get_encoding, make_preview_from, \
    generate_d3_object, merge_list, load_stastic, matrix_to_dict, \
    dict_to_matrix, html_escape, apply_function_exclude_tags, decode_bytes
import unittest


class TestGeneralFunctions(unittest.TestCase):
    def test_get_encoding(self):
        assert get_encoding(b"asdf") == "ascii"

    def test_make_preview_from(self):
        one_char = "x"
        less_than_500_char = "modgaecq"
        str_250 = "<KEY>" \
                  "<KEY>" \
                  "<KEY>" \
                  "<KEY>" \
                  "gccbardglcnedcqqxduuaauzyv"
        str_500 = str_250 + str_250
        more_than_500_char_even = \
            str_250 + less_than_500_char + less_than_500_char + str_250
        more_than_500_char_odd = \
            str_250 + less_than_500_char + one_char + less_than_500_char + \
            str_250
        middle = '\u2026 ' + '/n/n' + '\u2026'
        assert make_preview_from(less_than_500_char) == less_than_500_char
        assert make_preview_from(str_500) == str_500
        assert make_preview_from(
            more_than_500_char_even) == \
               str_250 + less_than_500_char + middle + less_than_500_char + \
               str_250

        assert make_preview_from(
            more_than_500_char_odd) == \
               str_250 + less_than_500_char + middle + one_char + str_250 + \
               less_than_500_char

    def test_generate_d3_object(self):
        assert generate_d3_object(
            {'a': 1, 'b': 2, 'c': 3, 'd': 4},
            "object", "word", "count") == {'name': 'object', 'children': [{
            'word': 'a', 'count': 1}, {'word': 'b', 'count': 2}, {<commit_msg>fix unit test errors<commit_after>from lexos.helpers.general_functions import get_encoding, make_preview_from, \
    generate_d3_object, merge_list, load_stastic, matrix_to_dict, \
    dict_to_matrix, html_escape, apply_function_exclude_tags, decode_bytes
import unittest


class TestGeneralFunctions(unittest.TestCase):
    def test_get_encoding(self):
        assert get_encoding(b"asdf") == "ascii"

    def test_make_preview_from(self):
        one_char = "x"
        less_than_500_char = "modgaecq"
        str_250 = "gjzeqagitanbwnuwjkfbtpixhkcxltlcmvrbunoxovjzhyoiptckkxmd" \
                  "brcnshyefsrqexbdeczdbqjvprgiyjwwsacutlahuwhmscyuwkqxfnxq" \
                  "zxyozedtwmrztwzzvoxrjnaypzbrkxfytpqeqmemxylvrvgtsthbalai" \
                  "byzxnoxxbtofhnpdepatvbihjoungenjidckhepgdlsmnrbqdgaalidw" \
                  "gccbardglcnedcqqxduuaauzyv"
        str_500 = str_250 + str_250
        more_than_500_char_even = \
            str_250 + less_than_500_char + less_than_500_char + str_250
        more_than_500_char_odd = \
            str_250 + less_than_500_char + one_char + less_than_500_char + \
            str_250
        middle = '\u2026 ' + '/n/n' + '\u2026'
        assert make_preview_from(less_than_500_char) == less_than_500_char
        assert make_preview_from(str_500) == str_500
        assert make_preview_from(
            more_than_500_char_even) == str_250 + middle + str_250

        assert make_preview_from(
            more_than_500_char_odd) == \
               str_250 + less_than_500_char + middle + one_char + str_250 + \
               less_than_500_char

    def test_generate_d3_object(self):
        assert generate_d3_object(
            {'a': 1, 'b': 2, 'c': 3, 'd': 4},
            "object", "word", "count") == {'name': 'object', 'children': [{
            'word': 'a', 'count': 1}, {'word': 'b', 'count': 2}, {
{'fraction_non_alphanumeric': 0.10097087378640776, 'fraction_numerical': 0.05603328710124827, 'mean_word_length': 2.5492125984251968, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '810772', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1242, 'n_words': 203}