<commit_before>from lexos.helpers.error_messages import NOT_ONE_REPLACEMENT_COLON_MESSAGE, \
    REPLACEMENT_RIGHT_OPERAND_MESSAGE, REPLACEMENT_NO_LEFTHAND_MESSAGE
from lexos.helpers.exceptions import LexosException
from lexos.processors.prepare.scrubber import replacement_handler, \
    get_remove_whitespace_map, replace_with_dict, get_remove_punctuation_map, \
    get_remove_digits_map, get_all_punctuation_map, delete_words, \
    handle_gutenberg, split_stop_keep_word_string, \
    get_special_char_dict_from_file, process_tag_replace_options, \
    scrub_select_apos, consolidate_hyphens, consolidate_ampers, \
    handle_file_and_manual_strings, remove_stopwords, keep_words
from test.helpers import special_chars_and_punct as chars, gutenberg as guten


class TestGetSpecialCharDictFromFile:

    def test_get_special_char_dict_from_file(self):
        assert get_special_char_dict_from_file(char_set="MUFI-3") == \
            chars.MUFI3
        assert get_special_char_dict_from_file(char_set="MUFI-4") == \
            chars.MUFI4
        # This option should be processed by handle_special_characters() only
        try:
            get_special_char_dict_from_file(char_set="doe-sgml")
        except ValueError:
            pass
        else:
            raise AssertionError
        try:
            get_special_char_dict_from_file(char_set="MADEUP-6")
        except ValueError:
            pass
        else:
            raise AssertionError


# handle_special_characters


class TestReplacementHandlerAlone:
    test_string = "Test string is testing"

    def test_not_lemma_normal(self):
        assert replacement_handler(
            text=self.test_string, replacer_string="s:f", is_lemma=False) == \
            "Teft ftring if tefting"
        assert replacement_handler(
            text=self.test_string, replacer_string="i,e:a", is_lemma=False) \
            == "Tast strang as tastang"
        assert replacement_handler(
            text=self.test_string, replacer_string="q:z", is_lemma=False) == \
            self.test_string<commit_msg>added test for early-english-html get special characters<commit_after>from lexos.helpers.error_messages import NOT_ONE_REPLACEMENT_COLON_MESSAGE, \
    REPLACEMENT_RIGHT_OPERAND_MESSAGE, REPLACEMENT_NO_LEFTHAND_MESSAGE
from lexos.helpers.exceptions import LexosException
from lexos.processors.prepare.scrubber import replacement_handler, \
    get_remove_whitespace_map, replace_with_dict, get_remove_punctuation_map, \
    get_remove_digits_map, get_all_punctuation_map, delete_words, \
    handle_gutenberg, split_stop_keep_word_string, \
    get_special_char_dict_from_file, process_tag_replace_options, \
    scrub_select_apos, consolidate_hyphens, consolidate_ampers, \
    handle_file_and_manual_strings, remove_stopwords, keep_words
from test.helpers import special_chars_and_punct as chars, gutenberg as guten


class TestGetSpecialCharDictFromFile:

    def test_get_special_char_dict_from_file(self):
        assert get_special_char_dict_from_file(char_set="MUFI-3") == \
            chars.MUFI3
        assert get_special_char_dict_from_file(char_set="MUFI-4") == \
            chars.MUFI4
        # This option should be processed by handle_special_characters() only
        try:
            get_special_char_dict_from_file(char_set="doe-sgml")
        except ValueError:
            pass
        else:
            raise AssertionError
        try:
            get_special_char_dict_from_file(char_set="early-english-html")
        except ValueError:
            pass
        else:
            raise AssertionError
        try:
            get_special_char_dict_from_file(char_set="MADEUP-6")
        except ValueError:
            pass
        else:
            raise AssertionError


# handle_special_characters


class TestReplacementHandlerAlone:
    test_string = "Test string is testing"

    def test_not_lemma_normal(self):
        assert replacement_handler(
            text=self.test_string, replacer_string="s:f", is_lemma=False) == \
            "Teft ftring if tefting"
        assert replacement_handler(
            text=self.test_string, replacer_string="i,e:a", is_lemma=False) \
            == "Tast strang as tastang"
        assert replacement_handler(
            text=self.test_string, replacer_string="q:z", is_lemma=False) == \
            self.test_string
{'fraction_non_alphanumeric': 0.06599450045829514, 'fraction_numerical': 0.00229147571035747, 'mean_word_length': 3.3004926108374386, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1187591', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1345, 'n_words': 227}