<commit_before>import pytest
import cleaner

class TestTagRemoval():
    def test_span_removal(self):
        text = ('<span style="font-family: &quot;helvetica neue&quot; ,'
        '&quot;arial&quot; , &quot;helvetica&quot; , sans-serif;">This is some'
        ' dummy text lalalala</span> This is some more dummy text '
        '<span>test</span>')

        expected = ('This is some dummy text lalalala This is some more dummy '
        'text test')

        cleaned = cleaner.remove_superflous_markup(text)

        assert cleaned == expected<commit_msg>Add test for getting pure html tag<commit_after>import pytest
import cleaner

class TestTagTools():
    def test_get_pure_tag(self):
        tag1 = '<div>'
        tag2 = '</div>'
        tag3 = '<pre class="prettyprint">'

        assert cleaner.get_pure_tag(tag1) == '<div>'
        assert cleaner.get_pure_tag(tag2) == '</div>'
        assert cleaner.get_pure_tag(tag3) == '<pre>'
{'fraction_non_alphanumeric': 0.12845010615711253, 'fraction_numerical': 0.006369426751592357, 'mean_word_length': 3.1359649122807016, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2779519', 'n_tokens_mistral': 308, 'n_tokens_neox': 294, 'n_words': 76}