<commit_before># -*- coding: utf-8 -*-
import re
import sys

from lxml import html

rex1 = re.compile(r'(?<=\n)( )+')
rex2 = re.compile(r'^[ \t\n\r\f]*')
rex3 = re.compile(r'[ \t]+(?=([ \t]))')
rex4 = re.compile(r'[ \n]+(?=(\n))')
rex5 = re.compile(r'[ \n]*$')


class Cleaner:
    """The Cleaner-Class tries to get the raw extracted text of the extractors
    in a comparable format. For this it deletes unnecessary whitespaces
    or in case of readability html-tags which are still in the extracted
    text.
    """

    def delete_tags(self, arg):
        """Removes html-tags from extracted data.

        :param arg: A string, the string which shall be cleaned
        :return: A string, the cleaned string
        """

        if len(arg) > 0:
            raw = html.fromstring(arg)
            return raw.text_content().strip()

        return arg

    def delete_whitespaces(self, arg):
        """Removes newlines, tabs and whitespaces at the beginning, the end and if there is more than one.

        :param arg: A string, the string which shell be cleaned
        :return: A string, the cleaned string
        """
        # Deletes whitespaces after a newline
        arg = re.sub(rex1, '', arg)
        # Deletes every whitespace, tabulator, newline at the beginning of the string
        arg = re.sub(rex2, '', arg)
        # Deletes whitespace or tabulator if followed by whitespace or tabulator
        arg = re.sub(rex3, '', arg)
        #  Deletes newline if it is followed by an other one
        arg = re.sub(rex4, '', arg)
        # Deletes newlines and whitespaces at the end of the string
        arg = re.sub(rex5, '', arg)
        return arg

    def do_cleaning(self, arg):
        """Does the actual cleaning by using the delete methods above.

        :param arg: A string, the string which shell be cleaned. Or a list, in which case each of the strings within the
        list is cleaned.
        :return: A string, the cleaned string. Or a list with cleaned string entries.
        """
        if arg is not None:
            if isinstance(arg, list):<commit_msg>Improve regex naming<commit_after># -*- coding: utf-8 -*-
import re
import sys

from lxml import html

re_newline_spc = re.compile(r'(?<=\n)( )+')
re_starting_whitespc = re.compile(r'^[ \t\n\r\f]*')
re_multi_spc_tab = re.compile(r'[ \t]+(?=([ \t]))')
re_double_newline = re.compile(r'[ \n]+(?=(\n))')
re_ending_spc_newline = re.compile(r'[ \n]*$')


class Cleaner:
    """The Cleaner-Class tries to get the raw extracted text of the extractors
    in a comparable format. For this it deletes unnecessary whitespaces
    or in case of readability html-tags which are still in the extracted
    text.
    """

    def delete_tags(self, arg):
        """Removes html-tags from extracted data.

        :param arg: A string, the string which shall be cleaned
        :return: A string, the cleaned string
        """

        if len(arg) > 0:
            raw = html.fromstring(arg)
            return raw.text_content().strip()

        return arg

    def delete_whitespaces(self, arg):
        """Removes newlines, tabs and whitespaces at the beginning, the end and if there is more than one.

        :param arg: A string, the string which shell be cleaned
        :return: A string, the cleaned string
        """
        # Deletes whitespaces after a newline
        arg = re.sub(re_newline_spc, '', arg)
        # Deletes every whitespace, tabulator, newline at the beginning of the string
        arg = re.sub(re_starting_whitespc, '', arg)
        # Deletes whitespace or tabulator if followed by whitespace or tabulator
        arg = re.sub(re_multi_spc_tab, '', arg)
        #  Deletes newline if it is followed by an other one
        arg = re.sub(re_double_newline, '', arg)
        # Deletes newlines and whitespaces at the end of the string
        arg = re.sub(re_ending_spc_newline, '', arg)
        return arg

    def do_cleaning(self, arg):
        """Does the actual cleaning by using the delete methods above.

        :param arg: A string, the string which shell be cleaned. Or a list, in which case each of the strings within the
        list is cleaned.
        :return: A string, the cleaned string. Or a list with cleaned string entries.
        """
        if arg is not None:
            if isinstance(arg, list):
{'fraction_non_alphanumeric': 0.10344031401523897, 'fraction_numerical': 0.0032325098129762178, 'mean_word_length': 2.4907332796132153, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '810251', 'n_tokens_mistral': 1314, 'n_tokens_neox': 1263, 'n_words': 509}