<commit_before>import codecs
import os
import string
import unicodedata

path = os.path.dirname(__file__)


class Normalizr:
    """
    This class offers methods for text normalization.

    Attributes:
        language (string): Language used for normalization.
    """
    __punctuation = set(string.punctuation)
    __stop_words = set()

    def __init__(self, language='en'):
        self._load_stop_words(language)

    def _load_stop_words(self, language):
        """
        Load stop words into __stop_words set.

        Stop words will be loaded according to the language code received during instantiation.

        Params:
            language (string): Language code.
        """
        with codecs.open(os.path.join(path, 'data/stop-' + language), 'r', 'UTF-8') as file:
            for line in file:
                fields = line.split('|')
                if fields:
                    word = fields[0].strip()
                    if word: self.__stop_words.add(word)

    def normalize(self, text, normalizations=None):
        """
        Normalize a given text applying all normalizations.

        Normalizations to apply can be specified through a list parameter and will be executed
        in the same order.

        Params:
            text (string): The text to be processed.
            normalizations (list): List of normalizations to apply.

        Returns:
            The text normalized.
        """
        if normalizations is None:
            normalizations = ['whitespaces', 'punctuation', 'symbols', 'stopwords']<commit_msg>Fix stop words list loading<commit_after>import codecs
import os
import string
import unicodedata

path = os.path.dirname(__file__)


class Normalizr:
    """
    This class offers methods for text normalization.

    Attributes:
        language (string): Language used for normalization.
    """
    __punctuation = set(string.punctuation)
    __stop_words = set()

    def __init__(self, language='en'):
        self._load_stop_words(language)

    def _load_stop_words(self, language):
        """
        Load stop words into __stop_words set.

        Stop words will be loaded according to the language code received during instantiation.

        Params:
            language (string): Language code.
        """
        with codecs.open(os.path.join(path, 'data/stop-' + language), 'r', 'UTF-8') as file:
            for line in file:
                fields = line.split('|')
                if fields:
                    for word in fields[0].split(): self.__stop_words.add(word.strip())

    def normalize(self, text, normalizations=None):
        """
        Normalize a given text applying all normalizations.

        Normalizations to apply can be specified through a list parameter and will be executed
        in the same order.

        Params:
            text (string): The text to be processed.
            normalizations (list): List of normalizations to apply.

        Returns:
            The text normalized.
        """
        if normalizations is None:
            normalizations = ['whitespaces', 'punctuation', 'symbols', 'stopwords']
{'fraction_non_alphanumeric': 0.08745611235237791, 'fraction_numerical': 0.0012767315671879987, 'mean_word_length': 2.214358974358974, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7622358', 'n_tokens_mistral': 828, 'n_tokens_neox': 786, 'n_words': 281}