<commit_before>""" A module for handling dynamic chained preprocessors. """

from abc import ABC, abstractmethod


class Preprocessor(ABC):
    """ A chainable preprocessor. """

    def __init__(self, tweets):
        """ Creates a new preprocessor. """
        self.tweet = tweets

    @abstractmethod
    def __iter__(self):
        """ Returns a generator for the actual preprocessing. """
        raise NotImplementedError

    @staticmethod
    def optimize(tweets, preprocessors):
        """ Optimize a tweet out of a iterator by applying a chain of preprocessors on it. """
        chain = None
        for i, preprocessor in enumerate(preprocessors):
            chain = (globals()[preprocessor])(tweets if i == 0 else chain)
        for tweet in chain:
            yield tweet<commit_msg>Add "RemoveUrls"<commit_after>""" A module for handling dynamic chained preprocessors. """

from abc import ABC, abstractmethod


class Preprocessor(ABC):
    """ A chainable preprocessor. """

    @abstractmethod
    def __iter__(self):
        """ Returns a generator for the actual preprocessing. """
        raise NotImplementedError

    @staticmethod
    def optimize(tweets, preprocessors):
        """ Optimize a tweet out of a iterator by applying a chain of preprocessors on it. """
        chain = None
        for i, preprocessor in enumerate(preprocessors):
            chain = (globals()[preprocessor])(tweets if i == 0 else chain)
        for tweet in chain:
            yield tweet


class RemoveUrls(Preprocessor):
    """ Replace links by a fixed keyword. """

    def __init__(self, tweets):
        """ Creates a new preprocessor. """
        self.tweets = tweets

    def __iter__(self):
        for tweet in self.tweets:
            tweet.text = [x if not x.startswith(
                'http') else 'URL' for x in tweet.text]
            yield tweet
{'fraction_non_alphanumeric': 0.0946524064171123, 'fraction_numerical': 0.0010695187165775401, 'mean_word_length': 2.5638095238095238, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5355455', 'n_tokens_mistral': 521, 'n_tokens_neox': 489, 'n_words': 184}