<commit_before>from collections import defaultdict

from sklearn.linear_model import SGDClassifier
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.pipeline import Pipeline

from samr.transformations import ExtractText


_valid_classifiers = {
    "sgd": SGDClassifier,
}


def target(phrases):
    return [datapoint.sentiment for datapoint in phrases]


class PhraseSentimentPredictor:
    def __init__(self, classifier="sgd", classifier_args=None, lowercase=True):
        if classifier_args is None:
            classifier_args = {}
        classifier = _valid_classifiers[classifier](**classifier_args)
        self.pipeline = Pipeline([
            ("extractor", ExtractText()),
            ("vectorizer", CountVectorizer(lowercase=lowercase)),
            ("classifier", classifier)
        ])

    def fit(self, phrases, y=None):
        self.pipeline.fit(phrases, target(phrases))
        return self

    def predict(self, phrases):
        return self.pipeline.predict(phrases)

    def score(self, phrases):
        return self.pipeline.score(phrases, target(phrases))

    def error_matrix(self, phrases):
        predictions = self.predict(phrases)
        matrix = defaultdict(list)
        for phrase, predicted in zip(phrases, predictions):
            if phrase.sentiment != predicted:
                matrix[(phrase.sentiment, predicted)].append(phrase)
        return matrix
<commit_msg>Modified predictor to incorporate the text replacement transformation and to be extensible to future transformations<commit_after>from collections import defaultdict

from sklearn.linear_model import SGDClassifier
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.pipeline import Pipeline

from samr.transformations import ExtractText, ReplaceText


_valid_classifiers = {
    "sgd": SGDClassifier,
}


def target(phrases):
    return [datapoint.sentiment for datapoint in phrases]


class PhraseSentimentPredictor:
    def __init__(self, classifier="sgd", classifier_args=None, lowercase=True,
                 text_replacements=None):
        if classifier_args is None:
            classifier_args = {}

        pipeline = [("extractor", ExtractText())]
        if text_replacements:
            pipeline.append(("replacements", ReplaceText(text_replacements)))
        pipeline.append(("vectorizer", CountVectorizer(lowercase=lowercase)))
        pipeline.append(("classifier", _valid_classifiers[classifier](**classifier_args)))
        self.pipeline = Pipeline(pipeline)

    def fit(self, phrases, y=None):
        self.pipeline.fit(phrases, target(phrases))
        return self

    def predict(self, phrases):
        return self.pipeline.predict(phrases)

    def score(self, phrases):
        return self.pipeline.score(phrases, target(phrases))

    def error_matrix(self, phrases):
        predictions = self.predict(phrases)
        matrix = defaultdict(list)
        for phrase, predicted in zip(phrases, predictions):
            if phrase.sentiment != predicted:
                matrix[(phrase.sentiment, predicted)].append(phrase)
        return matrix

{'fraction_non_alphanumeric': 0.09090909090909091, 'fraction_numerical': 0.0, 'mean_word_length': 3.4899425287356323, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2177646', 'n_tokens_mistral': 874, 'n_tokens_neox': 854, 'n_words': 203}