<commit_before># -*- coding: utf-8 -*-

from sklearn.pipeline import Pipeline
from sklearn.externals import six


class SparkPipeline(Pipeline):

    def _pre_transform(self, Z, **fit_params):
        fit_params_steps = dict((step, {}) for step, _ in self.steps)
        for pname, pval in six.iteritems(fit_params):
            step, param = pname.split('__', 1)
            fit_params_steps[step][param] = pval
        Zt = Z
        for name, transform in self.steps[:-1]:
            if hasattr(transform, "fit_transform"):
                Zt = transform.fit_transform(Zt, **fit_params_steps[name])
            else:
                Zt = transform.fit(Zt, **fit_params_steps[name]) \
                              .transform(Zt)
        return Zt, fit_params_steps[self.steps[-1][0]]

    def fit(self, Z, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then fit the transformed data using the final estimator.
        """
        Zt, fit_params = self._pre_transform(Z, **fit_params)
        self.steps[-1][-1].fit(Zt, **fit_params)
        return self

    def fit_transform(self, Z, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then use fit_transform on transformed data using the final
        estimator."""
        Zt, fit_params = self._pre_transform(Z, **fit_params)
        if hasattr(self.steps[-1][-1], 'fit_transform'):
            return self.steps[-1][-1].fit_transform(Zt, **fit_params)
        else:
            return self.steps[-1][-1].fit(Zt, **fit_params).transform(Zt)<commit_msg>Added corrected score method to pipeline<commit_after># -*- coding: utf-8 -*-

from sklearn.pipeline import Pipeline
from sklearn.externals import six


class SparkPipeline(Pipeline):

    def _pre_transform(self, Z, **fit_params):
        fit_params_steps = dict((step, {}) for step, _ in self.steps)
        for pname, pval in six.iteritems(fit_params):
            step, param = pname.split('__', 1)
            fit_params_steps[step][param] = pval
        Zt = Z
        for name, transform in self.steps[:-1]:
            if hasattr(transform, "fit_transform"):
                Zt = transform.fit_transform(Zt, **fit_params_steps[name])
            else:
                Zt = transform.fit(Zt, **fit_params_steps[name]) \
                              .transform(Zt)
        return Zt, fit_params_steps[self.steps[-1][0]]

    def fit(self, Z, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then fit the transformed data using the final estimator.
        """
        Zt, fit_params = self._pre_transform(Z, **fit_params)
        self.steps[-1][-1].fit(Zt, **fit_params)
        return self

    def fit_transform(self, Z, **fit_params):
        """Fit all the transforms one after the other and transform the
        data, then use fit_transform on transformed data using the final
        estimator."""
        Zt, fit_params = self._pre_transform(Z, **fit_params)
        if hasattr(self.steps[-1][-1], 'fit_transform'):
            return self.steps[-1][-1].fit_transform(Zt, **fit_params)
        else:
            return self.steps[-1][-1].fit(Zt, **fit_params).transform(Zt)

    def score(self, Z):
        """Applies transforms to the data, and the score method of the
        final estimator. Valid only if the final estimator implements
        score."""
        Zt = Z
        for name, transform in self.steps[:-1]:
            Zt = transform.transform(Zt)
        return self.steps[-1][-1].score(Zt)
{'fraction_non_alphanumeric': 0.12133891213389121, 'fraction_numerical': 0.008089260808926082, 'mean_word_length': 2.411988582302569, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2197802', 'n_tokens_mistral': 1088, 'n_tokens_neox': 1062, 'n_words': 295}