<commit_before>from sklearn import preprocessing

from wallace.predictive_models.predictive_model import PredictiveModel, TrainedPredictiveModel
from wallace.categorical_variable_encoder import CategoricalVariableEncoder

class SklearnModel(PredictiveModel):
    def get_dependent_variable_data(self, dataset):
        return dataset.get_filtered_column(self.dependent_variable)

    def get_independent_variable_data(self, dataset):
        filtered_matrix = dataset.get_filtered_matrix(self.independent_variables)
        filtered_data_types = dataset.get_filtered_data_types(self.independent_variables)
        categorical_features = self.get_categorical_features(filtered_data_types)

        variable_encoder = CategoricalVariableEncoder()
        filtered_matrix = variable_encoder.convert_categorical_variables(filtered_matrix, categorical_features)

        encoder = preprocessing.OneHotEncoder(categorical_features=categorical_features)
        return encoder.fit_transform(filtered_matrix)

    def get_categorical_features(self, filtered_data_types):
        categorical_features = []
        for i in xrange(len(filtered_data_types)):
            data_type = filtered_data_types[i]<commit_msg>Sometimes the encoder returns sparse matrices, we need to handle this.<commit_after>from sklearn import preprocessing

from wallace.predictive_models.predictive_model import PredictiveModel, TrainedPredictiveModel
from wallace.categorical_variable_encoder import CategoricalVariableEncoder

class SklearnModel(PredictiveModel):
    def get_dependent_variable_data(self, dataset):
        return dataset.get_filtered_column(self.dependent_variable)

    def get_independent_variable_data(self, dataset):
        filtered_matrix = dataset.get_filtered_matrix(self.independent_variables)
        filtered_data_types = dataset.get_filtered_data_types(self.independent_variables)
        categorical_features = self.get_categorical_features(filtered_data_types)

        variable_encoder = CategoricalVariableEncoder()
        filtered_matrix = variable_encoder.convert_categorical_variables(filtered_matrix, categorical_features)

        encoder = preprocessing.OneHotEncoder(categorical_features=categorical_features)
        data_result = encoder.fit_transform(filtered_matrix)

        # Take care of sparse and non-sparse matrices. If we have any categorical
        # variables, then it is likely we will get back a sparse matrix and will
        # need to convert it into an array.
        try:
            return data_result.toarray()
        except AttributeError:
            return data_result

    def get_categorical_features(self, filtered_data_types):
        categorical_features = []
        for i in xrange(len(filtered_data_types)):
            data_type = filtered_data_types[i]
{'fraction_non_alphanumeric': 0.053214285714285714, 'fraction_numerical': 0.0, 'mean_word_length': 4.799171842650104, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4429923', 'n_tokens_mistral': 761, 'n_tokens_neox': 721, 'n_words': 145}