<commit_before>import numpy as np
import pandas as pd

from sklearn.base import TransformerMixin
from imblearn.over_sampling import RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler
from sklearn.preprocessing import StandardScaler


class DataFrameImputer(TransformerMixin):
    """
    Impute missing values in a dataframe.

    Columns of dtype object or category (assumed categorical) are imputed with the mode (most frequent value in column).

    Columns of other types (assumed continuous) are imputed with mean of column.
    """

    def __init__(self, impute=True):
        self.impute = impute
        self.object_columns = None
        self.fill = None

    def fit(self, X, y=None):
        # Return if not imputing
        if self.impute is False:
            return self

        # Grab list of object column names before doing imputation
        self.object_columns = X.select_dtypes(include=['object']).columns.values

        self.fill = pd.Series([X[c].value_counts().index[0]
                               if X[c].dtype == np.dtype('O')
                               or pd.core.common.is_categorical_dtype(X[c])
                               else X[c].mean() for c in X], index=X.columns)

        num_nans = sum(X.select_dtypes(include=[np.number]).isnull().sum())
        num_total = sum(X.select_dtypes(include=[np.number]).count())
        percentage_imputed = num_nans/num_total*100

        print(f"Percentage Imputed: {percentage_imputed}%")

        # return self for scikit compatibility
        return self

    def transform(self, X, y=None):
        # Return if not imputing
        if self.impute is False:
            return X

        result = X.fillna(self.fill)

        for i in self.object_columns:
            if result[i].dtype not in ['object', 'category']:
                result[i] = result[i].astype('object')

        return result


class DataFrameConvertTargetToBinary(TransformerMixin):<commit_msg>Changed to python 3 string format, and not 3.6 formatting<commit_after>import numpy as np
import pandas as pd

from sklearn.base import TransformerMixin
from imblearn.over_sampling import RandomOverSampler
from imblearn.under_sampling import RandomUnderSampler
from sklearn.preprocessing import StandardScaler


class DataFrameImputer(TransformerMixin):
    """
    Impute missing values in a dataframe.

    Columns of dtype object or category (assumed categorical) are imputed with the mode (most frequent value in column).

    Columns of other types (assumed continuous) are imputed with mean of column.
    """

    def __init__(self, impute=True):
        self.impute = impute
        self.object_columns = None
        self.fill = None

    def fit(self, X, y=None):
        # Return if not imputing
        if self.impute is False:
            return self

        # Grab list of object column names before doing imputation
        self.object_columns = X.select_dtypes(include=['object']).columns.values

        self.fill = pd.Series([X[c].value_counts().index[0]
                               if X[c].dtype == np.dtype('O')
                               or pd.core.common.is_categorical_dtype(X[c])
                               else X[c].mean() for c in X], index=X.columns)

        num_nans = sum(X.select_dtypes(include=[np.number]).isnull().sum())
        num_total = sum(X.select_dtypes(include=[np.number]).count())
        percentage_imputed = num_nans/num_total*100

        print("Percentage Imputed: {}%".format(percentage_imputed))

        # return self for scikit compatibility
        return self

    def transform(self, X, y=None):
        # Return if not imputing
        if self.impute is False:
            return X

        result = X.fillna(self.fill)

        for i in self.object_columns:
            if result[i].dtype not in ['object', 'category']:
                result[i] = result[i].astype('object')

        return result


class DataFrameConvertTargetToBinary(TransformerMixin):
{'fraction_non_alphanumeric': 0.09084084084084085, 'fraction_numerical': 0.0027527527527527527, 'mean_word_length': 2.5246913580246915, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6796944', 'n_tokens_mistral': 1176, 'n_tokens_neox': 1121, 'n_words': 324}