<commit_before>from sklearn.preprocessing import *
import pandas as pd
from sys import argv


file_path = argv[7]
verbose = argv[8] == "true"
scaler = argv[9]


#####################################################################################
# Scaler Options
# (https://scikit-learn.org/stable/auto_examples/preprocessing/plot_all_scaling.html)
# Standard = StandardScaler
# Robust = RobustScaler
# MinMax = MinMaxScaler
# MaxAbs = MaxAbsScaler
# Power = PowerTransformer
# QuantileNormal = QuantileTransformer w/ normal output
# QuantileUniform = QuantileTransformer w/ uniform output
# Normalizer = Normalizer
#####################################################################################


def scale_data(file_path, scaler='robust', verbose=False):
    scaler = scaler.lower()
    gzip = False
    if file_path.endswith('.gz'):
        gzip = True
    data = pd.read_csv(file_path, sep='\t', index_col=0)
    index = data.index
    x = data.drop('Class', axis=1)
    x_columns = x.columns
    y = data[['Class']]
    if scaler == 'standard':
        transformer = StandardScaler()
    elif scaler == 'minmax':
        transformer = MinMaxScaler()
    elif scaler == 'maxabs':
        transformer = MaxAbsScaler()
    elif 'power' in scaler:
        transformer = PowerTransformer()
    elif 'quantile' in scaler:
        if 'normal' in scaler:
            if verbose:
                print('Outputting normal (Gaussian) distribution')
            transformer = QuantileTransformer(output_distribution='normal')
        else:
            transformer = QuantileTransformer(output_distribution='uniform')
            if verbose:
                print('Outputting uniform distribution')
    elif scaler == 'normalizer':
        transformer = Normalizer()
    else:
        transformer = RobustScaler()
    if verbose:
        print('Using scaler of class "{}"'.format(str(type(transformer)).split("'")[1]))
    x = transformer.fit_transform(x)
    x = pd.DataFrame(x, index=index, columns=x_columns)
    data = pd.concat([x, y], axis=1, join='inner')
    data.to_csv(file_path, index_label='', sep='\t', compression='gzip' if gzip else None)


scale_data(file_path, scaler, verbose)<commit_msg>detect and remove non-numerical and categorical columns before scaling<commit_after>from sklearn.preprocessing import *
import pandas as pd
from sys import argv


file_path = argv[7]
verbose = argv[8] == "true"
scaler = argv[9]


#####################################################################################
# Scaler Options
# (https://scikit-learn.org/stable/auto_examples/preprocessing/plot_all_scaling.html)
# Standard = StandardScaler
# Robust = RobustScaler
# MinMax = MinMaxScaler
# MaxAbs = MaxAbsScaler
# Power = PowerTransformer
# QuantileNormal = QuantileTransformer w/ normal output
# QuantileUniform = QuantileTransformer w/ uniform output
# Normalizer = Normalizer
#####################################################################################


def scale_data(file_path, scaler='robust', verbose=False):
    scaler = scaler.lower()
    data = pd.read_csv(file_path, sep='\t', index_col=0)
    index = data.index

    # Remove columns that don't contain numerical values
    non_num_columns = ['Class']
    dtypes = data.columns.to_series().groupby(data.dtypes).groups
    dtypes = {k.name: list(v) for k, v in dtypes.items()}
    for dtype in dtypes.keys():
        if 'float' not in dtype and 'int' not in dtype:
            non_num_columns += dtypes[dtype]
    if verbose:
        print('The following columns do not contain numerical values and will not be scaled:')
        print('\t', ', '.join(set(non_num_columns)))

    # Remove columns that contain numbers, but fewer than half are distinct values
    categorical_columns = []
    num_unique = data.columns.to_series().groupby(data.nunique()).groups
    num_unique = {k: list(v) for k, v in num_unique.items()}
    for num in num_unique.keys():
        if num < len(index) * 0.5:
            categorical_columns += num_unique[num]
    if verbose and len(categorical_columns) > 0:
        print('In the following columns, fewer than half are distinct values, so they will not be scaled:')
        print('\t', ', '.join(categorical_columns))

    excluded_columns = set(non_num_columns + categorical_columns)
    excluded_data = data[excluded_columns]
    scaled_data = data.drop(excluded_columns, axis=1)
    scaled_columns = scaled_data.columns
    if len(scaled_columns) > 0:
        if scaler == 'standard':
            transformer = StandardScaler()
        elif scaler == 'minmax':
            transformer = MinMaxScaler()
        elif scaler == 'maxabs':
            transformer = MaxAbsScaler()
        elif 'power' in scaler:
            transformer = PowerTransformer()
        elif 'quantile' in scaler:
            if 'normal' in scaler:
                if verbose:
                    print('Outputting normal (Gaussian) distribution')
                transformer = QuantileTransformer(output_distribution='normal')
            else:
                transformer = QuantileTransformer(output_distribution='uniform')
                if verbose:
                    print('Outputting uniform distribution')
        elif scaler == 'normalizer':
            transformer = Normalizer()
        else:
            transformer = RobustScaler()
        if verbose:
            print('Using scaler of class "{}" to scale data'.format(str(type(transformer)).split("'")[1]))
        scaled_data = transformer.fit_transform(scaled_data)
        scaled_data = pd.DataFrame(scaled_data, index=index, columns=scaled_columns)
        data = pd.concat([scaled_data, excluded_data], axis=1, join='inner')
        if verbose:
            print('Saving scaled version of data to {}'.format(file_path))
        data.to_csv(file_path, index_label='', sep='\t')
    else:
        if verbose:
            print('No columns remain to be scaled, no action was performed')


scale_data(file_path, scaler, verbose)
{'fraction_non_alphanumeric': 0.16032064128256512, 'fraction_numerical': 0.003006012024048096, 'mean_word_length': 3.324187725631769, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6296234', 'n_tokens_mistral': 1734, 'n_tokens_neox': 1668, 'n_words': 434}