<jupyter_start><jupyter_text>Pipeline B- Normalization Skew<jupyter_code># for preprocessing/eda models
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import power_transform
from scipy import stats
from scipy.stats import boxcox
from scipy.stats import kurtosis, skew
import matplotlib.pyplot as plt
import math
from scipy.stats import norm

# feature selection
from sklearn.feature_selection import RFE

# balancing
from imblearn.over_sampling import SMOTE

# accuracy metrics and data split models
from sklearn.model_selection import train_test_split
from sklearn import metrics, model_selection
from sklearn.metrics import confusion_matrix
from sklearn.metrics import classification_report
from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score
from sklearn.metrics import precision_recall_fscore_support

# machine learning
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn import metrics
from sklearn.ensemble import ExtraTreesClassifier
from xgboost import XGBClassifier
from sklearn.tree import DecisionTreeClassifier 
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier

pd.set_option('display.max_columns', 500) # display max 500 rows
# read in data, make a copy, and display first 5 rows
data = pd.read_csv('online_shoppers_intention-2.csv')
data1 = data.copy()
data1.head()<jupyter_output><empty_output><jupyter_text>Imputation<jupyter_code># impute our assumption (beccomes a new feature)
data1['ExitRatesImpute'] = data1['ExitRates'].replace(0,np.NaN)
data1['ExitRatesImpute'] = data1['ExitRatesImpute'].fillna(data1['ExitRatesImpute'].median())<jupyter_output><empty_output><jupyter_text>Feature Engineering (Ratios, aggregation)- calculate ratios for count of page visits- combine bounce and exit rates (average and weighted average - new features)    - they have a strong linear relationship- divide features that have strong non-linear relationship    - solve zero-division by replacing it with 0 (e.g. 1/0 = n/a)<jupyter_code># ratios for counts
data1['totalFracAdmin'] = data1['Administrative']/(data1['Administrative'] + data1['Informational'] + data1['ProductRelated'])
data1['totalFracInfo'] = data1['Informational']/(data1['Administrative'] + data1['Informational'] + data1['ProductRelated'])
data1['totalFracProd'] = data1['ProductRelated']/(data1['Administrative'] + data1['Informational'] + data1['ProductRelated'])

# average combining
data1['BounceExitAvg'] = (data1['BounceRates'] + data1['ExitRates'])/2

# weighted-average feature joining
data1['BounceExitW1'] = data1['BounceRates'] * 0.6 + data1['ExitRates'] * 0.4
data1['BounceExitW2'] = data1['BounceRates'] * 0.7 + data1['ExitRates'] * 0.3
data1['BounceExitW3'] = data1['BounceRates'] * 0.4 + data1['ExitRates'] * 0.6
data1['BounceExitW4'] = data1['BounceRates'] * 0.3 + data1['ExitRates'] * 0.7

# bounce and exit rates vs page values ratio 
data1['BouncePageRatio'] = data1['BounceRates']/data1['PageValues']
data1['ExitPageRatio'] = data1['ExitRates']/data1['PageValues']

# durations vs page values, bounce and exit rates
data1['InfoPageRatio'] = data1['Informational_Duration']/data1['PageValues']
data1['ProdRelPageRatio'] = data1['ProductRelated_Duration']/data1['PageValues']
data1['InfoBounceRatio'] = data1['Informational_Duration']/data1['BounceRates']
data1['AdminBounceRatio'] = data1['Administrative_Duration']/data1['BounceRates']
data1['ProdRelBounceRatio'] = data1['ProductRelated_Duration']/data1['BounceRates']
data1['InfoExitRatio'] = data1['Informational_Duration']/data1['ExitRates']
data1['AdminBounceRatio'] = data1['Administrative_Duration']/data1['ExitRates']
data1['ProdRelExitRatio'] = data1['ProductRelated_Duration']/data1['ExitRates']

# as there are many zero values --> e.g. x/0 (zero-division) = N/A or inf can occur 
# if there is zero-division treat that as a zero
data1 = data1.fillna(0) # fill N/A with 0 
data1 = data1.replace(np.inf, 0) # replace inf wit 0
data1 = data1.replace(-0, 0) # for some reason we also get -0 just fix it to 0 
print('Original number of features: ', len(data.columns))
print('Number of features added: ', len(data1.columns) - len(data.columns))
print('Total number of features after feature engineering: ', len(data1.columns) + len(data.columns))<jupyter_output>Original number of features:  18
Number of features added:  18
Total number of features after feature engineering:  54<jupyter_text>Normalization- Normalize continous values- Requirement: -0.5 < .skew() < 0.5- Yeo-Johnson: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.yeojohnson.html<jupyter_code># make a copy and select continous features
normalization_df = data1.select_dtypes(include='float64').drop('SpecialDay', axis = 1)
print('Nbr of continous features: ',len(normalization_df.columns))<jupyter_output>Nbr of continous features:  24<jupyter_text>Features that need to be Normalized<jupyter_code># find skewed features 
normalization_needed = [] # empty list for cols that need normalizing 
normalization_df_cols = normalization_df.columns
for col in normalization_df_cols:
    if normalization_df[col].skew() > 0.5:
        print('Right skew: ',col,normalization_df[col].skew())
        normalization_needed.append(col)
    elif normalization_df[col].skew() < -0.5:
        print('Left skew: ',col,normalization_df[col].skew())
        normalization_needed.append(col)
    else:
        ""
print(len(normalization_needed),' features need normalizing')<jupyter_output>24  features need normalizing<jupyter_text>Plot Distribution- Some are highly skewed and might not be possible to normalize<jupyter_code># Histogram with Normal distribution
# source: https://stackoverflow.com/questions/20011122/fitting-a-normal-distribution-to-1d-data
import numpy as np
from scipy.stats import norm
import matplotlib.pyplot as plt

def plot_hist(df, column, bin_amount=25):
    # Fit a normal distribution to the data:
    mu, std = norm.fit(df[column])

    # Plot the histogram.
    ## Set the alpha value used for blending (how transparent the color is)
    plt.hist(df[column], bins=bin_amount, density=True, alpha=0.8, color='g')

    # Plot the PDF.
    xmin, xmax = plt.xlim()
    x = np.linspace(xmin, xmax, 100)
    p = norm.pdf(x, mu, std)
    plt.plot(x, p, 'k', linewidth=2)
    title = "Fit results: mu = %.2f,  std = %.2f" % (mu, std)
    plt.xlabel(column)
    plt.title(title)
    plt.show()
for col in normalization_df_cols:
    plot_hist(normalization_df, col)<jupyter_output><empty_output><jupyter_text>Applying Normalization<jupyter_code># innitial yeo-johnson normalization
for col in normalization_needed:
    if normalization_df[col].skew() > 0.5:
        normalization_df[col] = stats.yeojohnson(normalization_df[col])[0]
    
# replace -0 with 0 
normalization_df = normalization_df.replace(-0, 0)

# square-root transformation for positively/rightly skewed features that still have a skew > 0.5 
for col in normalization_df_cols:
    if normalization_df[col].skew() > 0.5:
        normalization_df[col] = np.sqrt(normalization_df[col])
        
# left skew transformation
normalization_df['totalFracProd'] = normalization_df['totalFracProd']**2

# right manual transformation
normalization_df['AdminBounceRatio'] = np.sqrt(normalization_df['AdminBounceRatio'])
normalization_df['ProdRelBounceRatio'] = np.sqrt(np.sqrt(normalization_df['ProdRelBounceRatio']))<jupyter_output><empty_output><jupyter_text>Skew after normalization<jupyter_code>not_normalized = []
# if still not normal display 
for col in normalization_df_cols:
    if normalization_df[col].skew() > 0.5:
        print('Right skew: ',col,normalization_df[col].skew())
        not_normalized.append(col)
    elif normalization_df[col].skew() < -0.5:
        print('Left skew: ',col,normalization_df[col].skew())
        not_normalized.append(col)
    else:
         print('Normalized', col,normalization_df[col].skew())
print('Number of features that were not normalized: ',len(not_normalized))
print('Number of features that were normalized: ', len(normalization_needed) - len(not_normalized))
normalization_df.to_csv('PipelineB-Normalize_Skew.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11568276684555755, 'fraction_numerical': 0.02075134168157424, 'mean_word_length': 6.047058823529412, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 36, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '245563', 'license': 'MIT', 'n_tokens_mistral': 2814, 'n_tokens_neox': 2682, 'n_words': 653, 'path': 'project2-mem-master/3_Data_Preprocessing/PipelineB/PipelineB-Normalize_Skew.ipynb', 'repo_name': 'erikhren/Determining-shoppers', 'size': '8385'}