<jupyter_start><jupyter_text>Data preprocessing template  Libraries and dependencies<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os

from sklearn.impute import SimpleImputer<jupyter_output><empty_output><jupyter_text>Data Import<jupyter_code># Opening the csv file

filename = 'data_preprocessing.csv'
path = os.path.join('data', filename)
raw_data = pd.read_csv(path)

print(raw_data)
# feature_matrix -> often called x
# dependant_var -> often called y

feature_matrix = raw_data.iloc[:, :-1]
dependant_var = raw_data.iloc[:, -1]

print(feature_matrix)
print(dependant_var)<jupyter_output>0     No
1    Yes
2     No
3     No
4    Yes
5    Yes
6     No
7    Yes
8     No
9    Yes
Name: Purchased, dtype: object<jupyter_text>Replacing data with the average<jupyter_code>feature_matrix.iloc[:, 0]

imputer = SimpleImputer(missing_values=np.nan, strategy='mean')
imputer.fit(feature_matrix.iloc[:, 1:3])
feature_matrix.iloc[:, 1:3] = imputer.transform(feature_matrix.iloc[:, 1:3])
feature_matrix<jupyter_output><empty_output><jupyter_text>Encode categorical data<jupyter_code># Independent variable
from sklearn.compose import ColumnTransformer
from sklearn.preprocessing import OneHotEncoder

column_transform = ColumnTransformer(transformers=[('encoder', OneHotEncoder(), [0])], remainder='passthrough')

# Change the dataframe to a np array
feature_matrix = np.array(column_transform.fit_transform(feature_matrix))

print(feature_matrix)
# Dependent vatiable
# Change to binary

from sklearn.preprocessing import LabelEncoder

le = LabelEncoder()
dependant_var = le.fit_transform(dependant_var)
print(dependant_var)<jupyter_output>[0 1 0 0 1 1 0 1 0 1]<jupyter_text>Obtaining test and training set<jupyter_code>from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(feature_matrix, dependant_var, test_size = 0.2, random_state = 1)
print(X_train)
print(X_test)
print(y_train)
print(y_test)<jupyter_output>[0 1]<jupyter_text>Feature Scaling<jupyter_code>from sklearn.preprocessing import StandardScaler

sc = StandardScaler()

X_train[:, 3:] = sc.fit_transform(X_train[:, 3:])
X_test[:, 3:] = sc.transform(X_test[:, 3:])

print(X_train)
print(X_test)<jupyter_output>[[0.0e+00 1.0e+00 0.0e+00 3.0e+01 5.4e+04]
 [1.0e+00 0.0e+00 0.0e+00 3.7e+01 6.7e+04]]
{'fraction_non_alphanumeric': 0.11148936170212766, 'fraction_numerical': 0.03361702127659574, 'mean_word_length': 6.759075907590759, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '282656', 'license': 'MIT', 'n_tokens_mistral': 912, 'n_tokens_neox': 870, 'n_words': 187, 'path': 'preprocessing_template.ipynb', 'repo_name': 'JuanjoFernandez/MachineLearningAcademic', 'size': '2350'}