<jupyter_start><jupyter_text>Chapter 2: Processing data for machine learningTo simplify the code examples in these notebooks, we populate the namespace with functions from numpy and matplotlib:<jupyter_code>%pylab inline
# Import Pixie Debugger
import pixiedust<jupyter_output>Pixiedust database opened successfully<jupyter_text>Converting categorical data to numerical features<jupyter_code>cat_data = array(['male', 'female', 'male', 'male', 'female', 'male', 'female', 'female'])
def cat_to_num(data):
    categories = unique(data)
    features = []
    for cat in categories:
        binary = (data == cat)
        features.append(binary.astype("int"))
    return features
%%pixie_debugger
cat_to_num(cat_data)<jupyter_output><empty_output><jupyter_text>Simple feature engineering of the Titanic dataset<jupyter_code>cabin_data = array(["C65", "", "E36", "C54", "B57 B59 B63 B66"])
def cabin_features(data):
    features = []
    for cabin in data:
        cabins = cabin.split(" ")
        n_cabins = len(cabins)
        # First char is the cabin_char
        try:
            cabin_char = cabins[0][0]
        except IndexError:
            cabin_char = "X"
            n_cabins = 0
        # The rest is the cabin number
        try:
            cabin_num = int(cabins[0][1:]) 
        except:
            cabin_num = -1
        # Add 3 features for each passanger
        features.append( [cabin_char, cabin_num, n_cabins] )
    return features
cabin_features(cabin_data)<jupyter_output><empty_output><jupyter_text>Feature normalization<jupyter_code>num_data = array([1, 10, 0.5, 43, 0.12, 8])
def normalize_feature(data, f_min=-1, f_max=1):
    d_min, d_max = min(data), max(data)
    factor = (f_max - f_min) / (d_max - d_min)
    normalized = f_min + data*factor
    return normalized, factor

normalize_feature(num_data)
# Alternative: Convert to Pandas Dataframe
import pandas as pd
import numpy as np
num_data2 = np.array([1, 10, 0.5, 43, 0.12, 8])
df = pd.Series(num_data2)
normalized_df=(df-df.min())/(df.max()-df.min()) # normalize 0 - 1
normalized_df
normalized_df_mean=(df-df.mean())/df.std() # normalize around mean
normalized_df_mean<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11584249084249085, 'fraction_numerical': 0.022893772893772892, 'mean_word_length': 3.8555555555555556, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '4286', 'license': 'MIT', 'n_tokens_mistral': 789, 'n_tokens_neox': 737, 'n_words': 185, 'path': 'Chapter 2 - Data Processing.ipynb', 'repo_name': 'andreamartinelli1974/CorsoML', 'size': '2184'}