<jupyter_start><jupyter_text>Import Statements<jupyter_code>import pandas as pd
import numpy as np

from sklearn.preprocessing import MinMaxScaler

import matplotlib.pyplot as plt
import seaborn as sbn<jupyter_output><empty_output><jupyter_text>Load Datasets<jupyter_code>train_labels_df = pd.read_csv("../datasets/train_labels.csv")
train_values_df = pd.read_csv("../datasets/train_values.csv")
test_values_df = pd.read_csv("../datasets/test_values.csv")
submission_format_df = pd.read_csv("../datasets/submission_format.csv")<jupyter_output><empty_output><jupyter_text>Merge train data<jupyter_code>train_df = train_values_df.merge(train_labels_df, on="building_id")<jupyter_output><empty_output><jupyter_text>Function to factorize columns with dtype == np.object<jupyter_code>uniques_values = {}
def convert_object_to_numeric(data_frame : pd.DataFrame):
    for col_name in data_frame:
        if data_frame[col_name].dtypes == np.object:
            codes, uniques = pd.factorize(data_frame[col_name])
            uniques_values[col_name] = uniques
            data_frame[col_name] = pd.Series(codes)<jupyter_output><empty_output><jupyter_text>Convert train_df and test_values_df<jupyter_code>convert_object_to_numeric(train_df)
convert_object_to_numeric(test_values_df)<jupyter_output><empty_output><jupyter_text>Check for null values<jupyter_code>train_df.isnull().sum().sum()
test_values_df.isnull().sum().sum()<jupyter_output><empty_output><jupyter_text>Save dataframes<jupyter_code>train_df.to_csv("../datasets/preprocessed/train_no_object.csv", index=False, header=True)
test_values_df.to_csv("../datasets/preprocessed/test_no_object.csv", index=False, header=True)<jupyter_output><empty_output><jupyter_text>Mormalize large valued columns.<jupyter_code>def get_max_value(data_frame : pd.DataFrame):
    return [(data_frame[col_name].max(), col_name) for col_name in data_frame]
get_max_value(train_df)
get_max_value(test_values_df)
col_names = ['geo_level_1_id', 'geo_level_2_id', 'geo_level_3_id', 'age']
def normalize_columns(data_frame : pd.DataFrame):
    for col_name in col_names:
        values = data_frame[col_name].values
        scaler = MinMaxScaler(feature_range=(0, 10))
        scaled_values = scaler.fit_transform(values.reshape(-1, 1))
        data_frame[col_name] = pd.Series(scaled_values.flatten())
normalize_columns(train_df)
normalize_columns(test_values_df)<jupyter_output><empty_output><jupyter_text>Save normalized columns<jupyter_code>train_df.to_csv("../datasets/preprocessed/train_normalize.csv", index=False, header=True)
test_values_df.to_csv("../datasets/preprocessed/test_normalize.csv", index=False, header=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11839166046165302, 'fraction_numerical': 0.0029784065524944155, 'mean_word_length': 7.897350993377484, 'pattern_counts': {'":': 0, '<': 37, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '642979', 'license': 'MIT', 'n_tokens_mistral': 978, 'n_tokens_neox': 964, 'n_words': 115, 'path': 'Richters_Predictor/scripts/data_preprocessing.ipynb', 'repo_name': 'Onotseike/DataDriven', 'size': '2686'}