<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Load Data2&nbsp;&nbsp;Demo of Cleaning Functions2.1&nbsp;&nbsp;Columns2.2&nbsp;&nbsp;Outliers2.3&nbsp;&nbsp;Transformations<jupyter_code>import datetime as dt
import sys
from pathlib import Path

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
print(sys.executable)
print(sys.version)
print(f"Pandas {pd.__version__}")
print(f"Seaborn {sns.__version__}")
sys.path.append(str(Path.cwd().parent / 'src' / 'codebook'))
%load_ext autoreload
%autoreload 2

%matplotlib inline
# %config InlineBackend.figure_format = 'svg'
plt.style.use('raph-base')

from IPython.core.interactiveshell import InteractiveShell
InteractiveShell.ast_node_interactivity = 'all'

pd.set_option('precision', 2)
pd.set_option('display.max_columns', 30)
pd.set_option('display.expand_frame_repr', False)
pd.set_option('max_colwidth', 800)
import src.codebook.EDA as EDA
import src.codebook.clean as clean<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>df = pd.read_csv("../data/realWorldTestData.csv", 
                 low_memory=False, 
                 nrows=1000,
                 usecols=[2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 18]
                 )
df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
RangeIndex: 1000 entries, 0 to 999
Data columns (total 14 columns):
 #   Column                         Non-Null Count  Dtype  
---  ------                         --------------  -----  
 0   target_event                   1000 non-null   object 
 1   NUM_CONSEC_SERVICES            1000 non-null   int64  
 2   SUM_INVOICE_AMOUNT_IN_SERVICE  1000 non-null   float64
 3   SUM_INVOICE_AMOUNT             1000 non-null   float64
 4   NUM_EVENTS                     1000 non-null   int64  
 5   FIRST_EVT                      1000 non-null   object 
 6   LAST_EVT                       1000 non-null   object 
 7   LAST_MILEAGE                   1000 non-null   float64
 8   MEAN_MILEAGE_PER_MNTH          1000 non-null   float64
 9   AVG_DIFF_MNTH                  1000 non-null   int64  
 10  age_mnth                       1000 non-null   int64  
 11  KANTON_LICENCE_PLATE           991 non-null    object 
 12  INSPECTION_INTERVAL_UID        1000 non-null   ob[...]<jupyter_text>Demo of Cleaning Functions  Columns<jupyter_code># Prettify the column names 

df = clean.prettify_column_names(df)

# Check result
df.columns
# Delete columns

df_del = clean.delete_columns(df, cols_to_delete=["target_event", "first_evt"])

assert df_del.shape[1] == (df.shape[1] - 2)
# Downcast dtypes

df_lean = clean.downcast_dtypes(df)

# Check result
df_lean.dtypes<jupyter_output>Original df size before downcasting: 0.46 MB
 New df size after downcasting:0.16 MB<jupyter_text>A word of Warning: Downcasting the numerical dtypes this way can lead to problems with the power transforms that are demonstrated below. That's why we continue with the original frame here.  Outliers<jupyter_code># Count Outliers using the IQR-Method, with a distance of X

clean.count_outliers_IQR_method(df, iqr_dist=2)
# Remove outliers in two selected columns

outlier_cols=["avg_diff_mnth", "mean_mileage_per_mnth"]

df_outliers, deleted_idx = clean.remove_outliers_IQR_method(
    df,
    outlier_cols=outlier_cols,
    iqr_dist=2,
    return_idx_deleted=True
)
# Because we have set the `return_idx_deleted` param to true
# We have also received a list of the removed outliers' index values

print(deleted_idx)
# (As a sidenote) There is also a helper function, that is simply returning the outlier values 
# with the lower, upper threshold for an interable
outliers, lower_thx, upper_thx = clean.get_outlier_values_with_iqr_method(
    df["avg_diff_mnth"], iqr_dist=2
)

print(len(outliers))
print(f"Lower threshold: {lower_thx}")
print(f"Upper threshold: {upper_thx}")
# Winsorize outliers in two more columns
# (We use the IQR-Method output from above to define the quantiles)

w_dict = {
    "sum_invoice_amount": (None, 0.035),
    "last_mileage": (0.0005, 0.014),
}

df_w = clean.winsorize_outliers(df, w_dict)

# Check results
print(list(zip(
        df_w[list(w_dict.keys())].min(),
        df_w[list(w_dict.keys())].max()
        )))<jupyter_output>[(0.0, 4889.35), (1296.0, 171053.0)]<jupyter_text>Transformations<jupyter_code>df_transform = df[["last_mileage", "sum_invoice_amount", "mean_mileage_per_mnth"]]. copy()

EDA.plot_distr_histograms(df_transform)
df_log = clean.transform_data(df_transform, method="log")

EDA.plot_distr_histograms(df_log)
df_log10 = clean.transform_data(df_transform, method="log10")

EDA.plot_distr_histograms(df_log10)
df_bc = clean.transform_data(df_transform, method="box_cox")

EDA.plot_distr_histograms(df_bc)
df_jy = clean.transform_data(df_transform, method="yeo_johnson")

EDA.plot_distr_histograms(df_jy)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10158013544018059, 'fraction_numerical': 0.03755386825364252, 'mean_word_length': 3.7458617332035056, 'pattern_counts': {'":': 2, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '668215', 'license': 'MIT', 'n_tokens_mistral': 1883, 'n_tokens_neox': 1685, 'n_words': 389, 'path': 'demo/dev_clean.ipynb', 'repo_name': 'rbuerki/codebook', 'size': '4873'}