<jupyter_start><jupyter_text>Data Cleaning<jupyter_code>import pandas as pd
import numpy as np
df = pd.read_csv('../data/raw/train.csv')
df.head()
## convert the date strings to datetime
for i in range(3, 8):
    df.iloc[:, i] = pd.to_datetime(df.iloc[:, i], format="%b-%y")
# add a column for number of years data and convert payment notes to boolean
df['years_data'] = df.iloc[:, 3:8].max(axis=1).dt.year + 1 - df.iloc[:, 3:8].min(axis=1).dt.year
df['has_payment_note'] = np.where(df.payment_note_date.notnull(), 1, 0)
df.head()
# drop unnecessary columns
df.drop(columns=['payment_note_date', 'financials_date', 'financials_date-1', 'financials_date-2', 'financials_date-3', 'financials_date-4'], inplace=True)
# rearrange cols
col = df.pop('has_payment_note')
df.insert(1, 'has_payment_note', col)
# rearrange cols
col = df.pop('years_data')
df.insert(3, 'years_data', col)
#imputing na values to 0
df_clean = df.fillna(0)
df_clean.head()
df_clean.to_csv('../data/cleaned/train.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1750245821042281, 'fraction_numerical': 0.018682399213372666, 'mean_word_length': 5.925170068027211, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '169009', 'license': 'MIT', 'n_tokens_mistral': 409, 'n_tokens_neox': 399, 'n_words': 86, 'path': 'notebooks/data_cleaning.ipynb', 'repo_name': 'ChrisRSA/credit_score', 'size': '1017'}