<jupyter_start><jupyter_text>Size of original dataset is (1670214, 37)Size of updated dataset is  (1670214, 26)which proves that we have removed columns in which high number of column values were empty, but even then we have plenty of data to analyze.<jupyter_code>'''
Visualising columns with higher number of missing values
'''
Columns_with_Missing_values = df.isnull().mean()
# Only taking those columns where missing values are present
Columns_with_Missing_values = Columns_with_Missing_values[Columns_with_Missing_values.values > 0]
plt.figure(figsize=(20,4))
Columns_with_Missing_values.sort_values(ascending=False).plot(kind='bar')
plt.title('List of Columns where null values are not null but less than 30%')
plt.show()

'''
Now, lets try to impute values in those columns who have less then 30% missing values.
First, lets print such columns
'''
df[df.columns[ round(100*(df.isnull().sum()/len(df.index)), 2) > 0 ]].head()
# AMT_GOODS_PRICE is of float datatype
print("Total number of Missing values in column is ", df.loc[:, "AMT_ANNUITY"].isnull().sum())
# Imputing with mean is risky as outiers would always push mean to a faulty value
print("Standard Deviation of the column is ", df.loc[:, "AMT_ANNUITY"].std())
print("Mean of the column is ", df.loc[:, "AMT_ANNUITY"].mean())
print("Median of the column is", df.loc[:, "AMT_ANNUITY"].median())
# A high standard deviation indicates that the data points are spread out over a wider range of values.
# So, we will be using Median to impute missing values
df.loc[np.isnan(df["AMT_ANNUITY"]), ["AMT_ANNUITY"]] = df["AMT_ANNUITY"].median()
# Confirming whether all the NaN have been replacd with median
print(df.loc[:, "AMT_ANNUITY"].isnull().sum())
# Plotting the boxplot
plt.boxplot(df["AMT_ANNUITY"])
# PRODUCT_COMBINATION is a Categorical value since it only takes small set of values
df["PRODUCT_COMBINATION"] = df["PRODUCT_COMBINATION"].astype("category")
df["PRODUCT_COMBINATION"].value_counts()
# Since Cash has largest number of entries in the column, we would be imputing missing values with it.
df.loc[pd.isnull(df["PRODUCT_COMBINATION"]), ["PRODUCT_COMBINATION"]] = "Cash"
# Checking missing values of column after imputing
print("Missing entries in column are ", df.loc[:, "PRODUCT_COMBINATION"].isnull().sum())
'''
We took out rest of the columns which has missing values. We will analyze their min/max/sd values and based on that
we will be imputing NaN value(s)
'''
print("Min value is : ")
print(df[["AMT_GOODS_PRICE", "CNT_PAYMENT"]].min())

print("\nMax value is : ")
print(df[["AMT_GOODS_PRICE", "CNT_PAYMENT"]].max())

print("\nStandard Deviation is : ")
print(df[["AMT_GOODS_PRICE", "CNT_PAYMENT"]].std())
# After analysing the data, it seems that outliers might be present in following columns.
# Example Max value is hugely greater then normal values, and comparebly standard deviation also seems large
# So imputing with median
columns = ["AMT_GOODS_PRICE"]
for col in columns:
    df.loc[np.isnan(df[col]), [col]] = df[col].median()
# After analysing the data, it seems that outliers are not present in following columns.
# Example Max value feels like within range, and standard deviation is also less
# So imputing with mean
columns = ["CNT_PAYMENT"]
for col in columns:
    df.loc[np.isnan(df[col]), [col]] = df[col].mean()
'''
Since columns have been imputed, lets check whether any column is still left with NaN values
'''
df[df.columns[ round(100*(df.isnull().sum()/len(df.index)), 2) > 0 ]].head()
print("Shape of the dataframe is ", df.shape)
print("\nRows having missing values in rows\n", df[df.isnull().sum(axis=1) > 0].index)
print("\nRows having missing values in rows\n", len(df[df.isnull().sum(axis=1) > 0].index))
'''
There is only 1 rows which contains missing values.
Since, we have already imputed missing columns, and we have more then 3 lakhs rows, so we can delete such rows
'''
df=df[df.isnull().sum(axis=1) == 0]
print("Updated Shape of the dataframe is ", df.shape)
# DataFrame info about columns. Lets make sure that type of data frame columns is correctly updated
df.info()
'''
After analysing following columns, they seem to be of category type
'''
category_columns = ["NAME_CONTRACT_TYPE", "WEEKDAY_APPR_PROCESS_START", "FLAG_LAST_APPL_PER_CONTRACT", 
                    "NAME_CASH_LOAN_PURPOSE", "NAME_CONTRACT_STATUS", "NAME_PAYMENT_TYPE", 
                    "CODE_REJECT_REASON", "NAME_CLIENT_TYPE", "NAME_GOODS_CATEGORY", "NAME_PORTFOLIO", "NAME_PRODUCT_TYPE",
                    "CHANNEL_TYPE", "NAME_SELLER_INDUSTRY", "NAME_YIELD_GROUP"]
for col in category_columns:
    df[col] = df[col].astype('category')
# DataFrame info about columns. Lets make sure that type of data frame columns is correctly updated
df.info()
'''
As seen while imputing, there seems to be outlier presents across different numeric columns.
If outliers are removed, they will cause issues in our analysation process.
Lets visualize them through boxplot first
'''
numeric=["float64", "int64"]
for col in df.select_dtypes(include=numeric).columns:
        plt.figure()
        df.boxplot([col])
        plt.show()

df.head()
# Setting SK_ID_CURR as the index of the dataframe so that its easier to join and read/analyze data
df.set_index("SK_ID_CURR", inplace=True)
df.head()
'''
As seen in above figures too, there seems to be many outliers present across columns.
Lets remove outliers present in int and float columns
'''
numeric=["float64", "int64"]
desired_col=[]
for col in df.select_dtypes(include=numeric).columns:
    if col != "SK_ID_CURR":
        desired_col.append(col)
df_excluded = df.loc[:, desired_col]
df_excluded.head()
print("Shape of the dataframe is ", df.shape)
print("\nRows having missing values in rows\n", len(df_excluded[df_excluded.isnull().sum(axis=1) > 0].index))
print("\nRows having missing values in rows\n", df_excluded.isnull().sum() > 0)
df_excluded.describe()
df_excluded.head()
z = np.abs(stats.zscore(df_excluded))
print(z)
# Checking whether there are NaN present in z which might cause problems while filtering out outliers
np.argwhere(np.isnan(z))
# Lets keep a threshold of 3. Basically if z score is more then 3(after removing sign), then treating it as outlier
# Lets see how many outliers exists
threshold = 3
print(np.where(z > 3))
'''
The first array contains the list of row numbers and second array respective column numbers, 
which mean z[1670183][9] have a Z-score higher than 3.
'''
print(z[1670183][9])
# Removing all the outliers
df_excluded = df_excluded[(z < 3).all(axis=1)]
df_excluded.head()
print("df shape is ", df.shape)
print("df_excluded shape is ", df_excluded.shape)
print("%age of rows deleted during outlier removal process is", round(100*((df.shape[0]-df_excluded.shape[0])/df.shape[0]), 2))
df_excluded_columns = df_excluded.columns
df_columns = df.columns
df_columns_not_present_in_excluded_columns_df = list( set(df_columns) - set(df_excluded_columns) )
df_updated = copy.deepcopy(df_excluded.join(df[df_columns_not_present_in_excluded_columns_df]))
print("\nUpdated Dataframe shape is ", df_updated.shape)
df_updated.head()
df_updated.info()
'''
After handling outliers across various columns, lets draw boxplots of numeric columns, and find out if dataframe looks good
Lets visualize them
'''
numeric=["float64", "int64"]
for col in df_updated.select_dtypes(include=numeric).columns:
        plt.figure()
        df_updated.boxplot([col])
        plt.show()

'''
After looking at the boxplots, we can say that outliers have been removed.
There are some columns for which boxplots, still appears very small, but after checking their Standard Deviation, 
we can say that all values greater/less then 3*Standard Deviaton have been removed 
'''
df_updated.head()
'''
Lets derive some additional metrics which might prove helpful in during analysing process
'''
df_updated["Credit_Vs_Annuity"] = round(df_updated["AMT_CREDIT"] / df_updated["AMT_ANNUITY"], 2)
df_updated.head()

'''
Lets identify some numerical continuous variables, and check their distribution after binning.
It should give us fair idea about will binning be helpful
'''
some_continuous_variables = ["AMT_ANNUITY", "AMT_APPLICATION", "Credit_Vs_Annuity", "AMT_CREDIT", 
                             "AMT_GOODS_PRICE", "DAYS_DECISION", "SELLERPLACE_AREA", 
                             "CNT_PAYMENT", "AMT_GOODS_PRICE"]
for col in some_continuous_variables:
    df_updated.hist(col, bins=5, figsize=(12, 8))
df_updated.info()
df_updated.head()
df_updated.info()
'''
Lets remove columns which only contain single value
'''''
df_updated = df_updated.loc[:,df_updated.apply(pd.Series.nunique) != 1]
print("Current shape of data frame is ", df_updated.shape)
'''
Lets check the frequency of categorical values for df_updated
'''
category=["category"]
for col in df_updated.select_dtypes(include=category).columns:
    df_updated[col].value_counts().plot(kind='bar')
    plt.title(col)
    plt.show()
    
    
'''
Lets draw boxplots of numeric columns, and find out if dataframe looks good
Lets visualize them
'''
numeric=["float64", "int64"]
for col in df_updated.select_dtypes(include=numeric).columns:
    df_updated.boxplot([col])
    plt.title(col)
    plt.show()

# Lets find out the correlation matrix on the datasets
loan_correlation = df_updated.corr()
loan_correlation
# Plotting the correlation matrix
f, ax = plt.subplots(figsize=(14, 9))
sns.heatmap(loan_correlation, 
            xticklabels=loan_correlation.columns.values,
            yticklabels=loan_correlation.columns.values,annot= True)
plt.show()
category=["category"]
numeric=["float64", "int64"]
fig, ax = plt.subplots(figsize=(20, 10))
for category_col in df_updated.select_dtypes(include=category).columns:
    if category_col != "NAME_CONTRACT_STATUS":
        for numeric_col in df_updated.select_dtypes(include=numeric).columns:
            sns.catplot(x=category_col, y=numeric_col, hue="NAME_CONTRACT_STATUS", kind="box", data=df_updated, ax=ax);
            plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11669497452292936, 'fraction_numerical': 0.011090018982915376, 'mean_word_length': 5.182828906732551, 'pattern_counts': {'":': 2, '<': 6, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '149975', 'license': 'MIT', 'n_tokens_mistral': 3298, 'n_tokens_neox': 3122, 'n_words': 1017, 'path': 'Data_Science_Projects/Credit_EDA_Case_Study/Previous_Application_Analyze_Python_Notebook.ipynb', 'repo_name': 'akhilminhas/Projects', 'size': '10009'}