<jupyter_start><jupyter_text>**Data Overview**<jupyter_code>display(rawData.shape)
display(rawData.head())<jupyter_output><empty_output><jupyter_text>**Heat Map**<jupyter_code>from IPython.display import Image
from IPython.core.display import HTML 
Image(config['paths']['figures'] +'HeatMap.png')<jupyter_output><empty_output><jupyter_text>Preprocessing **Data Cleaning**- Removing string cols: nameOrig, nameDest- Drop previous model predictions and export as CSV for later<jupyter_code>#Create copy
processedData= rawData.copy()
processedData.drop(columns= ['nameOrig', 'nameDest','isFlaggedFraud'], axis=1, inplace= True)

#Check
display(processedData.head())
#Subset model predictions and target, export to CSV
previousPredictions= rawData.loc[:, ['isFraud', 'isFlaggedFraud']]

previousPredictions.to_csv(config['paths']['cleanedData'] + 'previousModelPredictions.csv')<jupyter_output><empty_output><jupyter_text>**Check for Multicollinearity**<jupyter_code>from statsmodels.stats.outliers_influence import variance_inflation_factor
from statsmodels.tools.tools import add_constant

#Remove object and bools
vifData= processedData.select_dtypes(exclude= ['bool', 'object'])

#init Stats_Check object and view VIF scores
from src.models.processing import Stats_Tests
test1= Stats_Tests(vifData)
test1.add_constant()
test1.VIF_analysis()
#Drop cols with highest collinearity and re-evaluate VIF scores
test1.drop_cols(columns= ['newbalanceOrig', 'newbalanceDest'])
test1.VIF_analysis()<jupyter_output><empty_output><jupyter_text>**Update processedData for Export**<jupyter_code>#Get column names to subset from test1
test1Cols=list(test1.data.columns)
test1Cols.pop()
test1Cols
#Subset cols to export final dataframe
processedDataCols= list(processedData.select_dtypes(include=['object', 'bool']).columns)
updatedCols= test1Cols + processedDataCols
updatedCols.sort()
toExportCsv= processedData[updatedCols]

#Check
display(toExportCsv.head())<jupyter_output><empty_output><jupyter_text>Export Processed Data<jupyter_code>from src.data.import_export_data import save_csv
save_csv(toExportCsv, path= config['paths']['cleanedData'], filename= 'processedData.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11890660592255126, 'fraction_numerical': 0.005466970387243736, 'mean_word_length': 11.0, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '188773', 'license': 'MIT', 'n_tokens_mistral': 746, 'n_tokens_neox': 722, 'n_words': 139, 'path': 'notebooks/Modeling/05282021_Feature_Engineering.ipynb', 'repo_name': 'KishenSharma6/CreditFraudAnalysis', 'size': '2195'}