<jupyter_start><jupyter_text>Fraudulent Transactions Prediction Predict fraudlent transactions- import dependencies- import dataset- explore data and visualize- perform machine learning- test model on testing set<jupyter_code># Install Kaggle
! pip install -q kaggle

# import function to upload files (upload kaggle.json)
from google.colab import files
files.upload()
# Create directory for kaggle.json
# Copy and Set permission
! mkdir ~/.kaggle
! cp kaggle.json ~/.kaggle/
! chmod 600 ~/.kaggle/kaggle.json
%mkdir train
%mkdir test
! kaggle competitions download -c ieee-fraud-detection
! unzip train_transaction.csv.zip -d train
! unzip train_identity.csv.zip -d train
! unzip test_transaction.csv.zip -d test
! unzip test_identity.csv.zip -d test
import random
from typing import List
import numpy as np
import pandas as pd
import sklearn as sk
import seaborn as sns
import matplotlib.pyplot as plt
from sklearn.preprocessing import scale

# Machine Learning
from sklearn import linear_model
from sklearn.model_selection import train_test_split
transaction_features = ['TransactionID', 'isFraud', 'TransactionAmt', 'P_emaildomain', 'R_emaildomain']
identity_features = ['TransactionID', 'DeviceType']
train_files = ['train/train_transaction.csv', 'train/train_identity.csv']
test_files = ['test/test_transaction.csv', 'test/test_identity.csv']
def get_transaction_dataframe(file_name: str, transaction_features: List[str]) -> "DataFrame":
    df = pd.read_csv(file_name, usecols=transaction_features, low_memory=False)
    df = df.sample(frac=1).reset_index(drop=True)
    return df
def get_identity_dataframe(file_name: str, identity_features: List[str]) -> "DataFrame":
    df = pd.read_csv(file_name, usecols=identity_features, low_memory=False)
    df = df.sample(frac=1).reset_index(drop=True)
    return df
# Merging 2 datasource for training data
df_train_transaction = get_transaction_dataframe(train_files[0], transaction_features)
df_train_identity = get_identity_dataframe(train_files[1], identity_features)
df_full = pd.merge(df_train_transaction, df_train_identity, on='TransactionID', how='inner')
df_full.head()
frauds = df_full.loc[df_full['isFraud'] == 1]
non_frauds = df_full.loc[df_full['isFraud'] == 0]
ax = frauds.plot.scatter(x='TransactionAmt', y='isFraud', color='Orange', label='Fraud')
non_frauds.plot.scatter(x='TransactionAmt', y='isFraud', color='Blue', label='Normal', ax=ax)

plt.show()
sns.countplot(x='P_emaildomain', data=df_full)
df_full[['P_emaildomain']].groupby(['P_emaildomain'])['P_emaildomain'].count().nlargest(10)
r_email_domain_to_is_fraud = df_full[['R_emaildomain','isFraud']].groupby(['R_emaildomain'], as_index=False).mean().sort_values(by='isFraud', ascending=False)
sns.barplot(x="R_emaildomain", y="isFraud", data=r_email_domain_to_is_fraud)
# R consist of all email domains in P
r_email_domain_mapping = {}
r_email_domain_to_is_fraud = df_full[['R_emaildomain','isFraud']].groupby(['R_emaildomain'], as_index=False).mean().sort_values(by='isFraud', ascending=False)
r_emaildomain_isfraud_map = dict(zip(r_email_domain_to_is_fraud['R_emaildomain'], r_email_domain_to_is_fraud['isFraud']))
r_emaildomain_isfraud_map[np.nan] = 0
r_emaildomain_isfraud_map
# Data cleaning
device_mapping = {'desktop': 2, 'mobile': 1, np.nan: 0}
df_full['DeviceType'] = df_full['DeviceType'].map(device_mapping)
df_full['P_emaildomain'] = df_full['P_emaildomain'].map(r_emaildomain_isfraud_map)
df_full['R_emaildomain'] = df_full['R_emaildomain'].map(r_emaildomain_isfraud_map)
df_full.head()
x = df_full.drop(columns=['isFraud'])
y = df_full['isFraud']
X_train, X_test, Y_train, Y_test = train_test_split(x, y, test_size=0.3)
len(X_train), len(X_test), len(Y_train), len(Y_test)
X_train.head(), X_test.head(), Y_train.head()
log_reg = linear_model.LogisticRegression(C=1e5) # C -> Regulate against overfitting
log_reg.fit(X_train, Y_train)
print("Score: ", log_reg.score(X_test, Y_test))
# For submission in Kaggle

test_transaction_features = ['TransactionID', 'TransactionAmt', 'P_emaildomain', 'R_emaildomain']
# Repeat process for Testing Data
df_test_transaction = get_transaction_dataframe(test_files[0], test_transaction_features)
df_test_identity = get_identity_dataframe(test_files[1], identity_features)
df_test_full = pd.merge(df_test_transaction, df_test_identity, on='TransactionID', how='outer')

# Data cleaning
device_mapping = {'desktop': 2, 'mobile': 1, np.nan: 0}
df_test_full['DeviceType'] = df_test_full['DeviceType'].map(device_mapping)
df_test_full['P_emaildomain'] = df_test_full['P_emaildomain'].map(r_emaildomain_isfraud_map)
df_test_full['R_emaildomain'] = df_test_full['R_emaildomain'].map(r_emaildomain_isfraud_map)
df_test_full.head()

submission_X_test = df_test_full.loc[:, :]
Y_pred = np.array(log_reg.predict(submission_X_test))


output = pd.DataFrame({'TransactionID': submission_X_test.TransactionID,
                       'isFraud': Y_pred})
output.to_csv('submission.csv', index=False)
len(Y_pred)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12972216669998002, 'fraction_numerical': 0.004997001798920648, 'mean_word_length': 8.114754098360656, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '190332', 'license': 'MIT', 'n_tokens_mistral': 1800, 'n_tokens_neox': 1736, 'n_words': 326, 'path': 'fraudulent_transactions_prediction.ipynb', 'repo_name': 'samueljklee/fraudulent-transactions-prediction', 'size': '5003'}