<jupyter_start><jupyter_text>AML Detection Challenge         Starter Solution Notebook ImportsInclude URLs to relevant examples<jupyter_code>import pandas as pd                    #Data structuring and analysis
import numpy as np                     #Arrays, matrices, linear algebra
import networkx as nx                  #Network / graph analysis
import sklearn as sk                   #Data analysis, machine learning

import copy
import tqdm                            #Utility to display progress bar
import datetime                        #Date and time representations
import seaborn as sns                  #Plotting and visualisation
from matplotlib import pyplot as plt   #Plotting and visualisation<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>print('Downloading data locally')
!curl https://raw.githubusercontent.com/PaulMercerAI/AML19/master/data/20191111_Accounts.csv.xz -L | xzcat > accounts.csv
!curl https://raw.githubusercontent.com/PaulMercerAI/AML19/master/data/20191111_Transactions.csv.xz -L | xzcat > transactions.csv
#Load accounts data file
ac = pd.read_csv("accounts.csv")
print('Number of accounts: ', len(ac))

#Load transactions data file 
date = ['timestamp']
tx = pd.read_csv("transactions.csv", parse_dates=date)
print('Number of transactions: ', len(tx))
print('Number of labelled transactions: ', len(tx[tx.is_suspx!='Unknown']))<jupyter_output>Number of accounts:  10000
Number of transactions:  515721
Number of labelled transactions:  257787<jupyter_text>Inspect Data<jupyter_code># Printing top 10 rows of accounts data
print('Top 10 rows from the accounts dataset:')
display(ac.head(10))

#printing top 10 rows of transactions data
print('\nTop 10 rows from the transactions dataset:')
display(tx.head(10))<jupyter_output>Top 10 rows from the accounts dataset:<jupyter_text>Data Dictionary|Field Name|Data Type|Description||----------|---------|-----------||id|Int|Id number of the transaction||timestamp|Date Type|Time at which the transaction took place||sender_account|String|Unique Id the account from which the transaction was made. This will match to the accounts table.||receiver_account|String|Unique Id of account to which the transaction was made. This will match to the accounts table.||amount|Float|The value of the transaction made.||is_suspx|String|Flags if the transaction is suspicious. This is a target column.||alert_id|String|Id of the alert that has been generated.||alert_type|String|Type of alert that has been generated. This is a target column.| Inspect an alert<jupyter_code># Next command gets the first A4 type alert id. 
# Hopefully the filter for A4 type alerts is obvious. 
# Then we select just the alert_id column to get an array (matrix with a single x dimension) of length len(tx)
# [0:1] means take 1 element from the array starting at element 0, but that returns an array, so to get just the value we add [0]
id = tx[tx.alert_type=='A4'].alert_id.unique()[0:1][0]
print("Inspecting alert id:"+id)
tx[tx.alert_id==id] 
#A4 type was selected because the sender/receiver arrangement is more interesting than usual for this type<jupyter_output>Inspecting alert id:36<jupyter_text>Visualise amount densities<jupyter_code>tx_0 = tx[tx['is_suspx']=='False'] #create a new matrix with the subset of transactions defined by the filter
tx_1 = tx[tx['is_suspx']=='True']  

#Don't worry too much about this. Just play with the density and alpha values to see what they do.
fig, ax = plt.subplots(figsize=(10,10))
_, bins, _ = plt.hist(tx_0['amount'], bins=100, density=True, alpha=0.8, label='FALSE')
_ = plt.hist(tx_1['amount'], bins=bins, alpha=0.8, density=True, label='TRUE')
plt.legend(loc='best')
plt.xlim([0,800])
plt.xlabel('amount'), plt.ylabel('p(amount)')
plt.show()
# https://matplotlib.org/api/pyplot_api.html<jupyter_output><empty_output><jupyter_text>Visualise alert patterns<jupyter_code># Take the first 6 alert IDs where alert type is A3
sample_a3 = tx[tx.alert_type=='A3'].alert_id.unique()[0:6] 
print("Sample of A3 Alert_Types:", sample_a3)

fig, ax = plt.subplots(2, 3, figsize=(15,10))

for count, alert in enumerate(sample_a3):

  # Taking a sample alert and generating a graph of the transactions where accounts are nodes and amounts are weights
  edgelist = tx.loc[tx.alert_id==alert,['sender_account', 'receiver_account', 'amount']].values.tolist()

  # Create a directional graph using the edge list
  graph = nx.DiGraph()
  edgelist_tuple = [tuple(x) for x in edgelist]
  graph.add_weighted_edges_from(edgelist_tuple)

  # Plotting the directional graph

  pos = nx.spring_layout(graph)
  nx.draw_networkx(graph, pos, with_labels=False, node_size = 20, ax=ax[count%2, count//2])

  labels = nx.get_edge_attributes(graph,'weight')
  nx.draw_networkx_edge_labels(graph,edge_labels=labels, pos=pos, ax=ax[count%2, count//2])
  ax[count%2, count//2].set_title(f'Alert_id: {alert}')

plt.tight_layout()
plt.show()<jupyter_output>Sample of A3 Alert_Types: ['83' '177' '27' '18' '78' '105']<jupyter_text>Join balance with transactions<jupyter_code>#initialise balance
balance = dict(zip(ac.acct_id, ac.initial_deposit))

#monitor debits and credits
for i in tqdm.tqdm(range(len(tx))):
    
    #debit sender account
    sender = tx.at[i,'sender_account']
    sender_balance_pre = balance[sender]
    sender_balance_post = sender_balance_pre - tx.at[i,'amount']
    
    #credit receiver account
    receiver = tx.at[i,'receiver_account']
    receiver_balance_pre = balance[receiver]
    receiver_balance_post = receiver_balance_pre + tx.loc[i,'amount']
    
    #create new columns in transactions
    tx.at[i,'sender_balance_pre'] = sender_balance_pre
    tx.at[i,'sender_balance_post'] = sender_balance_post
    tx.at[i,'receiver_balance_pre'] = receiver_balance_pre
    tx.at[i,'receiver_balance_post'] = receiver_balance_post
    
    #update balance dictionary
    balance[sender] = sender_balance_post
    balance[receiver] = receiver_balance_post<jupyter_output>100%|██████████| 515721/515721 [00:27<00:00, 18828.91it/s]<jupyter_text>Visualise balance profiles<jupyter_code>fig, ax = plt.subplots(figsize=(10,5))
a = []
a.append('b10e91b4-6ae5-4c9d-b9d3-cba066a0835e') #supicious instance
a.append('00cdcce8-17ed-4eb8-9739-1cff8cc99052') #not suspicious instance
label = ['False', 'True']
col = ['blue', 'red']

for i in range(2):
    temp = tx[tx['sender_account']==a[i]]
    bal = temp['sender_balance_pre']
    tim = (temp['timestamp'].dt.date - datetime.date(2017,1,1)).dt.days
    plt.plot(tim,bal,color=col[i],label=label[i])
plt.legend(loc='best')
plt.show()<jupyter_output><empty_output><jupyter_text>Generate Amount Profile Features<jupyter_code>def roll_count(x,w):
    return x.rolling(w).count()
def roll_sum(x,w):
    return x.rolling(w).sum()
def roll_mean(x,w):
    return x.rolling(w).mean()
def expand_count(x):
    return x.expanding().count()
def expand_sum(x):
    return x.expanding().sum()
def expand_mean(x):
    return x.expanding().mean()

# 7 day rolling profiles
print('Generating rolling profiles...')
tx.set_index('timestamp',inplace=True)
print('.....count')
tx['s_rolling_count_7D'] = tx.groupby('sender_account').amount.transform(roll_count, '7D')
print('.....sum amount')
tx['s_rolling_sum_7D'] = tx.groupby('sender_account').amount.transform(roll_sum, '7D')
print('.....mean amount')
tx['s_rolling_mean_7D'] = tx.groupby('sender_account').amount.transform(roll_mean, '7D')
#tx['sr_rolling_count_7D'] = tx.groupby(['sender_account','receiver_account']).amount.transform(roll_count, '7D')
#tx['sr_rolling_sum_7D'] = tx.groupby(['sender_account','receiver_account']).amount.transform(roll_sum, '7D')
#tx['sr_rolling_mean_7D'] = tx.groupby(['sender_account','receiver_account']).amount.transform(roll_mean, '7D')
tx.reset_index(inplace=True)

# expanding profiles
#print('Generating expanding profiles...')
#tx['s_expanding_count'] = tx.groupby('sender_account').amount.transform(expand_count)
#tx['s_expanding_sum'] = tx.groupby('sender_account').amount.transform(expand_sum)
#tx['s_expanding_mean'] = tx.groupby('sender_account').amount.transform(expand_mean)
#tx['sr_expanding_count'] = tx.groupby(['sender_account','receiver_account']).amount.transform(expand_count)
#tx['sr_expanding_sum'] = tx.groupby(['sender_account','receiver_account']).amount.transform(expand_sum)
#tx['sr_expanding_mean'] = tx.groupby(['sender_account','receiver_account']).amount.transform(expand_mean)<jupyter_output>Generating rolling profiles...
.....count
.....sum amount
.....mean amount<jupyter_text>Network Features<jupyter_code># This block takes quite a long time to run
def add_feature(tx,x,s):
    a = dict(x)
    for i in range(len(tx)):
        o = a.get(tx.at[i,'sender_account'])
        tx.at[i,'nameOrig'+s] = o
        d = a.get(tx.at[i,'receiver_account'])
        tx.at[i,'nameDest'+s] = d
    return (tx)
    
MG = nx.from_pandas_edgelist(tx, 'sender_account', 'receiver_account', create_using=nx.DiGraph())

degree = MG.degree()
tx = add_feature(tx,degree,'_degree')
print('degree feature added')

out_degree = MG.out_degree()
tx = add_feature(tx,out_degree,'_outdegree')
print('out-degree feature added')

in_degree = MG.in_degree()
tx = add_feature(tx,in_degree,'_indegree')
print('in-degree feature added')

G = nx.Graph(MG)
triangles = nx.triangles(G)
tx = add_feature(tx,triangles,'_triangles')
print('triangles feature added')

clust_coefficients = nx.clustering(G)
tx = add_feature(tx,clust_coefficients,'_clust')
print('clustering coeff feature added')
    
degree_centrality = nx.degree_centrality(G)
tx = add_feature(tx,degree_centrality,'_degcentral')
print('degree centrality feature added')

eigenvector_centrality = nx.eigenvector_centrality_numpy(G)
tx = add_feature(tx,eigenvector_centrality,'_eigcentral')
print('eigenvector centrality feature added')

betweenness_centrality = nx.betweenness_centrality(G)
tx = add_feature(tx,betweenness_centrality,'_bwcentral')
print('betweenness centrality feature added')

closeness_centrality = nx.closeness_centrality(G)
tx = add_feature(tx,closeness_centrality,'_closecentral')
print('closeness centrality feature added')

katz_centrality = nx.katz_centrality_numpy(G)
tx = add_feature(tx,katz_centrality,'_katzcentral')
print('katz centrality feature added')

load_centrality = nx.load_centrality(G)
tx = add_feature(tx,load_centrality,'_loadcentral')
print('load centrality feature added')

harmonic_centrality = nx.harmonic_centrality(G)
tx = add_feature(tx,harmonic_centrality,'_harmoniccentral')
print('harmonic centrality feature added')<jupyter_output>degree feature added
out-degree feature added
in-degree feature added
triangles feature added
clustering coeff feature added
degree centrality feature added
eigenvector centrality feature added
betweenness centrality feature added<jupyter_text>(Supervised) Machine Learning! Prep data<jupyter_code>from sklearn.ensemble import GradientBoostingClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import mean_squared_error
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import classification_report, confusion_matrix, roc_curve, auc
from sklearn.utils.multiclass import unique_labels

X = copy.deepcopy(tx)
y = copy.deepcopy(tx.is_suspx)

predict_df = X[X.is_suspx=='Unknown']
y = y[X.is_suspx!='Unknown']
y = y.factorize()[0]
X = X[X.is_suspx!='Unknown']

X = X.loc[:, ['amount', 's_rolling_count_7D', 's_rolling_sum_7D']]
predict_df = predict_df.loc[:, ['amount', 's_rolling_count_7D', 's_rolling_sum_7D']]

X_train, X_test, y_train, y_test = train_test_split(X, y)<jupyter_output><empty_output><jupyter_text>Build & validate model<jupyter_code>#add terminology cell in notebook
gb = GradientBoostingClassifier(
    max_depth=3,
    n_estimators=100,
    learning_rate=0.2
)
gb.fit(X_train, y_train)

print("Accuracy score (validation): {0:.3f}".format(gb.score(X_test, y_test)))
predictions = gb.predict(X_test)

print("Confusion Matrix:")
print(confusion_matrix(y_test, predictions))
print()
print("Classification Report")
print(classification_report(y_test, predictions))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1306715063520871, 'fraction_numerical': 0.018478798878072925, 'mean_word_length': 6.031902552204176, 'pattern_counts': {'":': 0, '<': 43, '<?xml version=': 0, '>': 44, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '552674', 'license': 'Apache-2.0', 'n_tokens_mistral': 4247, 'n_tokens_neox': 4057, 'n_words': 905, 'path': 'Introductory.ipynb', 'repo_name': 'PaulMercerAI/AML19', 'size': '12122'}