<jupyter_start><jupyter_text>Task - 1<jupyter_code>#import necessary libraries
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import datetime as dt
from sklearn.model_selection import train_test_split
import warnings
warnings.filterwarnings('ignore')
plt.rcParams['figure.figsize'] = [10,5]
#load the dataset
df = pd.read_excel('ANZ synthesised transaction dataset.xlsx')

#display fisrt 5 records
df.head()
#rows and columns in data
df.shape
#datatypes of features
df.dtypes
#info about the features
df.info()
#checking the missing values in data
df.isnull().sum()
#checking the percentage of missing values in data
np.round((df.isnull().sum()/ len(df))*100,2)
#summary statistics for numeric features
df.describe()
#summary statistics for categorical features
df.describe(exclude =np.number)
#unique values of all features
df.nunique()<jupyter_output><empty_output><jupyter_text>There are 100 unique customers(100 unique customer_id) in the transaction.<jupyter_code>#check for duplicated records
dups = df.duplicated()
print('Number of duplicate rows = %d' % (dups.sum()))
df[dups]<jupyter_output>Number of duplicate rows = 0<jupyter_text>No duplicate records are found. We can see that there are columns which are not of any use such as country, merchant_code, currency,bpay_biller_code these all have only one value so we can drop them.<jupyter_code>#rows and columns in data before dropping
df.shape
df.drop(['country', 'merchant_code', 'currency','bpay_biller_code'], axis = 1, inplace = True)
df.head(3)
#rows and columns in data after dropping
df.shape<jupyter_output><empty_output><jupyter_text>EDA (Exploratory Data Analysis)<jupyter_code>#feautres name in the data
df.columns
#extracting some particular features for further analysis
df_new = df[['status', 'card_present_flag', 'account', 'txn_description','merchant_id', 'first_name', 'balance','date', 
             'gender', 'age','merchant_suburb', 'merchant_state', 'extraction', 'amount','customer_id', 'movement']]
df_new.head()<jupyter_output><empty_output><jupyter_text>we are creating a new columns for monthwise and daywise<jupyter_code>df_new['weekday'] = df_new['date'].dt.day_name()
df_new['month_name'] = df_new['date'].dt.month_name()
df_new.head()<jupyter_output><empty_output><jupyter_text>Total number of transactions made on each day<jupyter_code>df_new['date'].value_counts()
df_new['weekday'].value_counts()<jupyter_output><empty_output><jupyter_text>Total number of transactions made by each customer<jupyter_code>df_new['customer_id'].value_counts()<jupyter_output><empty_output><jupyter_text>Total number of transactions in each month<jupyter_code>df_new['month_name'].value_counts()<jupyter_output><empty_output><jupyter_text>Total number of transactions by gender wise<jupyter_code>df_new['gender'].value_counts()<jupyter_output><empty_output><jupyter_text>Total number of transactions by age wise<jupyter_code>df_new['age'].value_counts()<jupyter_output><empty_output><jupyter_text>Minimum, Maximum, Average transaction amount by each customer<jupyter_code>pd.pivot_table(df_new,index='customer_id',values='amount',aggfunc=['min','max','mean'])<jupyter_output><empty_output><jupyter_text>Minimum, Maximum, Average, Total transaction amount monthwise<jupyter_code>pd.pivot_table(df_new,index='month_name',values='amount',aggfunc=['min','max','mean','sum'])<jupyter_output><empty_output><jupyter_text>Minimum, Maximum, Average, Total transaction amount day wise<jupyter_code>pd.pivot_table(df_new,index='weekday',values='amount',aggfunc=['min','max','mean','sum'])<jupyter_output><empty_output><jupyter_text>Top 10 customers<jupyter_code>topcustomer= df['first_name'].value_counts(sort=True).head(10)
topcustomer<jupyter_output><empty_output><jupyter_text>Minimum, Maximum, Average, Total transaction amount month wise<jupyter_code>pd.pivot_table(df_new,index='month_name',values='balance',aggfunc=['min','max','mean','sum'])<jupyter_output><empty_output><jupyter_text>Minimum, Maximum, Average, Total transaction amount day wise<jupyter_code>pd.pivot_table(df_new,index='weekday',values='balance',aggfunc=['min','max','mean','sum'])
df_new.head(3)
sns.countplot(data=df_new,x=df_new['gender'])
plt.title('Total number of Female vs Male')
sns.countplot(data=df_new,x=df_new['age'])
plt.title('Age wise Transactions')
sns.countplot(data=df_new,x=df_new['weekday'])
plt.title('Total Weekly Transactions')
sns.countplot(data=df_new,x=df_new['month_name'])
plt.title('Total Monthly Transactions')
sns.countplot(df_new['card_present_flag'])
plt.title('Card vs No Card Payment')
df_new['card_present_flag'].value_counts()
sns.countplot(df_new['movement'])
plt.title('Debit vs Credit')
df_new['movement'].value_counts()
sns.countplot(df_new['merchant_state'])
plt.title('Count vs Varioius state of merchant')
df_new['merchant_state'].value_counts()
fig,ax = plt.subplots(figsize=(10,10)) # (height,width)
topcustomer.plot.barh()
ax.set(title="Top 10 Customer",
      xlabel="Number of transaction made",
      ylabel="Name")
sns.countplot(df_new['txn_description'])
plt.title('Transaction Description')
sns.countplot(df_new['status'])
plt.title('Status')
sns.countplot(df_new['gender'],hue=df_new['card_present_flag'])
plt.title('Gender vs card_present_flag')
sns.countplot(df_new['gender'],hue=df_new['movement'])
plt.title('Gender vs movement')
plt.figure(figsize=(15, 10))
sns.countplot(df_new['gender'],hue=df_new['month_name'])
plt.title('Gender vs month wise')
plt.figure(figsize=(15, 10))
sns.countplot(df_new['gender'],hue=df_new['weekday'])
plt.title('Gender vs weekday')
plt.figure(figsize=(20, 10))
sns.countplot(df_new['age'],hue=df_new['month_name'])
plt.title('Age vs Month wise')
plt.figure(figsize=(7, 7))
df_new.weekday.value_counts().plot.pie(autopct='%1.1f%%', startangle=60)
plt.title("Transactions on daily basis \n")
plt.show()
plt.figure(figsize=(7, 7))
df_new.month_name.value_counts().plot.pie(autopct='%1.1f%%', startangle=60)
plt.title("Transactions on monthly basis \n")
plt.show()
plt.figure(figsize=(10, 6))
sns.lineplot(x = 'age', y = 'balance', data = df_new).set(title = "Age group with mode balance \n")
plt.show()
sns.heatmap(df_new.corr(),annot=True,cmap="YlGnBu")
plt.title("Correlation between features \n")<jupyter_output><empty_output><jupyter_text>Task - 2 : Predictive Analytics For this task, you’ll likely need to use statistical software such as R, SAS, or Python.Using the same transaction dataset, identify the annual salary for each customerExplore correlations between annual salary and various customer attributes (e.g. age). These attributes could be those that are readily available in the data (e.g. age) or those that you construct or derive yourself (e.g. those relating to purchasing behaviour). Visualise any interesting correlations using a scatter plot.Build a simple regression model to predict the annual salary for each customer using the attributes you identified aboveHow accurate is your model? Should ANZ use it to segment customers (for whom it does not have this data) into income brackets for reporting purposes?For a challenge: build a decision-tree based model to predict salary. Does it perform better? How would you accurately test the performance of this model?<jupyter_code>from sklearn.linear_model import LinearRegression
from sklearn.tree import DecisionTreeClassifier, DecisionTreeRegressor
#load the dataset
df = pd.read_excel('ANZ synthesised transaction dataset.xlsx')

#display fisrt 5 records
df.head()
df.columns
df['txn_description'].unique()
df_salary = df[df["txn_description"]=="PAY/SALARY"].groupby('customer_id').mean().reset_index()
df_salary.head()
df_salary=df_salary.drop(['card_present_flag','merchant_code'],axis=1)
df_salary=df_salary.rename(columns = {'amount':'salary'})
df_salary.head(3)
df_salary.columns
#rows and columns
df_salary.shape<jupyter_output><empty_output><jupyter_text>correlation between salary and all other features<jupyter_code>df_salary.corr()
sns.heatmap(df_salary.corr(),annot=True)<jupyter_output><empty_output><jupyter_text>Visualise correlations using scatter plot<jupyter_code>#scatter plot between age and balance
sns.scatterplot(x=df_salary.age, y=df_salary.balance,color='red')
#scatter plot between age and salary
sns.scatterplot(x=df_salary.age, y=df_salary.salary,color='orange')
#scatter plot between salary and balance
sns.scatterplot(x=df_salary.salary, y=df_salary.balance,color= 'green')
#scatter plot between balance and salary 
sns.scatterplot(x=df_salary.balance, y=df_salary.salary,color= 'magenta')
df_salary.dtypes<jupyter_output><empty_output><jupyter_text>Linear Regression:  OLS Method:<jupyter_code>import statsmodels.api as sm
from sklearn.metrics import mean_absolute_error
from sklearn.metrics import mean_squared_error
from sklearn.metrics import r2_score
from statsmodels.tools.eval_measures import rmse
from sklearn import metrics
X= df_salary.iloc[:,[1,2]]
X= sm.add_constant(X)

y =df_salary['salary']
X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.3, random_state = 10)
model =sm.OLS(y_train,X_train).fit()
model.summary()
y_pred = model.predict(X_test)
print(f"Mean Absolute Error of our model is : {mean_absolute_error(y_test, y_pred)}")
print(f"Mean Squared Error of our model is : {mean_squared_error(y_test, y_pred)}")
print(f"Root Mean Squared Error of our model is : {np.sqrt(metrics.mean_squared_error(y_test, y_pred))}")
print(f"Coefficient of determination: {r2_score(y_test, y_pred)}")
# Comparing Actual vs Predicted test results
results = pd.DataFrame({"Actual": y_test, "Predicted": y_pred})
results.reset_index(drop=True).head(10)<jupyter_output><empty_output><jupyter_text>Linear Regression:<jupyter_code>X= df_salary.iloc[:,[1,2]]

y =df_salary['salary']

X_train, X_test, y_train, y_test = train_test_split(X,y, test_size = 0.3, random_state = 10)
linreg = LinearRegression()
linreg.fit(X_train, y_train)
# Interpreting Model Coefficients
print('Intercept : ',linreg.intercept_)
print('Co efficient : ',linreg.coef_)
# Making predictions with our model
y_pred = linreg.predict(X_test)
y_pred[:5]
print(f"Mean Absolute Error of our model is : {mean_absolute_error(y_test, y_pred)}")
print(f"Mean Squared Error of our model is : {mean_squared_error(y_test, y_pred)}")
print(f"Root Mean Squared Error of our model is : {np.sqrt(metrics.mean_squared_error(y_test, y_pred))}")
print(f"Coefficient of determination: {r2_score(y_test, y_pred)}")
# Comparing Actual vs Predicted test results
results1 = pd.DataFrame({"Actual": y_test, "Predicted": y_pred})
results1.reset_index(drop=True).head(10)<jupyter_output><empty_output><jupyter_text>Decision Tree<jupyter_code>from sklearn.metrics import accuracy_score
X= df_salary.iloc[:,[1,2]]

y =df_salary['salary']
X_train,X_test,y_train,y_test = train_test_split(X,y,random_state=10,test_size=0.3)

model = DecisionTreeRegressor(random_state = 10)
decision_tree = model.fit(X_train, y_train)
# Making predictions with our model
y_pred = decision_tree.predict(X_test)
y_pred[:5]
print(f"Mean Absolute Error of our model is : {mean_absolute_error(y_test, y_pred)}")
print(f"Mean Squared Error of our model is : {mean_squared_error(y_test, y_pred)}")
print(f"Root Mean Squared Error of our model is : {np.sqrt(metrics.mean_squared_error(y_test, y_pred))}")
print(f"Coefficient of determination: {r2_score(y_test, y_pred)}")
# Comparing Actual vs Predicted test results
results1 = pd.DataFrame({"Actual": y_test, "Predicted": y_pred})
results1.reset_index(drop=True).head(10)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13107383710876708, 'fraction_numerical': 0.008300190212692375, 'mean_word_length': 8.877882152006832, 'pattern_counts': {'":': 6, '<': 88, '<?xml version=': 0, '>': 88, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '613081', 'license': 'Apache-2.0', 'n_tokens_mistral': 4114, 'n_tokens_neox': 3943, 'n_words': 877, 'path': 'ANZ_ Virtual_ Internship _ Task 2_ Swaruba .ipynb', 'repo_name': 'Swaruba-Paulraj/ANZ-virtual-internship-task', 'size': '11566'}