<jupyter_start><jupyter_text>Part 1: In your project you have been working with a bias dataset. Most probabely suupervised models were underfit. You should remember that this is a real dataset, and in real life most of the time you dont have access to clean and unbias dataset. In situations like this, none of the regression models will have a good fit for the dataset due to the large predence of the 0s. In the first part of this take-home exam, you are asked to be creative, and find a solution for this problem. For this dataset, you can break the problem, into a two-step machine learning model. First step is to classify the data points as positive revenue and zero revenue. This simply is a binary classification. You should remember that when you are dealing with biased datasets, accuracy is not a reliable scoring metric. So look for a robust scoring function. Next, you need to build a regression model to postprocess classification results. Here is a flow chart to show you the outline of the model. ![flowchart.png](attachment:flowchart.png)<jupyter_code>import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
# Import matplotlib & seaborn for visualisation
import simplejson as json
from pandas.io.json import json_normalize
import seaborn as sns 
from plotly.offline import init_notebook_mode, iplot
import plotly.graph_objs as go
from plotly import tools
import numpy as np
init_notebook_mode(connected=True)
import matplotlib as mpl
import matplotlib.pyplot as plt
%matplotlib inline
import scipy.stats as st
import missingno as msno
import ast
data=pd.read_csv('data.csv')
empty=[]
for index,row in data['customDimensions'].iteritems():
    #row2 = ast.literal_eval(row)
    row1=row.replace('[]',"{'index': 'None', 'value': 'None'}")
    row2=row1.replace('[','')
    row3=row2.replace(']','')
    row4=ast.literal_eval(row3)
    #row5 = ast.literal_eval(row3)
    empty.append(row4)
data.drop('customDimensions',axis=1,inplace=True)
empty_df = pd.DataFrame(empty)
data=data.merge(empty_df, right_index=True, left_index=True)
data
def todict(dic, key, value):
    if key in dic:
        dic[key].append(value)
    else:
        dic[key] = [value]
    return dic


def resolve_json(hitsdic, hits_json, key='NoneName'):
    if type(hits_json) == list:
        if len(hits_json) == 0:
            pass
        else:
            for subjson in hits_json:
                hitsdic = resolve_json(hitsdic, subjson)
    elif type(hits_json) == dict:
        for i in hits_json.keys():
            hitsdic = resolve_json(hitsdic, hits_json[i],i)
    else:
        hitsdic = todict(hitsdic, key, hits_json)
    return hitsdic


def complex_replace(x):
    dic = {}
    return resolve_json(dic, json.loads(x.replace('\'','\"'). \
                                        replace('TRUE','true'). \
                                        replace('True','true'). \
                                        replace('FALSE','false'). \
                                        replace('False','false'). \
                                        replace(', \"',', !&~'). \
                                        replace('\", ','!&~, '). \
                                        replace('\": ','!&~: '). \
                                        replace(': \"',': !&~'). \
                                        replace(' {\"',' {!&~'). \
                                        replace('\"}, ','!&~}, '). \
                                        replace('[{\"','[{!&~'). \
                                        replace('\"}]','!&~}]'). \
                                        replace('\"','_'). \
                                        replace('!&~','\"'). \
                                        encode('gbk','ignore'). \
                                        decode('utf-8','ignore'). \
                                        replace('\\','')))


def replace(x):
    return  json.loads(x.replace('\'','\"').replace('False', 'false').replace('True'))
def dict_to_columns(dataframe,column_name):
    df = dataframe.copy()
    df[column_name] = df[column_name].apply(lambda x: ast.literal_eval(x))
    new_dataframe = df[column_name].apply(pd.Series)
    dataframe.drop(columns = column_name)
    return new_dataframe
dict_names = ['device', 'geoNetwork', 'trafficSource']

#Iterate through each of the columns containing dictionary elements and convert them to columns
for name in dict_names:
    dict_df = dict_to_columns(data,name)
    data = pd.concat([data,dict_df],axis = 1)

#Droping dict columns after they have been converted
data = data.drop(columns = dict_names);
data=data.drop(columns = 'adwordsClickInfo')
hits_columns = json_normalize(data['hits'].apply(complex_replace))
hits_columns.head(3)
hits_columns=hits_columns.fillna(0)
cols=['transactionRevenue','transactionTax','transactionShipping','productRevenue']
l =pd.DataFrame()
for i in hits_columns['transactionTax']:
        if i !=0 :
            l=l.append(pd.DataFrame({cols[1]: i[0]}, index=[0]), ignore_index=True)
        else:
            l=l.append(pd.DataFrame({cols[1]: 0}, index=[0]), ignore_index=True)
l2=pd.DataFrame()
for i in hits_columns['transactionRevenue']:
        if i !=0 :
            l2=l2.append(pd.DataFrame({cols[0]: i[0]}, index=[0]), ignore_index=True,sort=True)
        else:
            l2=l2.append(pd.DataFrame({cols[0]: 0}, index=[0]), ignore_index=True,sort=True)

l3=pd.DataFrame()
for i in hits_columns['transactionShipping']:
        if i !=0 :
            l3=l3.append(pd.DataFrame({cols[2]: i[0]}, index=[0]), ignore_index=True,sort=True)
        else:
            l3=l3.append(pd.DataFrame({cols[2]: 0}, index=[0]), ignore_index=True,sort=True)
l4=pd.DataFrame()
for i in hits_columns['productRevenue']:
        if i !=0 :
            l4=l4.append(pd.DataFrame({cols[3]: i[0]}, index=[0]), ignore_index=True)
        else:
            l4=l4.append(pd.DataFrame({cols[3]: 0}, index=[0]), ignore_index=True)
data=data.drop(columns=['hits'])
df=pd.concat([l,l2,l3,l4],axis=1)
for column_name in data.columns:
    if type(data[column_name][1]) == str:
        if 'not available' in data[column_name][1]:
            data = data.drop(columns = column_name)

data.head(3)
data=pd.concat([data,df],axis=1)
data.head()
data.fillna(0, inplace=True)
data.head()
X1 = data.copy()
l=[]
for i in X1['totals_totalTransactionRevenue']:
    if i==0:
        l.append(0)
    else:
        l.append(1)
X1['classification']=l
y=X1['classification']
y2=X1['totals_transactionRevenue']
X=X1.drop(['totals_transactionRevenue','fullVisitorId','visitId','date','visitStartTime',
           'visitNumber','totals_sessionQualityDim','index','totals_bounces','subContinent','continent','metro'
          ,'country','socialEngagementType','city','keyword','value','region','referralPath','networkDomain',
           'campaign','adContent','classification'],axis=1)


X.head()
plt.barh(X['channelGrouping'],y2,color='indigo')
plt.barh(X['operatingSystem'],y2,color=(0.2, 0.4, 0.6, 0.6))
plt.barh(X['browser'],y2,color='lightcoral')
fig,axes= plt.subplots(2, 2)
x=X['transactionRevenue']
axes[0,0].scatter(x,y2)
x=X['transactionShipping']
axes[0,1].scatter(x,y2)
x=X['transactionTax']
axes[1,0].scatter(x,y2)
x=X['productRevenue']
axes[1,1].scatter(x,y2)
X=pd.get_dummies(X,drop_first=True)
X.shape
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=42)<jupyter_output><empty_output><jupyter_text>Classification Tasks<jupyter_code>from sklearn.ensemble import RandomForestClassifier
from sklearn.metrics import mean_squared_error
from sklearn.ensemble import RandomForestRegressor

model = RandomForestClassifier(n_estimators=100)
model.fit(X_train,y_train)
y_pred = model.predict(X)         
X.shape
from sklearn.metrics import accuracy_score
from sklearn.metrics import classification_report


print(classification_report(y,y_pred))
accuracy_score(y,y_pred)
X4=X1.copy()
X4=X1[X1['classification']==0]
pd.Series(y).value_counts()<jupyter_output><empty_output><jupyter_text>Regression Task<jupyter_code>X2=X1.copy()
X2=X1[X1['classification']!=0]
y2=X2['totals_transactionRevenue']
X3=X2.drop(['totals_transactionRevenue','fullVisitorId','visitId','date','visitStartTime',
           'visitNumber','totals_sessionQualityDim','index','totals_bounces','subContinent','continent','metro'
          ,'country','socialEngagementType','city','keyword','value','region','referralPath','networkDomain',
           'campaign','adContent','classification'],axis=1)
X3=pd.get_dummies(X3,drop_first=True)
from sklearn.preprocessing import MinMaxScaler
scaler = MinMaxScaler()
X3 = scaler.fit_transform(X3)



X3.shape
from sklearn.metrics import regression,r2_score,mean_squared_error
from sklearn.ensemble import RandomForestRegressor

X_train, X_test, y_train, y_test = train_test_split(X3, y2, test_size=0.20, random_state=42)

model = RandomForestRegressor(max_depth= 4,n_estimators= 300)
model.fit(X_train,y_train)
y_pred = model.predict(X3)
print(mean_squared_error(y2,y_pred))
print(np.sqrt(mean_squared_error(y2,y_pred)))
print("R2: ",r2_score(y2,y_pred))
id = list(X2.index)
classification = pd.DataFrame(y_pred,columns=['clasifications'])
classification.index = id
X2=X2.merge(classification, right_index=True, left_index=True)
X2
X2.drop('classification',axis = 1,inplace=True)
X2
X2.columns
X4['clasifications'] = np.nan
X4.columns<jupyter_output>D:\UTD\sem 3\python\Anaconda\lib\site-packages\ipykernel_launcher.py:1: SettingWithCopyWarning:


A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: http://pandas.pydata.org/pandas-docs/stable/indexing.html#indexing-view-versus-copy<jupyter_text>Concating two columns<jupyter_code>X5 = pd.concat([X2,X4],axis=0)
X5.columns
X5.fillna(0,inplace=True)
X6 = X5.copy()<jupyter_output><empty_output><jupyter_text>Final Rsquare for the given data<jupyter_code>from sklearn.metrics import regression,r2_score

print(r2_score(X5['totals_transactionRevenue'],X5['clasifications']))<jupyter_output>0.9051276699803114<jupyter_text>Part 2In this part you will create a Python function ``plot_decision_boundary`` that will accepts a machine learning model, feature set ``X``, and target set ``y`` as input and will plot the decision boundary of the input machine learning model  on ``(X,y)``. The following is a sample run of this function: ```Python%matplotlib inline Loading some example datairis = datasets.load_iris()X = iris.data[:, [0, 2]]y = iris.target Training a classifiersvm = SVC(C=0.5, kernel='linear')svm.fit(X, y) Plotting decision boundaryplot_decision_boundary(X, y, clf=svm) Adding axes annotationsplt.xlabel('sepal length [cm]')plt.ylabel('petal length [cm]')plt.title('SVM on Iris')```Sample output: ![decision_boundary.png](attachment:decision_boundary.png)<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>Making the function to plot decision boundary<jupyter_code>def plot_decision_boundary(X, y, model_class, **model_params):
    reduced_data = X[:, :2]                  #reducing the feature parameters to 2 
    model = model_class(**model_params)     #applying the model with given model parameters if any are available
    model.fit(reduced_data, y)              #fitting the target values with reduced data

                                            # Step size of the mesh. Decrease to increase the quality of the VQ.
    h = .02                                 # point in the mesh [x_min, m_max]x[y_min, y_max].    

                                            # Plot the decision boundary. For that, we will assign a color to each
    x_min, x_max = reduced_data[:, 0].min() - 1, reduced_data[:, 0].max() + 1
    y_min, y_max = reduced_data[:, 1].min() - 1, reduced_data[:, 1].max() + 1
    x1, y1 = np.meshgrid(np.arange(x_min, x_max, h), np.arange(y_min, y_max, h))

                                                        # Obtain labels for each point in mesh using the model.
    Z = model.predict(np.c_[x1.ravel(), y1.ravel()])    #returning the contiguous flattened array by using np.ravel() function

    x_min, x_max = X[:, 0].min() - 1, X[:, 0].max() + 1
    y_min, y_max = X[:, 1].min() - 1, X[:, 1].max() + 1
    x1, y1 = np.meshgrid(np.arange(x_min, x_max, 0.1),
                         np.arange(y_min, y_max, 0.1))

    Z = model.predict(np.c_[x1.ravel(), y1.ravel()]).reshape(x1.shape)

    plt.contourf(x1, y1, Z, alpha=0.5)
    plt.scatter(X[:, 0], X[:, 1], c=y, alpha=1,edgecolors='black')
    return plt<jupyter_output><empty_output><jupyter_text>for this function we only have to provide model class name with Values of X and y; it automatically reduces the values of features to 2 and plots the decision boundaries<jupyter_code>from sklearn.datasets import load_breast_cancer
from sklearn.model_selection import train_test_split
from sklearn.tree import DecisionTreeClassifier 
from sklearn.linear_model import LogisticRegression
from sklearn.svm import SVC
cancer=load_breast_cancer()
X_train, X_test, y_train, y_test = train_test_split(cancer.data, cancer.target, random_state=0)

dec = DecisionTreeClassifier()
dec.fit(X_train, y_train)
%matplotlib inline

plot_decision_boundary(X_train, y_train,SVC,C=2)

%matplotlib inline

plot_decision_boundary(X_train, y_train,RandomForestClassifier,n_estimators=10)
%matplotlib inline
plot_decision_boundary(X_train, y_train,LogisticRegression)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13523027280727487, 'fraction_numerical': 0.01811381636843649, 'mean_word_length': 3.6910904712762296, 'pattern_counts': {'":': 1, '<': 31, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 25, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '166255', 'license': 'MIT', 'n_tokens_mistral': 4580, 'n_tokens_neox': 4408, 'n_words': 1017, 'path': 'Take home exam.ipynb', 'repo_name': 'Rshah96/Machine-learning-', 'size': '13636'}