<jupyter_start><jupyter_text>Preprocessing<jupyter_code># open the dataset
df = pd.read_csv('data/train.csv')
df.head(3)
df.info()
df.describe().T
# data types transformation
# Converting Date type from object to datetime
df['Date'] = pd.to_datetime(df['Date'], format='%d-%b-%y')<jupyter_output><empty_output><jupyter_text>unbalenced data<jupyter_code>df['Buy'].value_counts()<jupyter_output><empty_output><jupyter_text>New features<jupyter_code># type of OS: mobile or computer
df['OS'].unique()
# 0 - PC/ 1 - Mobile
dict_os = {'MacOSX': 0, 'Windows': 0, 'Android': 1, 'Ubuntu': 0, 'iOS': 1, 'Other': 0, 'Fedora': 0, 'Chrome OS':0}
df['OS_type'] = df['OS'].map(dict_os)
# Type of Visitor
df['Type_of_Visitor'].value_counts()
# 0 - Returner/ 1 - New_Access/ 2 - Other
dict_visitor = {'Returner': 0, 'New_Access': 1, 'Other': 2}

df['Type_of_Visitor_type'] = df['Type_of_Visitor'].map(dict_visitor)<jupyter_output><empty_output><jupyter_text>Remove Outliers<jupyter_code># trashold for outliers
filters1 = (
    (df['AccountMng_Duration']<=2000)
    &
    (df['FAQ_Duration']<=1500)
    &
    (df['Product_Pages']<=500)
    &
    (df['Product_Duration']<=25000)
    &
    (df['GoogleAnalytics_PageValue']<=300)
)

df_1 = df[filters1]

print('Percentage of data kept after removing outliers:', np.round(df_1.shape[0] / df.shape[0], 4))

df = df[filters1]
df.describe().T<jupyter_output><empty_output><jupyter_text>Correlation<jupyter_code>df.columns
# Define metric and non-metric features
non_metric_features = ['Access_ID', 'Date', 'OS', 'Browser', 'Country','Type_of_Traffic', 'Type_of_Visitor',
                       'Buy', 'OS_type', 'Type_of_Visitor_type']
metric_features = df.columns.drop(non_metric_features).to_list()
def cor_heatmap(cor, ax):
    plt.figure(figsize=(12,10))
    sns.heatmap(data = cor, annot = True, cmap = plt.cm.Reds, fmt='.1', ax=ax)
    plt.show()
#build correlation matrix
cor_spearman = df[metric_features].corr(method ='spearman')
# cor_spearman

cor_pearson = df[metric_features].corr(method='pearson')

fig, ax1 = plt.subplots(1,1, figsize=(8,6))
cor_heatmap(cor_spearman, ax1)
fig, ax1 = plt.subplots(1,1, figsize=(8,6))
cor_heatmap(cor_pearson, ax1)
# drop 'AccountMng_Pages' 'FAQ Duration' 'Product Duration' 'GoogleAnalytics_BounceRate'
df.drop(['AccountMng_Pages','FAQ_Duration','Product_Duration', 'GoogleAnalytics_BounceRate'], axis=1, inplace=True)<jupyter_output><empty_output><jupyter_text>Dataset for DecisionTree Reg1<jupyter_code>df.columns
df_log1 = df[['AccountMng_Duration', 'FAQ_Pages', 'Product_Pages', 'GoogleAnalytics_ExitRate', 'GoogleAnalytics_PageValue',
              'Type_of_Traffic', 'Buy', 'OS_type', 'Type_of_Visitor_type']]
X = df_log1.drop('Buy', axis=1)
y = df_log1['Buy']
X<jupyter_output><empty_output><jupyter_text>Balance the y - test Oversampling<jupyter_code># Before Oversampling
print('X:', X.shape)
print('y:', y.shape)
y.value_counts()
ros = RandomOverSampler(random_state=0)
X_over, y_over = ros.fit_resample(X, y)
# After Oversampling
print('X_over:', X_over.shape)
print('y_over:', y_over.shape)
y_over.value_counts()<jupyter_output>X_over: (16858, 8)
y_over: (16858,)<jupyter_text>Scale - MaxMin<jupyter_code># split the dataset
X_train, X_val, y_train, y_val = train_test_split(X_over, y_over, test_size=0.2, random_state=5, stratify=y_over)
def calculate_F1(interval, parameter, max_depth = None):
    skf = StratifiedKFold(n_splits=10)
    train_results = []
    avg_train_results = []
    test_results = []
    avg_test_results = []
    
    for value in interval:
        for train_index, test_index in skf.split(X_over,y_over):
            X_train, X_test = X_over.iloc[train_index], X_over.iloc[test_index]
            y_train, y_test = y_over.iloc[train_index], y_over.iloc[test_index]
            dt = DecisionTreeClassifier(max_depth = value)
            dt.fit(X_train, y_train)
            train_pred = dt.predict(X_train)
            f1_train = f1_score(y_train, train_pred)
            train_results.append(f1_train)
            val_pred = dt.predict(X_test)
            f1_val = f1_score(y_test, val_pred)
            test_results.append(f1_val)
        avg_train_results.append(round(np.mean(train_results),3))
        avg_test_results.append(round(np.mean(test_results),3))
    
    print('The best f1 score for train is ' + str(max(avg_train_results)) + ' for the depth of ' + str(avg_train_results.index(max(avg_train_results))+1))
    print('The best f1 score for test is ' + str(max(avg_test_results)) + ' for the depth of ' + str(avg_test_results.index(max(avg_test_results))+1))
        
    
    plt.plot(interval, avg_train_results, label = "Train F1")
    plt.plot(interval, avg_test_results, label = "Test F1")
    plt.xticks(interval)
    plt.legend()
    plt.ylabel("F1 score")
    plt.xlabel(str(parameter))
    plt.show()
calculate_F1(np.arange(1,20), 'depth')
model_dt = DecisionTreeClassifier(max_depth = 9).fit(X_train, y_train)

y_pred_train = model_dt.predict(X_train)
y_pred_val = model_dt.predict(X_val)
# evaluation:
print('confusion matrix')
print(confusion_matrix(y_val, y_pred_val))

print('accuracy:', accuracy_score(y_val, y_pred_val))
print('precision:', precision_score(y_val, y_pred_val))
print('recall:', recall_score(y_val, y_pred_val))
print('f1_train:', f1_score(y_train, y_pred_train))
print('f1_val:', f1_score(y_val, y_pred_val))<jupyter_output>confusion matrix
[[1474  212]
 [ 247 1439]]
accuracy: 0.8638790035587188
precision: 0.8715929739551787
recall: 0.8534994068801898
f1_train: 0.8804274211754082
f1_val: 0.8624513035660774<jupyter_text>Feature importance DT<jupyter_code>gini_importance = DecisionTreeClassifier().fit(X_train, y_train).feature_importances_
entropy_importance = DecisionTreeClassifier(criterion='entropy').fit(X_train, y_train).feature_importances_
zippy = pd.DataFrame(zip(gini_importance, entropy_importance), columns = ['gini','entropy'])
zippy['col'] = X_train.columns
tidy = zippy.melt(id_vars='col').rename(columns=str.title)
tidy.sort_values(['Value'], ascending = False, inplace = True)

plt.figure(figsize=(15,8))
sns.barplot(y='Col', x='Value', hue='Variable', data=tidy)<jupyter_output><empty_output><jupyter_text>Dataset for test<jupyter_code>test_set = pd.read_csv('data/test.csv')
# test_set.set_index('Access_ID', inplace = True)
test_set.info()
test_set.head()
test_set.describe().T
test_set['Type_of_Visitor'].unique()
test_set['OS'].unique()
# select the columns for the model

# 0 - PC/ 1 - Mobile
dict_os = {'MacOSX': 0, 'Windows': 0, 'Android': 1, 'Ubuntu': 0, 'iOS': 1, 'Other': 0, 'Fedora': 0, 'Chrome OS':0}
test_set['OS_type'] = test_set['OS'].map(dict_os)

# 0 - Returner/ 1 - New_Access/ 2 - Other
dict_visitor = {'Returner': 0, 'New_Access': 1, 'Other': 2}
test_set['Type_of_Visitor_type'] = test_set['Type_of_Visitor'].map(dict_visitor)

test = test_set[['AccountMng_Duration', 'FAQ_Pages', 'Product_Pages', 'GoogleAnalytics_ExitRate',
                 'GoogleAnalytics_PageValue', 'Type_of_Traffic', 'OS_type', 'Type_of_Visitor_type']]
test.columns
y_pred_test = model_dt.predict(test)
y_pred_test.shape
df_submission = pd.concat([test_set['Access_ID'], pd.DataFrame(columns=['Buy'], data=y_pred_test)], axis=1)
df_submission['Buy'].value_counts()
df_submission.to_csv('dt_9maxdepth_over_noscale.csv', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1525562372188139, 'fraction_numerical': 0.031492842535787324, 'mean_word_length': 6.053846153846154, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 39, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '572287', 'license': 'Unlicense', 'n_tokens_mistral': 2977, 'n_tokens_neox': 2798, 'n_words': 456, 'path': 'Ana/ML_decisiontree_over_noscale.ipynb', 'repo_name': 'iryna-savchuk/group_work_ml', 'size': '7335'}