<jupyter_start><jupyter_text>Inspirational Notebooks Generating new festures according to these notebooks * https://www.kaggle.com/nuhsikander/lgbm-new-features-corrected* https://www.kaggle.com/rteja1113/lightgbm-with-count-features* https://www.kaggle.com/aharless/swetha-s-xgboost-revised* https://www.kaggle.com/bk0000/non-blending-lightgbm-model-lb-0-977  Load Data<jupyter_code>%matplotlib inline 
import matplotlib.pyplot as plt 

import math
import numpy as np
import pandas as pd
import seaborn as sns
from pandas import read_csv
import sklearn
from sklearn import preprocessing
from sklearn.preprocessing import MinMaxScaler
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LogisticRegression
from sklearn.cross_validation import train_test_split

import warnings
warnings.filterwarnings('ignore')

plt.rc("font", size=14)
sns.set(style="white")
sns.set(style="whitegrid", color_codes=True)
import keras
from keras.datasets import mnist
from keras.models import Sequential
from keras.layers import Dense
from keras.optimizers import SGD
from keras.optimizers import RMSprop
from keras.optimizers import Adagrad
from keras.optimizers import Adadelta
# df_train= pd.read_csv('data/train.csv',nrows=200000, parse_dates=['click_time'])
df_train = pd.read_csv('data/train_new_cols.csv',nrows=200000) #train data subset, original too large
df_train.dropna()
df_train.info()
df_train.columns.values
cols = ['ip', 'app', 'device', 'os', 'channel', 'click_time',
       'attributed_time', 'is_attributed', 'day', 'hour', 'minute',
       'second', 'ip_confRate', 'app_confRate', 'device_confRate',
       'os_confRate', 'channel_confRate', 'app_channel_confRate',
       'app_os_confRate', 'app_device_confRate', 'channel_os_confRate',
       'channel_device_confRate', 'os_device_confRate',
       'ip_app_channel_var_day', 'ip_app_os_var_hour',
       'ip_day_channel_var_hour_x', 'ip_day_hour_count_channel',
       'ip_app_count_channel', 'ip_app_os_count_channel',
       'ip_app_day_hour_count_channel', 'ip_app_channel_mean_hour',
       'app_AvgViewPerDistinct_ip', 'app_count_channel',
       'channel_count_app', 'ip_nunique_channel', 'ip_nunique_app',
       'ip_day_nunique_hour', 'ip_app_nunique_os', 'ip_nunique_device',
       'app_nunique_channel', 'ip_device_os_nunique_app',
       'ip_device_os_cumcount_app', 'ip_cumcount_app', 'ip_cumcount_os',
       'ip_day_channel_var_hour_y', 'ip_nextClick', 'ip_app_nextClick',
       'ip_channel_nextClick', 'ip_os_nextClick',
       'ip_app_device_os_channel_nextClick', 'ip_os_device_nextClick',
       'ip_os_device_app_nextClick', 'prev_identical_clicks',
       'future_identical_clicks', 'prev_app_clicks', 'future_app_clicks']
df_train = df_train.loc[:,cols]
df_train.head(10)
df_train['is_attributed'].value_counts()
sns.countplot(x='is_attributed', data=df_train, palette='hls')
plt.show()
# The ratio of df_train to df_test is 0.8 to 0.2 or 0.75 to 0.25
df_test = pd.read_csv('data/train_new_cols.csv', nrows=50000,skiprows=range(1, 400000))
df_test.dropna()
df_test.info()
df_test = df_test.loc[:,cols]
df_test.head(10)
df_test['is_attributed'].value_counts()
sns.countplot(x='is_attributed', data=df_test, palette='hls')
plt.show()
# Get columes names except the click_time (object), attributed_time (object) and is_attributed
train_cols = []
for each_value in df_train.columns.values:
    if each_value == 'click_time' or each_value == 'attributed_time' or each_value == 'is_attributed':
        continue
    train_cols.append(each_value)
train_cols
# Data_X
X_train = df_train.loc[:,train_cols]
X_test = df_test.loc[:,train_cols]
X_train.shape
X_test.shape
# Data_Y
y_train = df_train[['is_attributed']]
y_test = df_test[['is_attributed']]
y_train.shape
y_test.shape
y_test_val = y_test.values<jupyter_output><empty_output><jupyter_text>Algorithms  1. Random Forest<jupyter_code>from sklearn.ensemble import RandomForestClassifier
def randomForest_pre(X_train, y_train, X_test, y_test):
    X_train = np.nan_to_num(X_train)
    y_train = np.nan_to_num(y_train)
    X_test = np.nan_to_num(X_test)
    y_test = np.nan_to_num(y_test)
    RF_model = RandomForestClassifier()
    # Train the model
    RF_fit = RF_model.fit(X_train, y_train)
    # Get the prediction of the test data
    RF_predict = RF_model.predict(X_test)
    # Compare the prediction with the known values
    RF_acc = sklearn.metrics.accuracy_score(np.array(RF_predict)[:], 
                                         np.array(y_test_val)[:])
    # Plot the data
    plt.figure(figsize=(10,5))
    plt.plot(RF_predict, color='red', label='Prediction')
    plt.plot(y_test, label='Y_test')
    plt.legend(['Prediction', 'Y_test'])
    _ = plt.ylim()
    
    return RF_fit, RF_predict, RF_acc, RF_model
# RF_fit, RF_pre, RF_acc, rf_model = randomForest_pre(X_train, y_train, X_test, y_test)
# RF_acc<jupyter_output><empty_output><jupyter_text>2. Gradient Boosting & AdaBoost<jupyter_code>from sklearn.ensemble import GradientBoostingClassifier
def gradientBoosting_pre(X_train, y_train, X_test, y_test):
    X_train = np.nan_to_num(X_train)
    y_train = np.nan_to_num(y_train)
    X_test = np.nan_to_num(X_test)
    y_test = np.nan_to_num(y_test)
    GB_model = GradientBoostingClassifier(n_estimators=100, learning_rate=1.0, max_depth=1, random_state=0)
    # Train the model
    GB_fit = GB_model.fit(X_train, y_train)
    # Get the prediction of the test data
    GB_predict = GB_model.predict(X_test)
    # Compare the prediction with the known values
    GB_acc = sklearn.metrics.accuracy_score(np.array(GB_predict)[:], 
                                         np.array(y_test_val)[:])
    
    # Plot the data
    plt.figure(figsize=(10,5))
    plt.plot(GB_predict, color='red', label='Prediction')
    plt.plot(y_test, label='Y_test')
    plt.legend(['Prediction', 'Y_test'])
    _ = plt.ylim()
    
    return GB_fit, GB_predict, GB_acc, GB_model
# GB_fit, GB_predict, GB_acc, gb_model = gradientBoosting_pre(X_train, y_train, X_test, y_test)
# GB_acc<jupyter_output><empty_output><jupyter_text>3. Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression
def logisticRegression_pre(X_train, y_train, X_test, y_test):
    X_train = np.nan_to_num(X_train)
    y_train = np.nan_to_num(y_train)
    X_test = np.nan_to_num(X_test)
    y_test = np.nan_to_num(y_test)
    LG_model = LogisticRegression()
    # Train the model
    LG_fit = LG_model.fit(X_train, y_train)
    # Get the prediction of the test data
    LG_predict = LG_model.predict(X_test)
    # Compare the prediction with the known values
    LG_acc = sklearn.metrics.accuracy_score(np.array(LG_predict)[:], 
                                         np.array(y_test_val)[:])
    
    return LG_fit, LG_predict, LG_acc, LG_model
# LG_fit, LG_predict, LG_acc, lg_model = logisticRegression_pre(X_train, y_train, X_test, y_test)
# LG_acc<jupyter_output><empty_output><jupyter_text>4. SVM - Support Vector Machine<jupyter_code>from sklearn import svm
def svm_pre(X_train, y_train, X_test, y_test):
    X_train = np.nan_to_num(X_train)
    y_train = np.nan_to_num(y_train)
    X_test = np.nan_to_num(X_test)
    y_test = np.nan_to_num(y_test)
    svm_model = svm.SVC()
    uniq = np.unique(y_train[9000:10000])
    # Train the model
    SVM_fit = svm_model.fit(X_train[9000:10000], y_train[9000:10000])
    # Get the prediction of the test data
    SVM_predict = svm_model.predict(X_test)
    # Compare the prediction with the known values
    SVM_acc = sklearn.metrics.accuracy_score(np.array(LG_predict)[:], 
                                     np.array(y_test_val)[:])
    return SVM_fit, SVM_predict, SVM_acc, svm_model
# svm_fit, svm_predict, svm_acc, svm_model = svm_pre(X_train, y_train, X_test, y_test)
# svm_acc<jupyter_output><empty_output><jupyter_text>5. ANN - Artificial Neural Network<jupyter_code>def shallow_net_A(n=55,i=len(train_cols),o=2):
    # Create simple one dense layer net
    # Default 55 neurons, input 5, output 2
    net = Sequential()
    net.add(Dense(n, activation='sigmoid', input_shape=(i,)))
    net.add(Dense(2, activation='softmax'))
    # Compile net
    net.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])
    return net
def ann_pre(X_train, y_train, X_test, y_test):
    ann_model = shallow_net_A()
    ann_summary = ann_model.summary()
    # Convert the values
    X_train_ann = np.nan_to_num(X_train)
    y_train_ann = np.nan_to_num(y_train)
    X_test_ann = np.nan_to_num(X_test)
    y_test_ann = np.nan_to_num(y_test)
    # Conver the matrix, finally we have two classes (n_classes), the original one has oly one class
    n_classes = 2
    y_train_ann = keras.utils.to_categorical(y_train_ann, n_classes)
    y_test_ann = keras.utils.to_categorical(y_test_ann, n_classes)
    # Training the model
    ann_fit = ann_model.fit(X_train_ann, y_train_ann, batch_size=128, epochs=99, verbose=1, validation_data=(X_test_ann, y_test_ann))
    
    # Evaluate: loss & accuracy -> Using Evaluation to get the accracy
    ann_evaluate = ann_model.evaluate(X_test_ann, y_test_ann)
    
    # Using prediction
    ann_pre = ann_model.predict(X_test)
    # Convert value to boolean value
    y_pre = (ann_pre > 0.5)
    # Counting the boolean value, counting the accuracy by using basic calculation
    from sklearn.metrics import confusion_matrix
    ann_output = confusion_matrix(y_test_ann.argmax(axis=1), y_pre.argmax(axis=1))
    ann_prediction_acc = ann_output[0][0]/(ann_output[0][0]+ann_output[1][0])
    
    return ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model
# ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model = ann_pre(X_train, y_train, X_test, y_test)
# ann_prediction_acc<jupyter_output><empty_output><jupyter_text>6. MLP - Multi-layered Neural Network<jupyter_code>def shallow_net_C(n=55,i=len(train_cols),o=2):
    # Create simple one dense layer net
    # Default 55 neurons, input 5, output 2, here we have more hidden layers with different activation
    net = Sequential()
    net.add(Dense(n, activation='sigmoid', input_shape=(i,)))
    net.add(Dense(n, activation='relu', input_shape=(i,)))
    net.add(Dense(n, activation='tanh', input_shape=(i,)))
    net.add(Dense(n, activation='elu', input_shape=(i,)))
    net.add(Dense(2, activation='softmax'))
    # Compile net
    net.compile(loss='mean_squared_error', optimizer=SGD(lr=0.01), metrics=['accuracy'])
    return net
def mlp_pre(X_train, y_train, X_test, y_test):
    mlp_model = shallow_net_C()
    mlp_summary = mlp_model.summary()
    # Convert the values
    X_train_mlp = np.nan_to_num(X_train)
    y_train_mlp = np.nan_to_num(y_train)
    X_test_mlp = np.nan_to_num(X_test)
    y_test_mlp = np.nan_to_num(y_test)
    # Conver the matrix, finally we have two classes (n_classes)
    n_classes = 2
    y_train_mlp = keras.utils.to_categorical(y_train_mlp, n_classes)
    y_test_mlp = keras.utils.to_categorical(y_test_mlp, n_classes)
    # Training the model
    mlp_fit = mlp_model.fit(X_train_mlp, y_train_mlp, batch_size=128, epochs=99, verbose=1, validation_data=(X_test_mlp, y_test_mlp))
    
    # Evaluate: loss & accuracy -> Using Evaluation to get the accracy
    mlp_evaluate = mlp_model.evaluate(X_test_mlp, y_test_mlp)
    
    # Using prediction
    mlp_pre = mlp_model.predict(X_test)
    # Convert value to boolean value
    y_pre = (mlp_pre > 0.5)
    # Counting the boolean value, counting the accuracy by using basic calculation
    from sklearn.metrics import confusion_matrix
    mlp_output = confusion_matrix(y_test_mlp.argmax(axis=1), y_pre.argmax(axis=1))
    mlp_prediction_acc = mlp_output[0][0]/(mlp_output[0][0]+mlp_output[1][0])
    
    return mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model
# mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model = mlp_pre(X_train, y_train, X_test, y_test)
# mlp_prediction_acc<jupyter_output><empty_output><jupyter_text>7. RNN - Recurrent Neural Network (LSTM)<jupyter_code>from keras.layers import LSTM
def rnn_pre(df_train, train_rate=0.75):
    # Set the dataset for train and test
    df_rnn_train = df_train.loc[:,train_cols + ['is_attributed']]
    df_rnn_test = df_test.loc[:,train_cols + ['is_attributed']]
    df_rnn = df_rnn_train.append(df_rnn_test)
    
    pre_col_index = list(df_rnn_train).index('is_attributed')
    dataset = df_rnn.values.astype('float32')
    dataset = np.nan_to_num(dataset)
    
    # Normalize the dataset, set all the data of the dataset to be in the range between 0 and 1
    scaler = MinMaxScaler(feature_range=(0, 1))
    dataset = scaler.fit_transform(dataset)
    
    # Split into train and test sets
    train_size = int(len(dataset) * train_rate)
    test_size = len(dataset) - train_size
    train, test = dataset[0:train_size,:], dataset[train_size:len(dataset),:]
    
    # Use this function to prepare the train and test datasets for modeling
    look_back = 1
    trainY = train[:, pre_col_index]
    trainX = np.delete(train, pre_col_index, axis = 1) 
    testY = test[:, pre_col_index]
    testX = np.delete(test, pre_col_index, axis = 1) 
    
    # Reshape input to be [samples, time steps, features], here it changes the dimension from 2D to 3D
    trainX = np.reshape(trainX, (trainX.shape[0], 1, trainX.shape[1]))
    testX = np.reshape(testX, (testX.shape[0], 1, testX.shape[1]))
    
    # Create and fit the LSTM network
    RNN_model = Sequential()
    RNN_model.add(LSTM(5, input_shape=(1, len(trainX[0][0]))))
    RNN_model.add(Dense(1))
    RNN_model.compile(loss='mean_squared_error', optimizer='adam')
    RNN_model.fit(trainX, trainY, epochs=10, batch_size=128, verbose=2)
    
    # Make predictions, trainPredict should be 1D array
    trainPredict = RNN_model.predict(trainX)
    testPredict = RNN_model.predict(testX)
    
    # Change the dimension from 3D to 2D
    trainX_2D = trainX.transpose([1,0,2]).reshape(len(trainX),len(trainX[0][0]))
    testX_2D = testX.transpose([1,0,2]).reshape(len(testX),len(testX[0][0]))
    
    # Append prediction back to the model
    trainPredict_6cols = np.append(trainX_2D, trainPredict, 1)
    testPredict_6cols = np.append(testX_2D, testPredict, 1)

    # Invert predictions back to normal values
    trainPredict_6cols = scaler.inverse_transform(trainPredict_6cols)
    testPredict_6cols = scaler.inverse_transform(testPredict_6cols)
    
    # Calculating the RMSE
    trainScore = math.sqrt(mean_squared_error(trainY, trainPredict_6cols[:, pre_col_index]))
    print('Train Score: %.2f RMSE' % (trainScore))
    testScore = math.sqrt(mean_squared_error(testY, testPredict_6cols[:, pre_col_index]))
    print('Test Score: %.2f RMSE' % (testScore))
    
    final_prediction_train = np.where(trainPredict_6cols[:, pre_col_index] > 0, 1, 0)
    final_prediction_test = np.where(testPredict_6cols[:, pre_col_index] > 0, 1, 0)
    
    # Change dimension from 2D to 1D
    final_prediction_train = np.reshape(final_prediction_train, (-1, 1))
    final_prediction_test = np.reshape(final_prediction_test, (-1, 1))
    
    # Counting the accuracy by using basic calculation
    rnn_acc_train = sklearn.metrics.accuracy_score(np.array(final_prediction_train)[:], 
                                     np.array(trainY)[:])
    rnn_acc_test = sklearn.metrics.accuracy_score(np.array(final_prediction_test)[:], 
                                     np.array(testY)[:])
    return rnn_acc_train
# rnn_acc = rnn_pre(df_train)
# rnn_acc<jupyter_output><empty_output><jupyter_text>Train & Test Data (Call the function)  1. Random Forest<jupyter_code>RF_fit, RF_pre, RF_acc, rf_model = randomForest_pre(X_train, y_train, X_test, y_test)
print('Random Forest accuracy: {}%'.format(RF_acc * 100))<jupyter_output>Random Forest accuracy: 99.902%<jupyter_text>2. Gradient Boosting<jupyter_code>GB_fit, GB_predict, GB_acc, gb_model = gradientBoosting_pre(X_train, y_train, X_test, y_test)
print('Gradient Boosting accuracy: {}%'.format(GB_acc * 100))<jupyter_output>Gradient Boosting accuracy: 99.634%<jupyter_text>3. Logistic Regression<jupyter_code>LG_fit, LG_predict, LG_acc, lg_model = logisticRegression_pre(X_train, y_train, X_test, y_test)
print('Logistic Regression accuracy: {}%'.format(LG_acc * 100))<jupyter_output>Logistic Regression accuracy: 99.824%<jupyter_text>4. SVM - Support Vector Machine<jupyter_code>svm_fit, svm_predict, svm_acc, svm_model = svm_pre(X_train, y_train, X_test, y_test)
print('SVM accuracy: {}%'.format(svm_acc * 100))<jupyter_output>SVM accuracy: 99.824%<jupyter_text>5. ANN - Artificial Neural Network<jupyter_code>ann_summary, ann_fit, ann_evaluate, ann_prediction_acc, ann_model = ann_pre(X_train, y_train, X_test, y_test)
print('ANN accuracy: {}%'.format(ann_prediction_acc * 100))<jupyter_output>ANN accuracy: 99.824%<jupyter_text>6. MLP - Multi-layered Neural Network<jupyter_code>mlp_summary, mlp_fit, mlp_evaluate, mlp_prediction_acc, mlp_model = mlp_pre(X_train, y_train, X_test, y_test)
print('MLP accuracy: {}%'.format(mlp_prediction_acc * 100))<jupyter_output>MLP accuracy: 99.824%<jupyter_text>7. RNN - Recurrent Neural Network<jupyter_code>rnn_acc = rnn_pre(df_train)
print('RNN accuracy: {}%'.format(rnn_acc * 100))<jupyter_output>Epoch 1/10
 - 6s - loss: 0.0014
Epoch 2/10
 - 5s - loss: 0.0012
Epoch 3/10
 - 5s - loss: 0.0012
Epoch 4/10
 - 5s - loss: 0.0012
Epoch 5/10
 - 5s - loss: 0.0012
Epoch 6/10
 - 5s - loss: 0.0012
Epoch 7/10
 - 5s - loss: 0.0012
Epoch 8/10
 - 5s - loss: 0.0012
Epoch 9/10
 - 5s - loss: 0.0012
Epoch 10/10
 - 4s - loss: 0.0012
Train Score: 0.03 RMSE
Test Score: 0.03 RMSE
RNN accuracy: 92.27680000000001%<jupyter_text>Conclusion<jupyter_code>print('Random Forest accuracy: {}%'.format(RF_acc * 100))
print('Gradient Boosting accuracy: {}%'.format(GB_acc * 100))
print('Logistic Regression accuracy: {}%'.format(LG_acc * 100))
print('SVM accuracy: {}%'.format(svm_acc * 100))
print('ANN accuracy: {}%'.format(ann_prediction_acc * 100))
print('MLP accuracy: {}%'.format(mlp_prediction_acc * 100))
print('RNN accuracy: {}%'.format(rnn_acc * 100))<jupyter_output>Random Forest accuracy: 99.92%
Gradient Boosting accuracy: 99.634%
Logistic Regression accuracy: 99.824%
SVM accuracy: 99.824%
ANN accuracy: 99.824%
MLP accuracy: 99.824%
RNN accuracy: 92.27680000000001%<jupyter_text>Random Forest is the best one for our project  Prediction for test.csv<jupyter_code># Columns for our current analsis
train_cols
# Read the test data
df = pd.read_csv('data/test_small_all_features.csv')[train_cols].astype('float64')
df = np.nan_to_num(df)
# Read the output of the test data
sample_out = pd.read_csv('data/sample_submission.csv', nrows=1000000)[['is_attributed']].astype('float64')
df_predict = rf_model.predict(df)
# Compare the prediction with the known values
df_acc = sklearn.metrics.accuracy_score(np.array(df_predict)[:], 
                                         np.array(sample_out)[:])
print('By using the best algorittm, the accuracy of the prediction: {}%'.format(df_acc * 100))<jupyter_output>By using the best algorittm, the accuracy of the prediction: 100.0%
{'fraction_non_alphanumeric': 0.11562843820401321, 'fraction_numerical': 0.025409964897574265, 'mean_word_length': 4.765025672002416, 'pattern_counts': {'":': 0, '<': 60, '<?xml version=': 0, '>': 66, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '17', 'dir': 'jupyter-structured-clean-dedup', 'id': '7808', 'license': 'MIT', 'n_tokens_mistral': 7522, 'n_tokens_neox': 7088, 'n_words': 1383, 'path': 'Prediction_all_features.ipynb', 'repo_name': 'Yidi0221/TalkingData-Ad-Fraud-Detection', 'size': '19087'}