<jupyter_start><jupyter_text><jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# Python ≥3.5 is required
import sys
assert sys.version_info >= (3, 5)

# Scikit-Learn ≥0.20 is required
import sklearn
assert sklearn.__version__ >= "0.20"

# Common imports
import numpy as np
import os

# to make this notebook's output stable across runs
np.random.seed(42)

# To plot pretty figures
%matplotlib inline
import matplotlib as mpl
import matplotlib.pyplot as plt
mpl.rc('axes', labelsize=14)
mpl.rc('xtick', labelsize=12)
mpl.rc('ytick', labelsize=12)

# Where to save the figures
PROJECT_ROOT_DIR = "./midtermproject"
CHAPTER_ID = "GoogleStore"
IMAGES_PATH = os.path.join(PROJECT_ROOT_DIR, "images", CHAPTER_ID)
os.makedirs(IMAGES_PATH, exist_ok=True)

def save_fig(fig_id, tight_layout=True, fig_extension="png", resolution=300):
    path = os.path.join(IMAGES_PATH, fig_id + "." + fig_extension)
    print("Saving figure", fig_id)
    if tight_layout:
        plt.tight_layout()
    plt.savefig(path, format=fig_extension, dpi=resolution)

# Ignore useless warnings (see SciPy issue #5998)
import warnings
warnings.filterwarnings(action="ignore", message="^internal gelsd")
# List of libraries

import numpy as np # linear algebra
import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)
import json
from pandas.io.json import json_normalize
import seaborn as sns
import matplotlib.pyplot as plt
from plotly.offline import init_notebook_mode, iplot
import plotly.graph_objs as go
from plotly import tools
import plotly.figure_factory as ff
import numpy as np
init_notebook_mode(connected = True)
import datetime
import missingno as msno
# Nitesh's truncated file
import os
os.environ['KAGGLE_USERNAME'] = "hl533njit"
os.environ['KAGGLE_KEY'] = "<KEY>"
!kaggle datasets download -d niteshmistry/minigacustomerrevenueprediction 
# api copied from kaggle
# extract zipfile
import zipfile
zip = zipfile.ZipFile('minigacustomerrevenueprediction.zip')
zip.extractall()
#read first five rows of training dataset
import pandas as pd
df_traingoogle = pd.read_csv('mini-train.csv')
df_traingoogle.head()
#return information about the dataframe including the data types of each column and memory usage
# of the entire data
df_traingoogle.info()
#read first five rows of test dataset
import pandas as pd
df_testgoogle = pd.read_csv('mini-test.csv')
df_testgoogle.head()
df_testgoogle.info()
# Descriptive statistics that summarize the central tendency, dispersion and shape of a dataset’s distribution, excluding NaN values
df_traingoogle.describe()
# importing pandas as pd 
import pandas as pd 
  
# Creating the dataframe 
df = pd.DataFrame({'Date':['05/11/2018'], 
                'fullVisitorId':['7460955084541987166'], 
                'visitId':[1526099341]}) 
  
# Print the dataframe 
print(df) 
  
# Now we will check the data type  
# of the 'Date' column 
df.info() 
# convert the 'Date' column to datetime format 
df['Date']= pd.to_datetime(df['Date']) 
  
# Check the format of 'Date' column 
df.info()
# Common imports
# Sum up the transaction revenue at user level and then do a scatter plot
import numpy as np

df_traingoogle["totals.transactionRevenue"] = df_traingoogle["totals.transactionRevenue"].astype('float')
gdf = df_traingoogle.groupby("fullVisitorId")["totals.transactionRevenue"].sum().reset_index()

plt.figure(figsize=(8,6))
plt.scatter(range(gdf.shape[0]), np.sort(np.log1p(gdf["totals.transactionRevenue"].values)))
plt.xlabel('index', fontsize=12)
plt.ylabel('TransactionRevenue', fontsize=12)
plt.show()
# Utility functions for plots
def plotlybar(labels1 = None, values1 = None, name1 = None,labels2 = None, values2 = None, name2 =None, markercolor1 = '#9ECAE1', markercolor2 = '#ff99ff', title= None, mode = 'group', orientation = 'v'):
    trace1 = go.Bar(
    x = labels1,
    y = values1,
    orientation = orientation, 
    text = values1,
    name = name1,
    textposition = 'auto',
    marker=dict(
        color=markercolor1#'rgb(58,200,225)',
        )
    )    
    
    if labels2 is not None:
        trace2 = go.Bar(
            x = labels2,
            y = values2,
            name = name2,
            text = values2,
            textposition = 'auto',
            marker=dict(
                color=markercolor2#'rgb(58,200,225)',
            )
        )
        data = [trace1, trace2]
    
    else:
        data = [trace1]
    layout = go.Layout(title = title,xaxis=dict(tickangle=-25),
    barmode=mode)
    fig = go.Figure(data = data, layout = layout)
    iplot(fig)
    
def scatter_plot(x,y, title):
    data = [go.Scatter(
    x = x,
    y = y,
    mode = 'lines+markers',
    name = 'lines+markers'
    )]
    layout = go.Layout(title = title,xaxis=dict(tickangle=-25))
    fig = go.Figure(data = data, layout = layout)
    iplot(fig)
# Data Preparation - Checking for missing data
total = df_traingoogle.isnull().sum().sort_values(ascending = True)
percent = (df_traingoogle.isnull().sum() / df_traingoogle.isnull().count() * 100).sort_values(ascending = True)
missing_data = pd.concat([total, percent], axis = 1, keys = ['Total','Percent'])
missing_data
# List of features not including columns with null values
const_cols = [c for c in df_traingoogle.columns if df_traingoogle[c].nunique(dropna=False)==1 ]
const_cols
print("Number of unique visitors in train set : ",df_traingoogle.fullVisitorId.nunique(), " out of rows : ",df_traingoogle.shape[0])
print("Number of unique visitors in test set : ",df_testgoogle.fullVisitorId.nunique(), " out of rows : ",df_testgoogle.shape[0])
print("Number of common visitors in train and test set : ",len(set(df_traingoogle.fullVisitorId.unique()).intersection(set(df_testgoogle.fullVisitorId.unique())) ))
print("Variables not in test but in train : ", set(df_traingoogle.columns).difference(set(df_testgoogle.columns)))
#Development and validation splits based on time to build the model. We can take the last two months as validation sample.
# Impute 0 for missing target values

df_traingoogle["totals.transactionRevenue"].fillna(0, inplace=True)
train_y = df_traingoogle["totals.transactionRevenue"].values
train_id = df_traingoogle["fullVisitorId"].values
test_id = df_testgoogle["fullVisitorId"].values
from sklearn import preprocessing
import datetime
# Impute 0 for missing target values
df_traingoogle["totals.transactionRevenue"].fillna(0, inplace=True)
train_y = df_traingoogle["totals.transactionRevenue"].values
train_id = df_traingoogle["fullVisitorId"].values
test_id = df_testgoogle["fullVisitorId"].values


# label encode the categorical variables and convert the numerical variables to float
cat_cols = ["channelGrouping", "device.browser", 
            "device.deviceCategory", "device.operatingSystem", 
            "geoNetwork.city", "geoNetwork.continent", 
            "geoNetwork.country", "geoNetwork.metro",
            "geoNetwork.networkDomain", "geoNetwork.region", 
            "geoNetwork.subContinent", "trafficSource.adContent", 
            "trafficSource.adwordsClickInfo.adNetworkType", 
            "trafficSource.adwordsClickInfo.gclId", 
            "trafficSource.adwordsClickInfo.page", 
            "trafficSource.adwordsClickInfo.slot", "trafficSource.campaign",
            "trafficSource.keyword", "trafficSource.medium", 
            "trafficSource.referralPath", "trafficSource.source",
            'trafficSource.adwordsClickInfo.isVideoAd', 'trafficSource.isTrueDirect']
for col in cat_cols:
    print(col)
    lbl = preprocessing.LabelEncoder()
    lbl.fit(list(df_traingoogle[col].values.astype('str')) + list(df_testgoogle[col].values.astype('str')))
    df_traingoogle[col] = lbl.transform(list(df_traingoogle[col].values.astype('str')))
    df_testgoogle[col] = lbl.transform(list(df_testgoogle[col].values.astype('str')))


num_cols = ["totals.hits", "totals.pageviews", "visitNumber", "visitStartTime", 'totals.bounces',  'totals.newVisits']    
for col in num_cols:
    df_traingoogle[col] = df_traingoogle[col].astype(float)
    df_testgoogle[col] = df_testgoogle[col].astype(float)

# Split the train dataset into development and values based on time 
df_traingoogle['date'] = df_traingoogle['date'].astype('datetime64[ns]')

dev_df = df_traingoogle[df_traingoogle['date'] <= datetime.date(2017,5,31)]
val_df = df_traingoogle[df_traingoogle['date'] <= datetime.date(2017,5,31)]
dev_y = np.log1p(dev_df["totals.transactionRevenue"].values)
val_y = np.log1p(val_df["totals.transactionRevenue"].values)

dev_X = dev_df[cat_cols + num_cols] 
val_X = val_df[cat_cols + num_cols] 
test_X = df_testgoogle[cat_cols + num_cols] 
# custom function to run light gbm model
import lightgbm as lgb
def run_lgb(train_X, train_y, val_X, val_y, test_X):
    params = {
        "objective" : "regression",
        "metric" : "rmse", 
        "num_leaves" : 30,
        "min_child_samples" : 100,
        "learning_rate" : 0.1,
        "bagging_fraction" : 0.7,
        "feature_fraction" : 0.5,
        "bagging_frequency" : 5,
        "bagging_seed" : 2018,
        "verbosity" : -1
    }
    
    lgtrain = lgb.Dataset(train_X, label=train_y)
    lgval = lgb.Dataset(val_X, label=val_y)
    model = lgb.train(params, lgtrain, 1000, valid_sets=[lgval], early_stopping_rounds=100, verbose_eval=100)
    
    pred_test_y = model.predict(test_X, num_iteration=model.best_iteration)
    pred_val_y = model.predict(val_X, num_iteration=model.best_iteration)
    return pred_test_y, model, pred_val_y

# Training the model #
pred_test, model, pred_val = run_lgb(dev_X, dev_y, val_X, val_y, test_X)
from sklearn import metrics
import numpy as np
import statsmodels
import seaborn as sns
from matplotlib import pyplot as plt
%matplotlib inline

pred_val[pred_val<0] = 0

val_pred_df = pd.DataFrame({"fullVisitorId":val_df["fullVisitorId"].values})
val_pred_df["transactionRevenue"] = val_df["totals.transactionRevenue"].values
val_pred_df["PredictedRevenue"] = np.expm1(pred_val)

#print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df["transactionRevenue"].values), np.log1p(val_pred_df["PredictedRevenue"].values))))
val_pred_df = val_pred_df.groupby("fullVisitorId")["transactionRevenue", "PredictedRevenue"].sum().reset_index()
print(np.sqrt(metrics.mean_squared_error(np.log1p(val_pred_df["transactionRevenue"].values), np.log1p(val_pred_df["PredictedRevenue"].values))))
sub_df = pd.DataFrame({"fullVisitorId":test_id})
pred_test[pred_test<0] = 0
sub_df["PredictedLogRevenue"] = np.expm1(pred_test)
sub_df = sub_df.groupby("fullVisitorId")["PredictedLogRevenue"].sum().reset_index()
sub_df.columns = ["fullVisitorId", "PredictedLogRevenue"]
sub_df["PredictedLogRevenue"] = np.log1p(sub_df["PredictedLogRevenue"])
sub_df.to_csv("baseline_lgb.csv", index=False)
sub_df.head()
from google.colab import files
files.download('baseline_lgb.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12211981566820276, 'fraction_numerical': 0.017327188940092168, 'mean_word_length': 4.753446447507954, 'pattern_counts': {'":': 2, '<': 10, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '79502', 'license': 'MIT', 'n_tokens_mistral': 3787, 'n_tokens_neox': 3587, 'n_words': 802, 'path': 'CS634Final_NYC_Taxi.ipynb', 'repo_name': 'hl533/CS634FinalProject', 'size': '10877'}