<jupyter_start><jupyter_text>Exploratory Data Analysis... Histograms of category click rates for each feature We're looking for features with well-sampled categories that show significant variation in click rate. We expect such features to be useful to help predict click rates in new data. Categories of a feature with high cardinality tend to be sparsely sampled, so variations in category click rate may not be significant. For this reason, we limit our attention to features with relatively low cardinality -- up to a few dozen or so categories. Unfortunately, it seems that click rates are too noisy to be of any use in prediction...<jupyter_code># Avazu competitition using pandas and sklearn library
import numpy as np
import pandas as pd
import lightgbm as lgb
import matplotlib.pyplot as plt
import category_encoders as ce
import copy
import pickle
import gc
#import xlearn as xl
%matplotlib inline
from datetime import datetime, date, time
from sklearn.linear_model import SGDClassifier
from sklearn.tree import DecisionTreeClassifier
from sklearn.metrics import log_loss
from sklearn.feature_extraction import FeatureHasher
from sklearn import preprocessing 
from sklearn.pipeline import Pipeline
from sklearn import model_selection
from sklearn.linear_model import LogisticRegression
from sklearn.externals import joblib
from sklearn.model_selection import GridSearchCV
from sklearn.metrics import classification_report
from xgboost import XGBClassifier #note: activate the xgboost environment
from lightgbm import LGBMClassifier


# set control parameters

important_columns = []

# set the number of hash bins
# 2**27 gets same result as 2**29, 2**30 produces memory error
n_features = 2**20
#n_features = 2**16

# method of handling categoricals: 'encode' or 'hash'
# categorical_method='encode'

# chunk_size
chunk_size= 1e5

# feature columns to drop
drop_columns = [[]]
drop_column = drop_columns[0]

# file and folder paths
train_file = 'train_shuffled.csv' #!!!!! no header
# train_file = 'train.csv'
model_path = './'

# number of training examples
n_rows = chunk_size

# seed for random generator
seed = 99

# option to archive model
archive_model = True

# set start time
start = datetime.now()

# column names in original data file
header = ['id','click','hour','C1','banner_pos','site_id','site_domain','site_category','app_id','app_domain','app_category','device_id',\
        'device_ip','device_model','device_type','device_conn_type','C14','C15','C16','C17','C18','C19','C20','C21']

# list of regularization hyperparameter values
# C_range = [1]

# select classifier
classifier = 'lgb'

# Option to tune the hyperparameters by cross-validation
cv_tune = False

#scale_pos_weight = sum(y_train==0)/sum(y_train==1)
scale_pos_weight = 1

# set model parameters
params = {}
params['boosting_type'] = 'gbdt'
params['objective'] = 'binary'
params['metric'] = 'binary_logloss'
# params['num_iterations'] = 100 # default
params['feature_fraction'] = 0.5
params['bagging_fraction'] = 0.5
params['bagging_freq'] = 1
params['num_threads'] = 4 # number of CPUs
params['is_unbalance'] = False
params['max_bin'] = n_features
params['importance_type'] = 'gain' 
#params['feature_name'] = 'auto' 
#params['categorical_feature'] = 'auto'
#params['n_estimators'] = 300
params['is_unbalance'] = False
params['scale_pos_weight'] = 1

# correction for unbalanced data only makes things worse!
# neg to pos class ratio

# fraction of training data to use 
train_frac = 0.8

# read data and preprocess
rawdata = pd.read_table(train_file, sep=',', chunksize=chunk_size, names=header,header=None,\
       nrows = n_rows).get_chunk()
#rawdata  = pd.read_table(train_file, sep=',', chunksize=chunk_size, names=None, header=0,\
#       nrows = n_rows).get_chunk()

# add hour and dayoftheweek features, drop 'id'
add_engineered_datetime_features = True
if(add_engineered_datetime_features): 
    print('adding engineered datetime features...')
    rawdata['hour']=rawdata['hour'].map(lambda x: datetime.strptime(str(x),"%y%m%d%H"))
    rawdata['dayoftheweek']=rawdata['hour'].map(lambda x:  x.weekday())
    rawdata['hour']=rawdata['hour'].map(lambda x:  x.hour)
rawdata.drop('id',axis=1,inplace=True)
# feature cardinalities
cardinality = {}
for col in rawdata.columns.tolist():
    cardinality[col]=len(rawdata[col].unique())
print(cardinality)


# mean click rate by hour -- not much variation,
# perhaps it can be mildly predictive
hs=rawdata.groupby('hour')['click'].mean()
hs.plot(kind='barh', figsize=(6,6))

# std dev of click rate by hour -- constant but very large!!
hs=rawdata.groupby('hour')['click'].std()
hs.plot(kind='barh', figsize=(6,6))


# CTR by hour is noisy!
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))

hs1=rawdata.groupby('hour')['click'].mean()
ax1.barh(hs1.index,hs1.values)

hs2=rawdata.groupby('hour')['click'].std()
ax2.barh(hs2.index,hs2.values)



#SNR plot
hs3=rawdata.groupby('hour')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

# number of impressions shown by hour
# is this solely determined by user actvity? 
# or is there a way the advertiser can control this?
# !!!!!predictive
hs=rawdata.groupby('hour')['click'].count()
hs.plot(kind='barh', figsize=(6,6))

# click counts by hour -- as expected, correlates well with number of impressions shown
# should be predictive
hs=rawdata.groupby('hour')['click'].sum()
hs.plot(kind='barh', figsize=(6,6))

# dayofweek CTR is noisy
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby('dayoftheweek')['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby('dayoftheweek')['click'].std()
ax2.barh(hs2.index,hs2.values)


# click rate by dayoftheweek
#hs=rawdata.groupby('dayoftheweek')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))

# total clicks
print(rawdata.groupby('dayoftheweek')['click'].sum())

#SNR plot
hs3=rawdata.groupby('dayoftheweek')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

# C1 CTR is noisy
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby('C1')['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby('C1')['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('C1')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# click rate by C1 category
# to noisy to tell if it's predictive
#hs=rawdata.groupby('C1')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C1')['click'].sum())

#SNR plot shows that 1002 and 1005 have very strong signals
hs3=rawdata.groupby('C1')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

# bannerpos CTR is noisy
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby('banner_pos')['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby('banner_pos')['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('banner_pos')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# click rate by banner_pos
#hs=rawdata.groupby('banner_pos')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('banner_pos')['click'].sum())

#SNR plot -- only positions 0 and 1 have strong signal
hs3=rawdata.groupby('banner_pos')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

# site_category CTR is noisy
attr = 'site_category'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('site_category')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# click rate by site_category
#hs=rawdata.groupby('site_category')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('site_category')['click'].sum())

#SNR plot
hs3=rawdata.groupby('site_category')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

# app_category CTR is noisy
attr = 'app_category'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('app_category')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# click rate by app_category
#hs=rawdata.groupby('app_category')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('app_category')['click'].sum())

#SNR plot -- there are certain app categories that are strongly predictive, 
#    because their click rates differ by a statistically significant amount
hs3=rawdata.groupby('app_category')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

# device_type CTR is noisy
attr = 'device_type'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('device_type')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# click rate by device_type
#hs=rawdata.groupby('device_type')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('device_type')['click'].sum())

# device_type CTR is noisy
attr = 'device_type'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].mad()
ax2.barh(hs2.index,hs2.values)

#SNR plot -- categories 0 and 1 strongly predictive, 
#    because their click rates differ by a statistically significant amount
hs3=rawdata.groupby('device_type')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

# device_conn_type CTR is noisy
attr = 'device_conn_type'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].mad()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('device_conn_type')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# click rate by device_conn_type
#hs=rawdata.groupby('device_conn_type')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('device_conn_type')['click'].sum())

#SNR plot -- categories 0, 2 and 3 are strongly predictive, 
#    because their click rates differ by a statistically significant amount
hs3=rawdata.groupby('device_conn_type')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

#!!!!! figure out how to display y-axis as category names, not numbers
# C14 CTR is noisy
attr = 'C14'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].mad()
ax2.barh(hs2.index,hs2.values)


# click rate by C14
#hs=rawdata.groupby('C14')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C14')['click'].sum())

#!!!!! figure out how to display y-axis as category names, not numbers
# C15 CTR is noisy
attr = 'C15'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('C15')['click'].count())
ax2.barh(hs2a.index,hs2a.values)
print(hs1,hs2a)

# click rate by C15
#hs=rawdata.groupby('C15')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C15')['click'].sum())

#SNR plot -- categories 300 and 320 are strongly predictive, 
#    because their click rates differ by a statistically significant amount
hs3=rawdata.groupby('C15')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

#!!!!! figure out how to display y-axis as category names, not numbers
# C16 CTR is noisy
attr = 'C16'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('C16')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# C16
#hs=rawdata.groupby('C16')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C16')['click'].count())
print(rawdata.groupby('C16')['click'].mean())
print(hs2a)
#SNR plot -- categories 50 and 250 are strongly predictive, 
#    because their click rates differ by a statistically significant amount
hs3=rawdata.groupby('C16')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))

#!!!!! figure out how to display y-axis as category names, not numbers
# C17 CTR is noisy
attr = 'C17'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].mad()
ax2.barh(hs2.index,hs2.values)


# click rate by C17
#hs=rawdata.groupby('C17')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C17')['click'].sum())

# C18 CTR is noisy
attr = 'C18'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('C18')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# click rate by C18
#hs=rawdata.groupby('C18')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C18')['click'].sum())
#SNR plot -- categories 0, 2, 3 are strongly predictive, 
#    because their click rates differ by a statistically significant amount
hs3=rawdata.groupby('C18')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))
print(hs4)
#!!!!! figure out how to display y-axis as category names, not numbers
# C19 CTR is noisy
attr = 'C19'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].mad()
ax2.barh(hs2.index,hs2.values)


# click rate by C19
#hs=rawdata.groupby('C19')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C19')['click'].sum())

#!!!!! figure out how to display y-axis as category names, not numbers
# C20 CTR is noisy
attr = 'C20'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].mad()
ax2.barh(hs2.index,hs2.values)


# click rate by C20
#hs=rawdata.groupby('C20')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C20')['click'].sum())

#!!!!! figure out how to display y-axis as category names, not numbers
# C21 CTR is noisy
attr = 'C21'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].mad()
ax2.barh(hs2.index,hs2.values)


# click rate by C21
#hs=rawdata.groupby('C21')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('C21')['click'].sum())

#!!!!! figure out how to display y-axis as category names, not numbers
# dayoftheweek CTR is noisy
attr = 'dayoftheweek'
f, (ax1, ax2) = plt.subplots(1, 2, sharey=True, figsize=(6,6))
hs1=rawdata.groupby(attr)['click'].mean()
ax1.barh(hs1.index,hs1.values)
hs2=rawdata.groupby(attr)['click'].std()
# Show "standard error" rather than "standard deviation"
hs2a = hs2/np.sqrt(rawdata.groupby('dayoftheweek')['click'].count())
ax2.barh(hs2a.index,hs2a.values)


# click rate by dayoftheweek
#hs=rawdata.groupby('dayoftheweek')['click'].mean()
#hs.plot(kind='barh', figsize=(6,6))
# total clicks
print(rawdata.groupby('dayoftheweek')['click'].sum())

# SNR plot -- all the categories are predictive, 
# because their click rates differ by a statistically significant amount
hs3=rawdata.groupby('dayoftheweek')['click'].count()
hs4 = (hs1/hs2)*np.sqrt(hs3)
hs4.plot(kind='barh', figsize=(6,6))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.177522197723616, 'fraction_numerical': 0.03581431380728205, 'mean_word_length': 6.505366726296959, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '338695', 'license': 'MIT', 'n_tokens_mistral': 6522, 'n_tokens_neox': 6255, 'n_words': 1366, 'path': 'sklearn/lgb_exploratory_data_analysis.ipynb', 'repo_name': 'jcatanza/kaggle_avazu_clickthrough_rate', 'size': '16781'}