<jupyter_start><jupyter_text>Import<jupyter_code>import pandas as pd
import numpy as np
import os
import pathlib
from tqdm import tqdm
import pickle
from xgboost import XGBClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score
from bayes_opt import BayesianOptimization
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import cross_val_score
from sklearn import metrics
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.model_selection import StratifiedKFold
import lightgbm
from sklearn.preprocessing import StandardScaler<jupyter_output><empty_output><jupyter_text>Data pre-processing  Loading data<jupyter_code>DATA_DIR = './Data'
campaign_data_df = pd.read_csv(os.path.join(DATA_DIR, 'campaign_data.csv'), parse_dates=['start_date','end_date'], dayfirst=True)

coupon_item_map_df = pd.read_csv(os.path.join(DATA_DIR, 'coupon_item_mapping.csv'))

customer_demographics_df = pd.read_csv(os.path.join(DATA_DIR, 'customer_demographics.csv'))
customer_transaction_data_df = pd.read_csv(os.path.join(DATA_DIR, 'customer_transaction_data.csv'))
item_data_df = pd.read_csv(os.path.join(DATA_DIR, 'item_data.csv'))
train_df = pd.read_csv(os.path.join(DATA_DIR, 'train.csv'))
test_df = pd.read_csv(os.path.join(DATA_DIR, 'test_QyjYwdj.csv'))
sample_submission_df = pd.read_csv(os.path.join(DATA_DIR, 'sample_submission_Byiv0dS.csv'))<jupyter_output><empty_output><jupyter_text>Exploration  Campaign Data<jupyter_code>campaign_data_df.head()
campaign_data_df['campaign_type'].unique()
campaign_data_df.shape
campaign_data_df['campaign_id'].unique()
campaign_data_df['campaign_id'].nunique()
campaign_data_df.info()
campaign_data_df['start_date'].min(), campaign_data_df['start_date'].max()
campaign_data_df['end_date'].min(), campaign_data_df['end_date'].max()
campaign_data_df['num_days'] = (campaign_data_df['end_date'] - campaign_data_df['start_date']).dt.days
campaign_data_df.describe()<jupyter_output><empty_output><jupyter_text>Observation - 28 different campaigns, each of either type X or type Y. - These campaigns are between 10th August 2012 to 1st December of 2013- The end dates vary from 21st September 2012 to 20th December 2013- Average number of days a campaign lasts is aobut 41 days ; Minimum is 32 days ; Maximum is about 70 days!  Coupon-Item Data<jupyter_code>coupon_item_map_df.head()
# verification for below
coupon_item_map_df[coupon_item_map_df['coupon_id'] == 1].shape
items_gby_coupon_df = coupon_item_map_df.groupby('coupon_id')[['item_id']].count()
items_gby_coupon_df.head()
items_gby_coupon_df.sort_values(by='item_id', ascending=False).head()
# verify that all are unique
coupon_item_map_df[coupon_item_map_df['coupon_id'] == 32][['item_id']].nunique()
items_gby_coupon_df.describe()
items_gby_coupon_df.shape<jupyter_output><empty_output><jupyter_text>--------<jupyter_code>coupon_groupby_items_df = coupon_item_map_df.groupby('item_id')[['coupon_id']].count()
coupon_groupby_items_df.head()
# verification for below
coupon_item_map_df[coupon_item_map_df['item_id'] == 4].shape
coupon_groupby_items_df.head()
# verify that all are unique
coupon_item_map_df[coupon_item_map_df['item_id'] == 10][['coupon_id']].nunique()
coupon_groupby_items_df.shape
coupon_groupby_items_df.describe()<jupyter_output><empty_output><jupyter_text>Observations- Some coupons have as many as 11800 items under them- Average items a coupon has are 83- There are a total of 36289 items which get featured in coupons- Some items appear in as 11 coupons- Average item gets featured in 2-3 coupons- Total number of coupons are 1116  Customer Demographics Data<jupyter_code>customer_demographics_df = pd.read_csv(os.path.join(DATA_DIR, 'customer_demographics.csv'))
customer_demographics_df.head()
customer_demographics_df['age_range'].unique()
customer_demographics_df['marital_status'].unique()
customer_demographics_df.isnull().sum()
customer_demographics_df.shape
customer_demographics_df['rented'].unique()
customer_demographics_df['family_size'].unique()
customer_demographics_df[customer_demographics_df['rented'] == 0].shape
customer_demographics_df[customer_demographics_df['rented'] == 1].shape
customer_demographics_df['income_bracket'].unique()<jupyter_output><empty_output><jupyter_text>Observations- 760 customers' data is there- Age ranges from 18 to 70+ - Age range is categorical but there's an inherent order so we need to change it to an ordered variable- Marital status has NaN for many- Very few customers have rented recommendations. Most have it bought! No particular correlation of such folks with their family size or no_of_children as such- Income bracket goes from 1 to 11. 11 being highest!- Family size is 1,2,3,4,5+ --- hence categorical Actions- Family size mein 5+ can be changed to just 5. So, it can be treated as an ordial number- Try removing marital status from data? Does it affect negatively?- Try removing no_of_children from data? Does it affect negatively?- Somehow incorporate both and see  Customer Transaction Data<jupyter_code>customer_transaction_df = pd.read_csv(os.path.join(DATA_DIR, 'customer_transaction_data.csv'), parse_dates=['date'], dayfirst=True)
customer_transaction_df.head()
customer_transaction_df.info()
customer_transaction_df['date'].min(), customer_transaction_df['date'].max()
customer_transaction_df['customer_id'].nunique()
customer_transaction_df['item_id'].nunique()
customer_transaction_df.shape
customer_transaction_gby_customer_df = customer_transaction_df.groupby('customer_id')[['item_id']].count()
# most shopaholic users
customer_transaction_gby_customer_df.sort_values(by='item_id', ascending=False)
customer_transaction_gby_customer_df.shape
# popular items
customer_transaction_gby_item_df = customer_transaction_df.groupby('item_id')[['customer_id']].count()
customer_transaction_gby_item_df.sort_values(by='customer_id', ascending=False)
customer_transaction_gby_item_df.shape<jupyter_output><empty_output><jupyter_text>------------------------------<jupyter_code>customer_transactions_with_some_kind_of_discounts_df = customer_transaction_df[(customer_transaction_df['coupon_discount'] < 0) | (customer_transaction_df['other_discount'] < 0)]
customer_transactions_with_some_kind_of_discounts_df.shape
customer_transaction_with_some_kind_of_discounts_gby_customer_df = customer_transactions_with_some_kind_of_discounts_df.groupby('customer_id')[['item_id']].count()
# most shopaholic users
customer_transaction_with_some_kind_of_discounts_gby_customer_df.sort_values(by='item_id', ascending=False)
customer_transaction_with_some_kind_of_discounts_gby_customer_df.shape
# popular items
customer_transaction_with_some_kind_of_discounts_gby_item_df = customer_transactions_with_some_kind_of_discounts_df.groupby('item_id')[['customer_id']].count()
customer_transaction_with_some_kind_of_discounts_gby_item_df.sort_values(by='customer_id', ascending=False)
customer_transaction_with_some_kind_of_discounts_gby_item_df.shape<jupyter_output><empty_output><jupyter_text>------------------------------<jupyter_code>customer_transactions_with_discounts_df = customer_transaction_df[customer_transaction_df['coupon_discount'] < 0]
customer_transactions_with_discounts_df.shape
customer_transactions_with_discounts_gby_customer_df = customer_transactions_with_discounts_df.groupby('customer_id')[['item_id']].count()
# most shopaholic users
customer_transactions_with_discounts_gby_customer_df.sort_values(by='item_id', ascending=False)
customer_transactions_with_discounts_gby_customer_df.shape
# popular items
customer_transactions_with_discounts_gby_item_df = customer_transactions_with_discounts_df.groupby('item_id')[['customer_id']].count()
customer_transactions_with_discounts_gby_item_df.sort_values(by='customer_id', ascending=False)
customer_transactions_with_discounts_gby_item_df.shape<jupyter_output><empty_output><jupyter_text>Observations- ~1600 customers involved in transactions ; ~74000 items being purchased- Some customers have done as much as 1500 transactions vs minimum number of transactions are around 42 by a customer (can become a feature)- Some items have been bought as much as 13000 times, versus some which are bought only once (can become a feature)- Distribution of above two things change a bit if we just consider transactions which had `other_discount` or `coupon_discount` (so we'll need features based on these as well)- About 13 lac transactions, out of which 6.8 lacs had some `other_discount` and only ~`22000` had `coupon_discount`. Of these that had coupon discount also not all were coupon that are there in our training data. Some are retailer coupons which we lack date for (got to know this from slack channel)                                                                                                                                  Item Data<jupyter_code>item_data_df.head()
item_data_df.info()
item_data_df['item_id'].nunique()
item_data_df['brand'].nunique()
item_data_df['brand_type'].unique()
item_data_df['category'].unique()
item_data_df[item_data_df['brand']==56]['category'].unique()
item_data_df[item_data_df['brand']==56]['brand_type'].unique()
item_data_df_gby_brand = item_data_df.groupby('brand')[['item_id']].count()
item_data_df_gby_brand.sort_values(by='item_id', ascending=False)<jupyter_output><empty_output><jupyter_text>Observations- If you look at items it looks like some big basket kinda shop which has these items- The brand 56 is a local brand which has enormously large set of items and of different types. It seems like some big shot.- Other than 56, highest number of items a brand has is about 1100, while lowest is 1.  Train data<jupyter_code>train_df.head()
train_df.shape
train_when_redeemed_df = train_df[train_df['redemption_status'] == 1]
train_when_redeemed_df.shape
train_df['customer_id'].nunique()
train_when_redeemed_df['customer_id'].nunique()
train_df['coupon_id'].nunique()
train_when_redeemed_df['coupon_id'].nunique()
train_df['campaign_id'].nunique()
train_when_redeemed_df['campaign_id'].nunique()<jupyter_output><empty_output><jupyter_text>Observations- Highly imbalanced. 99% of training data redemption doesn't happen- Only 247 customers of 1800 have redeemed couoon- Only 239 coupons of 866 have been redeemed- All campaigns exist in both redeemed and non-redeemed data  Pre-processing  Considering it to be a binary classification problemWe need a set of features which can help. Set of features to extract from above exploration:    **campaign data**- get number of days of campaign- get month of campaign- keep start date and end date as is- convert X and Y to one-hot **coupon-item mapping**- coupon id's number of items offered (how global kinda is the coupon)- for item id -- number of coupons available on it (will be based on date though)- for item id -- whether it gets featured in any coupon at all (again based on date0- coupon id to one-hot of itmems ?? (so depending upon which exact items are in discount might get redeemed)**customer demographics**- Data for 760 customers is there. Need to check how many of these occur in customer transaction and how many in train data- age_range needs to change to ordinal feature - 1,2,3,4,5 or so. Via a dict.- marital_status has NaN for many. Keep three possible values - married, single, and unknown. So, three features.- Income bracket is already ordinal- Family size mein 5+ can be replaced with 5 and then int operation can be applied on all- no_of_children NaN can be changed to 0?**customer transactions**- 1600 customers involved in purchasing 74,000 items- Extract shopaholic customers -- basically number of transactions done by user- Extract number of times item has been bought as a feature- Then extract the two above features when other_discount is > 0- Extract the two above when other_discount and coupon_discount > 0- Extract the two above when coupon_discount > 0- About 13 lac transactions, out of which 6.8 lacs had some `other_discount` and only ~`22000` had `coupon_discount`. Of these that had coupon discount also not all were coupon that are there in our training data. Some are retailer coupons which we lack date for (got to know this from slack channel)                                                   **item data**- can't do much here- about 5500 brands, 2 brand_type and 19 categories- Finally a coupon has multiple items in it, we can utilise that data to assign percentage of a coupon to each brand, brand_type and category**train data**  - Highly skewed- 18 campaigns exist in both redeemed and non-redeemed time- Only 239 coupons have been redeemed out of 866- Only 247 customers of 1800 have redeemed**test data**- many customers are common between test and train- the campaigns that exist in test are completely separate from campaigns in train- So, we need to make sure to not use campaign id as a feature.------------------------  Final decided features **from campaign_id**- campaign_type- no. of days it ran- start_date- end_date- no_of_coupons it has- % of brand- % of local brand- % of category**from coupon_id**- no of items offered- % of categories- % of brand- % of brand_type**from customer_id**- age_range (1,2,3,4,5)- family size- no of children (0 if NaN)- rented - marital status - three values- no. of transactions in total- no. of transactions in campaign_id days- no of transactions in total when other_discount > 0- no of transactions in total when other_discount > 0 in campaign id dates- no of transactions in total when discount_coupon > 0- no of transactions in total when discount_coupon > 0 in campaign id dates- no of transactions in total when discount_coupon > 0 and other_discount > 0 - no of transactions in total when discount_coupon > 0 and other_discount > 0 in campaign id dates- total quantities bought- ***predicted* income bracket** - how many times he has redeemed coupon before- no. of times he has redeemed same coupon before- Total quantities bought % based on categories- Total quantities bought % based on brand- Total quantities bought % based on brand_type- Average % discount for customer_id<jupyter_code>coupon_item_map_df.head()
item_data_df.head()
item_brand_details_df = item_data_df[['item_id','brand']]
item_brand_details_df.head(2)
item_brand_details_df = pd.concat([item_brand_details_df, pd.get_dummies(item_brand_details_df['brand'], prefix = 'brand')], axis=1)

del item_brand_details_df['brand']
item_brand_details_df.shape
item_brand_details_df.head(2)
item_brand_details_df = item_brand_details_df.set_index('item_id')
item_brand_details_df.shape
item_brand_details_df.head(2)
item_brand_np = item_brand_details_df.to_numpy()
item_ids_idx_list = item_brand_details_df.index.tolist()
item_brand_np.shape
# do same for brand type
item_brand_type_df = item_data_df[['item_id','brand_type']]
item_brand_type_df = pd.concat([item_brand_type_df, pd.get_dummies(item_brand_type_df['brand_type'], prefix = 'brand_type')], axis=1)
del item_brand_type_df['brand_type']
item_brand_type_df = item_brand_type_df.set_index('item_id')
item_brand_type_np = item_brand_type_df.to_numpy()
item_brand_type_df.head(3)
item_brand_type_np[0:3]
# do same for category
item_category_df = item_data_df[['item_id','category']]
item_category_df = pd.concat([item_category_df, pd.get_dummies(item_category_df['category'], prefix = 'category')], axis=1)
del item_category_df['category']
item_category_df = item_category_df.set_index('item_id')
item_category_np = item_category_df.to_numpy()
item_category_df.head(2)
item_category_np[0:2]
item_category_np.shape<jupyter_output><empty_output><jupyter_text>From campaign_id  - campaign_type- no. of days it ran- start_date- end_date- **no_of_coupons it has** (not taking right now)- **% of brand**- **% of local brand**- **% of category**<jupyter_code>campaign_data_df.head(3)
campaign_data_preprocessed_df = campaign_data_df.copy()
campaign_data_preprocessed_df = pd.concat([campaign_data_preprocessed_df, pd.get_dummies(campaign_data_preprocessed_df['campaign_type'], prefix = 'campaign_type')], axis=1)
del campaign_data_preprocessed_df['campaign_type']
campaign_data_preprocessed_df.head(1)
campaign_data_preprocessed_df['num_days_campaign'] =  (campaign_data_preprocessed_df['end_date'] - campaign_data_preprocessed_df['start_date']).dt.days
campaign_data_preprocessed_df.head(1)<jupyter_output><empty_output><jupyter_text>From coupon_id  - no of items offered- % of categories- % of brand- % of brand_type<jupyter_code>coupon_item_map_df.head(2)
coupon_item_gby_coupon_df = coupon_item_map_df.groupby('coupon_id').count()
coupon_item_gby_coupon_df.head()
coupon_item_gby_coupon_df = coupon_item_gby_coupon_df.rename(columns={'item_id': 'num_of_items_in_coupon'})
coupon_item_gby_coupon_df.head()
# unit test
coupon_item_map_df[coupon_item_map_df['coupon_id'] == 4][['item_id']].nunique()
coupon_item_gby_coupon_df = coupon_item_gby_coupon_df.reset_index()
coupon_item_gby_coupon_df.head()
# add X
coupon_id_list_in_order = coupon_item_gby_coupon_df['coupon_id'].tolist()
brand_mat = []
brand_type_mat = []
category_mat = []
for coupon_id in tqdm(coupon_id_list_in_order):
    list_of_items_in_coupon = coupon_item_map_df[coupon_item_map_df['coupon_id'] == coupon_id][['item_id']].values
    list_of_items_in_coupon = [a[0] for a in list_of_items_in_coupon]
    list_of_items_indices = [(a-1) for a in list_of_items_in_coupon]

    # coupon_brands_arr = item_brand_np[list_of_items_indices]
    # brand_arr = (np.sum(coupon_brands_arr, axis=0)[np.newaxis, :] / np.sum(coupon_brands_arr))
    
    coupon_brand_type_arr = item_brand_type_np[list_of_items_indices]
    brand_type_arr = (np.sum(coupon_brand_type_arr, axis=0)[np.newaxis, :] / np.sum(coupon_brand_type_arr))
    
    coupon_category_arr = item_category_np[list_of_items_indices]
    category_arr = (np.sum(coupon_category_arr, axis=0)[np.newaxis, :] / np.sum(coupon_category_arr))
    
    # brand_mat.append(brand_arr)
    brand_type_mat.append(brand_type_arr)
    category_mat.append(category_arr)
brand_mat_np = np.array(brand_mat)
brand_type_mat_np = np.array(brand_type_mat)
category_mat_np = np.array(category_mat)
brand_mat_np = np.squeeze(brand_mat_np)
brand_type_mat_np = np.squeeze(brand_type_mat_np)
category_mat_np = np.squeeze(category_mat_np)

coupon_item_preprocessed_df = pd.concat([coupon_item_gby_coupon_df, pd.DataFrame(brand_type_mat_np), pd.DataFrame(category_mat_np)], axis=1)

coupon_item_preprocessed_df.head()<jupyter_output><empty_output><jupyter_text>From customer_id  - age_range (1,2,3,4,5)- family size- no of children (0 if NaN)- rented - marital status - three values- no. of transactions in total- *no. of transactions in campaign_id days*- no of transactions in total when other_discount > 0- *no of transactions in total when other_discount > 0 in campaign id dates*- no of transactions in total when discount_coupon > 0- *no of transactions in total when discount_coupon > 0 in campaign id dates*- no of transactions in total when discount_coupon > 0 and other_discount > 0 - *no of transactions in total when discount_coupon > 0 and other_discount > 0 in campaign id dates*- **total quantities bought**- ***predicted* income bracket**- how many times he has redeemed coupon before- *no. of times he has redeemed same coupon before*- **Total quantities bought % based on categories**- **Total quantities bought % based on brand**- **Total quantities bought % based on brand_type**- Average % discount for customer_id<jupyter_code>def get_transaction_dicts_for_customer(train_df, customer_transaction_data_df):
    customer_ids_from_train = train_df['customer_id'].tolist()
    customer_ids_from_transactions = customer_transaction_data_df['customer_id'].tolist()
    list_customer_ids = list(set(customer_ids_from_train).union(set(customer_ids_from_transactions)))
    data_for_columns = {
        'customer_id':[],
        'no_of_transactions':[],
        'no_of_transactions_with_odr_discount_gt0':[],
        'no_of_transactions_with_coupon_discount_gt0':[],
        'no_of_transactions_with_both_discount_gt0':[],
        'times_coupon_redeemed_before': [],
        'ever_redeemed_before': [],
        'average_other_discount': [],
        'average_coupon_discount': [],
        'average_combined_discount': [],
    }
    for customer_id in tqdm(list_customer_ids):
        data_for_columns['customer_id'].append(customer_id)
        cus_txn_df = customer_transaction_data_df[customer_transaction_data_df['customer_id']==customer_id]
        # transaction features
        data_for_columns['no_of_transactions'].append(cus_txn_df.shape[0])
        data_for_columns['no_of_transactions_with_odr_discount_gt0'].append(cus_txn_df[cus_txn_df['other_discount'] < 0].shape[0])
        data_for_columns['no_of_transactions_with_coupon_discount_gt0'].append(cus_txn_df[cus_txn_df['coupon_discount'] < 0].shape[0])
        data_for_columns['no_of_transactions_with_both_discount_gt0'].append(cus_txn_df[(cus_txn_df['coupon_discount'] < 0) & (cus_txn_df['other_discount'] < 0)].shape[0])
        data_for_columns['average_other_discount'].append(cus_txn_df['other_discount'].mean())
        data_for_columns['average_coupon_discount'].append(cus_txn_df['coupon_discount'].mean())
        data_for_columns['average_combined_discount'].append((cus_txn_df['coupon_discount'] + cus_txn_df['other_discount']).mean())
        # coupon redemption features
        cus_train_df = train_df[train_df['customer_id'] == customer_id]
        times_redeemed_before = cus_train_df[cus_train_df['redemption_status'] > 0].shape[0]
        data_for_columns['times_coupon_redeemed_before'].append(times_redeemed_before)
        if times_redeemed_before > 0:
            data_for_columns['ever_redeemed_before'].append(1)
        else:
            data_for_columns['ever_redeemed_before'].append(0)

    return pd.DataFrame.from_dict(data_for_columns)
customers_preprocessed_df = get_transaction_dicts_for_customer(train_df, customer_transaction_data_df)
customers_preprocessed_df.head()
def get_features_df_related_to_customer_and_coupon_and_campaign(df_to_fill, 
                                                                train_df, 
                                                                customer_transaction_data_df, 
                                                                campaign_data_df):
    
    feature_data_df = df_to_fill.copy()
    feature_data_df['no_of_transactions_in_campaign'] = 0
    feature_data_df['no_of_transactions_in_campaign_with_odr_discount_gt0'] = 0
    feature_data_df['no_of_transactions_in_campaign_with_coupon_discount_gt0'] = 0
    feature_data_df['no_of_transactions_in_campaign_with_both_discount_gt0'] = 0
    feature_data_df['no_of_times_this_coupon_redeemed_before'] = 0    
    
    for iter, row in tqdm(df_to_fill.iterrows(), total=df_to_fill.shape[0]):
        customer_id = row['customer_id']
        coupon_id = row['coupon_id']
        campaign_id = row['campaign_id']
        
        campaign_start_date, campaign_end_date = pd.to_datetime(campaign_data_df[campaign_data_df['campaign_id']==campaign_id][['start_date','end_date']].values[0])
        cus_txn_df = customer_transaction_data_df[(customer_transaction_data_df['customer_id']==customer_id) 
                                                  & (pd.to_datetime(customer_transaction_data_df['date']) >= campaign_start_date)
                                                  & (pd.to_datetime(customer_transaction_data_df['date']) <= campaign_end_date)]
        
        # transactions
        feature_data_df.at[iter,'no_of_transactions_in_campaign'] = cus_txn_df.shape[0]
        feature_data_df.at[iter,'no_of_transactions_in_campaign_with_odr_discount_gt0'] = cus_txn_df[cus_txn_df['other_discount'] < 0].shape[0]
        feature_data_df.at[iter,'no_of_transactions_in_campaign_with_coupon_discount_gt0'] = cus_txn_df[cus_txn_df['coupon_discount'] < 0].shape[0]
        feature_data_df.at[iter,'no_of_transactions_in_campaign_with_both_discount_gt0'] = cus_txn_df[(cus_txn_df['coupon_discount'] < 0) & (cus_txn_df['other_discount'] < 0)].shape[0]
        
        # coupon
        feature_data_df.at[iter,'no_of_times_this_coupon_redeemed_before'] = train_df[(train_df['customer_id'] == customer_id) & (train_df['coupon_id'] == coupon_id) & (train_df['redemption_status'] == 1)].shape[0]
    
    return feature_data_df
customer_local_features_df = get_features_df_related_to_customer_and_coupon_and_campaign(train_df, train_df, customer_transaction_df, campaign_data_df)
with open('./Data/customer_train_df_local_features_df.pkl', 'wb') as f:
    pickle.dump(customer_local_features_df, f)
with open('./Data/customer_train_df_local_features_df.pkl', 'rb') as f:
    customer_local_features_df = pickle.load(f)<jupyter_output><empty_output><jupyter_text>Merging Pre-processed Data- `customers_preprocessed_df` : containing customer_id related features. 1 row for each customer_id- `customer_local_features_df` : built by adding new columns to train_df / test_df- `coupon_item_preprocessed_df` : one row for each coupon id. Contains info related to brand %, item % etc.- `campaign_data_preprocessed_df` : one row for each campaign. 4-5 columns onlySteps :   1. Build `customer_local_features_df`2. Merge `customers_preprocessed_df` into it based on `customer_id`. Check that columns should only increase, no rows increase.3. Merge `coupon_item_preprocessed_df` into it. Only cols should increase.4. Merge `campaign_data_preprocessed_df` ; only columns should increase.5. Remove unnecessary fields and use data to train algorithms.***Later try to also merge customer demographic data. But that's gonna introduce more categorical variables, which most ML algos don't like. However, lightGBM can work with categorical columns!***<jupyter_code># lets build upon train_i_df
train_i_df = customer_local_features_df.copy()
train_i_df.shape
train_i_df.head(2)
train_i_df = train_i_df.merge(customers_preprocessed_df, on='customer_id')
train_i_df.shape
train_i_df = train_i_df.merge(coupon_item_preprocessed_df, on='coupon_id')
train_i_df.shape
train_i_df = train_i_df.merge(campaign_data_preprocessed_df, on='campaign_id')
train_i_df.shape
train_i_df.head(1)<jupyter_output><empty_output><jupyter_text>Models  Xgboost<jupyter_code># get the labels
y_i = train_i_df['redemption_status'].values

train_i_df_fin = train_i_df.drop(['id', 'campaign_id', 'coupon_id', 'customer_id' ,'redemption_status', 'start_date', 'end_date'], axis=1)
x_i = train_i_df_fin.values


#
# Create training and validation sets
#
x_tr_i, x_val_i, y_tr_i, y_val_i = train_test_split(x_i, y_i, test_size=0.4, random_state=42, stratify=y_i)

model_i = XGBClassifier(n_jobs=6)
model_i.fit(x_tr_i, y_tr_i)<jupyter_output><empty_output><jupyter_text>This job started at around Thu Oct  3 20:01:42 UTC 2019  ( or around 1:30am)<jupyter_code>from datetime import datetime
print(datetime.now())
y_pred_val = model_i.predict(x_val_i)
accuracy = accuracy_score(y_val_i, y_pred_val)
accuracy
y_pred_val_prob = model_i.predict_proba(x_val_i)<jupyter_output><empty_output><jupyter_text>Predicting for test<jupyter_code>test_data_df = pd.read_csv('./Data/test_QyjYwdj.csv')
test_data_df.head(2)
test_data_i_df = get_features_df_related_to_customer_and_coupon_and_campaign(test_data_df, train_df, customer_transaction_df, campaign_data_df)

with open('./Data/test_data_i_df.pkl', 'wb') as f:
    pickle.dump(test_data_i_df, f)
with open('./Data/test_data_i_df.pkl', 'rb') as f:
    test_data_i_df = pickle.load(f)
test_data_i_df.shape
test_data_i_df = test_data_i_df.merge(customers_preprocessed_df, on='customer_id')
test_data_i_df = test_data_i_df.merge(coupon_item_preprocessed_df, on='coupon_id')
test_data_i_df = test_data_i_df.merge(campaign_data_preprocessed_df, on='campaign_id')
test_data_i_df.shape
test_data_i_df_fin = test_data_i_df.drop(['id', 'campaign_id', 'coupon_id', 'customer_id' , 'start_date', 'end_date'], axis=1)
x_test_i = test_data_i_df_fin.values

x_test_i.shape
y_pred_test_prob_i = model_i.predict_proba(x_test_i)
y_pred_test_i = model_i.predict(x_test_i)<jupyter_output><empty_output><jupyter_text>Submitting<jupyter_code>submit_i = pd.read_csv('./Data/sample_submission_Byiv0dS.csv')
submit_i.head(2)
submit_i.shape
submit_i[submit_i['redemption_status']==1].shape
submit_i['redemption_status'] = y_pred_test_i
submit_i[submit_i['redemption_status']==1].shape
submit_i.shape
submit_i.to_csv('./Data/submit_i.csv', index=False)
submit_i_probs = pd.read_csv('./Data/sample_submission_Byiv0dS.csv')
submit_i_probs['redemption_status'] = y_pred_test_prob_i
submit_i_probs.to_csv('./Data/submit_i_probs.csv', index=False)<jupyter_output><empty_output><jupyter_text>Using lightgbm<jupyter_code>x_tr_i, x_val_i, y_tr_i, y_val_i
x_tr_i.shape
sc = StandardScaler()
train_data = lightgbm.Dataset(sc.fit_transform(x_tr_i), label=y_tr_i)
test_data = lightgbm.Dataset(sc.fit_transform(x_val_i), label=y_val_i)


#
# Train the model
#

parameters = {
    'application': 'binary',
    'objective': 'binary',
    'metric': 'auc',
    'is_unbalance': 'true',
    'boosting': 'gbdt',
    'num_leaves': 31,
    'feature_fraction': 0.5,
    'bagging_fraction': 0.5,
    'bagging_freq': 20,
    'learning_rate': 0.05,
    'verbose': 0
}

model = lightgbm.train(parameters,
                       train_data,
                       valid_sets=test_data,
                       num_boost_round=5000,
                       early_stopping_rounds=100)
y_pred_test_prob_ii = model.predict(sc.fit_transform(x_test_i))
y_pred_test_prob_ii[0:10]
submit_ii = pd.read_csv('./Data/sample_submission_Byiv0dS.csv')
submit_ii['redemption_status'] = y_pred_test_prob_ii
submit_ii.to_csv('./Data/submit_ii_iv.csv', index=False)<jupyter_output><empty_output><jupyter_text>Using xgboost cross validation<jupyter_code>kf = StratifiedKFold(n_splits=10, shuffle = True, random_state = 228)
fold_splits = kf.split(x_i, y_i)
cv_scores = []
pred_full_test = 0
pred_train = np.zeros((x_i.shape[0]))
i = 1
for dev_index, val_index in fold_splits:
    print('Started fold ' + str(i))
    dev_X, val_X = x_i[dev_index], x_i[val_index]
    dev_y, val_y = y_i[dev_index], y_i[val_index]
    
    # model train
    model = XGBClassifier(n_jobs=6)
    model.fit(dev_X, dev_y)
    pred_val_y = model.predict(val_X)
    # pred_full_test = pred_full_test + pred_test_y
    pred_train[val_index] = pred_val_y
    cv_score = metrics.roc_auc_score(val_y, pred_val_y)
    cv_scores.append(cv_score)
    print(' cv score {}: {}'.format(i, cv_score))
    i += 1
label = 'xgboost-model'
print('{} cv scores : {}'.format(label, cv_scores))
print('{} cv mean score : {}'.format(label, np.mean(cv_scores)))
print('{} cv std score : {}'.format(label, np.std(cv_scores)))
# pred_full_test = pred_full_test / 5.0
results = {'label': label,
          'train': pred_train, 'test': pred_full_test,
          'cv': cv_scores}<jupyter_output>Started fold 1
 cv score 1: 0.999806800618238
Started fold 2
 cv score 2: 0.986043770687331
Started fold 3
 cv score 3: 0.9998712004121586
Started fold 4
 cv score 4: 0.9995492014425553
Started fold 5
 cv score 5: 0.9999356002060794
Started fold 6
 cv score 6: 0.9999356002060794
Started fold 7
 cv score 7: 0.9997424008243174
Started fold 8
 cv score 8: 0.9998712004121586
Started fold 9
 cv score 9: 0.9927642861679827
Started fold 10
 cv score 10: 0.992991155761635
xgboost-model cv scores : [0.999806800618238, 0.986043770687331, 0.9998712004121586, 0.9995492014425553, 0.9999356002060794, 0.9999356002060794, 0.9997424008243174, 0.9998712004121586, 0.9927642861679827, 0.992991155761635]
xgboost-model cv mean score : 0.9970511216738535
xgboost-model cv std score : 0.004578704303954754<jupyter_text>So, actually problem is with my features only; not with algorithm. Cuz here it gives 0.99 and there it becomes 0.5.  Remove unused variables<jupyter_code>import sys
def sizeof_fmt(num, suffix='B'):
    ''' by <NAME>,  https://stackoverflow.com/a/1094933/1870254, modified'''
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f %s%s" % (num, 'Yi', suffix)

for name, size in sorted(((name, sys.getsizeof(value)) for name, value in locals().items()),
                         key= lambda x: -x[1])[:10]:
    print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
# remove unused variables
del _34
del train_i_df_fin
del test_data_i_df_fin
del X_train
del X_test
import sys
def sizeof_fmt(num, suffix='B'):
    ''' by <NAME>,  https://stackoverflow.com/a/1094933/1870254, modified'''
    for unit in ['','Ki','Mi','Gi','Ti','Pi','Ei','Zi']:
        if abs(num) < 1024.0:
            return "%3.1f %s%s" % (num, unit, suffix)
        num /= 1024.0
    return "%.1f %s%s" % (num, 'Yi', suffix)

for name, size in sorted(((name, sys.getsizeof(value)) for name, value in locals().items()),
                         key= lambda x: -x[1])[:10]:
    print("{:>30}: {:>8}".format(name, sizeof_fmt(size)))
del _159
del _119
aa<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10407799812852787, 'fraction_numerical': 0.025113948504331553, 'mean_word_length': 5.842213961173069, 'pattern_counts': {'":': 0, '<': 108, '<?xml version=': 0, '>': 120, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '216073', 'license': 'Apache-2.0', 'n_tokens_mistral': 11939, 'n_tokens_neox': 11213, 'n_words': 2389, 'path': 'Amexpert-2019.ipynb', 'repo_name': 'bhavul/AmExpert2019Hackathon', 'size': '33139'}