<jupyter_start><jupyter_text><jupyter_code># Import libraries
# Basic libraries
import pandas as pd
import numpy as np

# Preprocesing and feature extraction
from sklearn.model_selection import KFold
from itertools import product
from sklearn import preprocessing
from sklearn.feature_extraction.text import TfidfVectorizer

# Modeling
from sklearn.linear_model import LinearRegression
from xgboost import XGBRegressor
from xgboost import plot_importance

# Ploting
import matplotlib.pyplot as plt
from matplotlib.pyplot import figure
import seaborn as sns

# To deal with downloaded files
from google.colab import files

# Others
import time
# Functions used

# Downcast variables 
def downcast(df):
  '''
    Change columns types from 64 to 32 bits
  '''
  float64columns = df.select_dtypes(['float64']).columns.tolist()
  int64columns = df.select_dtypes(['int64']).columns.tolist()
  df[float64columns] = df[float64columns].astype('float32')
  df[int64columns] = df[int64columns].astype('int32')
  return df

# Averages of target variables
def averagevalues(df, group, target):
  '''
  Takes a group of variables (from one two three) and a target variables to return the average 
  '''
  grouped = df.groupby(group).agg({target: ['mean']})
  
  if (len(group) == 1):
    grouped.columns = [group[0]+'_avg_'+target]
  elif (len(group) == 2):
    grouped.columns = [group[0]+'_'+group[1]+'_avg_'+target]
  else:
    grouped.columns = [group[0]+'_'+group[1]+'-'+group[2]+'_avg_'+target]

  grouped.reset_index(inplace=True)
  df = pd.merge(df, grouped, on=group, how='left')
  return df

# Function for mean encodings using LOO scheme
def meaneconloo(df, groupvar):
    """
    df: Pandas data frame that contains categorical and target features
    catfeature: feature we want to create the mean encoding regularized through LOO scheme
    """
    target_sum = df.groupby(groupvar)['target'].transform('sum')
    n_objects  = df.groupby(groupvar)['target'].transform('count')
    df[groupvar+'_target_ME'] = (target_sum - df['target']) / (n_objects - 1)
    df[groupvar+'_target_ME'].fillna(df[groupvar+'_target_ME'].mean(), inplace=True)
    print('Correlation of', groupvar, '_target_ME:', np.corrcoef(df['target'].values, df[groupvar+'_target_ME'].values)[0][1])
    return df

# Function for mean encoding with smothing method
def me_smoothing(df, groupvar, alpha):
  n_objects = df.groupby(groupvar)['target'].transform('count')
  meantarget = df.groupby(groupvar)['target'].transform('mean')
  df[groupvar+'_target_ME'] = (meantarget * n_objects + 0.3343 * alpha) / (n_objects + alpha)
  print('Correlation of', groupvar, '_target_ME:', np.corrcoef(df['target'].values, df[groupvar+'_target_ME'].values)[0][1])
  return df

# Function for kfold mean target encoding
def me_kfold(df, folds, groupvar):
  df[groupvar+'_target_ME'] = 0
  kf = KFold(folds, shuffle = False, random_state = 123)

  for tr_ind_ME, val_ind_ME in kf.split(train):
    X_tr_ME, X_val_ME = df.loc[tr_ind_ME], df.loc[val_ind_ME]
    X_val_ME[groupvar+'_target_ME'] = X_val_ME[groupvar].map(X_tr_ME.groupby(groupvar).target.mean())
    df.loc[val_ind_ME] = X_val_ME
  df[groupvar+'_target_ME'].fillna(0.3343, inplace=True)
  print('Correlation of', groupvar, '_target_ME:', np.corrcoef(df['target'].values, df[groupvar+'_target_ME'].values)[0][1])
  return df

# Function that creates the lags of given columns
def lag_feature(df, lags, col):
    tmp = df[['date_block_num','shop_id','item_id', col]]
    for i in lags:
        shifted = tmp.copy()
        shifted.columns = ['date_block_num','shop_id','item_id', col+'_lag_'+ str(i)]
        shifted['date_block_num'] += i
        shifted[col+'_lag_'+ str(i)] = shifted[col+'_lag_'+ str(i)].astype('float16')
        df = pd.merge(df, shifted, on=['date_block_num','shop_id','item_id'], how='left')
    return df    
# Load de the data
transactions    = pd.read_csv('/content/drive/My Drive/DataScience/data/competitive-data-science-predict-future-sales/sales_train.csv')
items           = pd.read_csv('/content/drive/My Drive/DataScience/data/competitive-data-science-predict-future-sales/items.csv')
item_categories = pd.read_csv('/content/drive/My Drive/DataScience/data/competitive-data-science-predict-future-sales/item_categories.csv')
shops           = pd.read_csv('/content/drive/My Drive/DataScience/data/competitive-data-science-predict-future-sales/shops.csv')
transactions_test    = pd.read_csv('/content/drive/My Drive/DataScience/data/competitive-data-science-predict-future-sales/test.csv')

index_cols = ['shop_id', 'item_id', 'date_block_num']

# For every month we create a grid from all shops/items combinations from that month
grid = [] 
for block_num in transactions['date_block_num'].unique():
    cur_shops = transactions[transactions['date_block_num']==block_num]['shop_id'].unique()
    cur_items = transactions[transactions['date_block_num']==block_num]['item_id'].unique()
    grid.append(np.array(list(product(*[cur_shops, cur_items, [block_num]])), dtype = 'int16'))

# Turn the grid into pandas dataframe
grid = pd.DataFrame(np.vstack(grid), columns = index_cols, dtype = np.int32)

# Get aggregated values of sales by shop_id, item_id
gb = transactions.groupby(index_cols).agg({'item_cnt_day':'sum'})
gb.columns = ['target']
gb.reset_index(inplace = True)

# Join aggregated data to the grid and sort the data
train = pd.merge(grid, gb, how='left',on=index_cols).fillna(0)
train['target'] = train['target'].clip(0,20)
train.sort_values(['date_block_num','shop_id','item_id'],inplace=True)

del grid, gb
# Set ID as index and join train with test data
transactions_test.set_index('ID', inplace = True)
transactions_test['date_block_num'] = 34
data = pd.concat([train, transactions_test], sort = False)
data = downcast(data)

del train
# Add items and shops to test data
data = pd.merge(data, items, on = ['item_id']) # Join item_name and item category
data = pd.merge(data, shops, on = ['shop_id']) # Join shop_name
del items, shops, item_categories, 
data.columns
data.fillna(0, inplace = True)
data.isnull().sum()
# Mean encoding with smothing regularization  
data
# Simple grouping
data = me_smoothing(data, 'date_block_num', 1000)
data = me_smoothing(data, 'shop_id', 1000)
data = me_smoothing(data, 'item_category_id', 1000)
data = me_smoothing(data, 'item_id', 1000)

# Pairs of vars
data['date_item'] = data['date_block_num'].astype(str) + data['item_id'].astype(str)
data['date_shop'] = data['date_block_num'].astype(str) + data['shop_id'].astype(str)
data['date_cat'] = data['date_block_num'].astype(str) + data['item_category_id'].astype(str)
data['shop_cat'] = data['shop_id'].astype(str) + data['item_category_id'].astype(str)

data = me_smoothing(data, 'date_item', 1000)
data = me_smoothing(data, 'date_shop', 1000)
data = me_smoothing(data, 'date_cat', 1000)
data = me_smoothing(data, 'shop_cat', 1000)

data.drop(['date_item', 'date_shop', 'date_cat', 'shop_cat'], axis = 1, inplace=True)
data.columns
# Lagged values
ts = time.time()

data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'target')
data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'date_block_num_target_ME')
data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'shop_id_target_ME')
data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'item_category_id_target_ME')
data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'item_id_target_ME')
data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'date_item_target_ME')
data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'date_shop_target_ME')
data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'date_cat_target_ME')
data = lag_feature(data, [1, 2, 3, 4, 5, 12], 'shop_cat_target_ME')

data = downcast(data)

time.time() - ts
# Drop level 0 lagged features
data.drop(['date_block_num_target_ME', 'shop_id_target_ME', 'item_category_id_target_ME', 'item_id_target_ME', 'date_item_target_ME', 'date_shop_target_ME', 'date_cat_target_ME', 'shop_cat_target_ME'], axis = 1, inplace = True) # Drop object columns
data = downcast(data)
data.fillna(0, inplace=True)
data.drop(['item_name', 'shop_name'], axis = 1, inplace = True) # Drop object columns

X_train = data[(data.date_block_num > 11) & (data.date_block_num < 33)].drop(['target'], axis=1)
Y_train = data[(data.date_block_num > 11) & (data.date_block_num < 33)]['target']
X_valid = data[data.date_block_num == 33].drop(['target'], axis=1)
Y_valid = data[data.date_block_num == 33]['target']
X_test = data[data.date_block_num == 34].drop(['target'], axis=1)

# del data
ts = time.time()

model = XGBRegressor(
    tree_method = "gpu_hist",
    max_depth = 6, # Tree related parameter: determines how deeply each tree is allowed to grow during any boosting round
    min_child_weight = 600, # 
    colsample_bytree=0.8, # percentage of features used per tree. High value can lead to overfitting
    n_estimators = 500, # number of trees you want to build.
    subsample=0.8, # Boosting parameter: percentage of samples used per tree. Low value can lead to underfitting
    eta=0.3, #  Boosting parameter
    seed=123)

model.fit(
    X_train, 
    Y_train, 
    eval_metric="rmse", 
    eval_set=[(X_train, Y_train), (X_valid, Y_valid)], 
    verbose=True, 
    early_stopping_rounds = 10)

time.time() - ts

feat_importances = pd.Series(model.feature_importances_, index=X_train.columns)
feat_importances.nlargest(15).plot(kind='barh')
Y_test = model.predict(X_test).clip(0, 20)
testdata = X_test
testdata['Y_test'] = Y_test

# Add target variable to transactions_test and leave only ID and target
transactions_test.reset_index(inplace = True)
submition = pd.merge(transactions_test, testdata[['shop_id', 'item_id', 'Y_test']], on = ['shop_id', 'item_id'])
submition = submition[['ID', 'Y_test']]
submition.columns = ['ID', 'item_cnt_month']
# Export results
submition.to_csv('V16_Smooth1000_All11.csv', index=False)
files.download('V16_Smooth1000_All11.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14866916304017813, 'fraction_numerical': 0.02246736160307661, 'mean_word_length': 5.601202404809619, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '285633', 'license': 'MIT', 'n_tokens_mistral': 3682, 'n_tokens_neox': 3464, 'n_words': 791, 'path': 'HWDCC_LFTK_Final_Proyect.ipynb', 'repo_name': 'andresvir14/HWDCC_LFTK-', 'size': '9881'}