<jupyter_start><jupyter_text>0.0. IMPORTS<jupyter_code>#import PANDAS 
import pandas as pd
import inflection
import math
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
from IPython.core.display import HTML<jupyter_output><empty_output><jupyter_text>0.1. Helper Functions  0.2. Load Datasets<jupyter_code>#read dataset 
df_sales_raw = pd.read_csv('../datasets/train.csv', low_memory=False)
df_store_raw = pd.read_csv('../datasets/store.csv', low_memory=False)


#merge df's

df_raw = pd.merge(df_sales_raw, df_store_raw, how='left', on="Store")
#Show the sample of the data
df_raw.sample()<jupyter_output><empty_output><jupyter_text>1.0. DATA DESCRITPTION<jupyter_code>#Copy the original data
df1 = df_raw.copy()<jupyter_output><empty_output><jupyter_text>1.1. Rename Columns<jupyter_code>cols_old = ['Store', 'DayOfWeek', 'Date', 'Sales', 'Customers', 'Open', 'Promo', 'StateHoliday', 'SchoolHoliday', 'StoreType', 'Assortment',
       'CompetitionDistance', 'CompetitionOpenSinceMonth', 'CompetitionOpenSinceYear', 'Promo2', 'Promo2SinceWeek','Promo2SinceYear', 'PromoInterval']


#Function Snake Case
snackecase = lambda x: inflection.underscore(x)

#Map of the names
cols_new = list(map(snackecase, cols_old))

#Rename name columns
df1.columns = cols_new<jupyter_output><empty_output><jupyter_text>1.2 Data Dimensions<jupyter_code>print('Number of the rows: {}'.format(df1.shape[0]))
print('Number of the Cols: {}'.format(df1.shape[1]))<jupyter_output>Number of the rows: 1017209
Number of the Cols: 18<jupyter_text>1.3 Data Types<jupyter_code>#Change the type for DATETIME the column DATE
df1['date'] = pd.to_datetime(df1['date'])<jupyter_output><empty_output><jupyter_text>1.4 Check NA<jupyter_code>#Check SUM of the NA
df1.isna().sum()<jupyter_output><empty_output><jupyter_text>1.5. Fillout NA<jupyter_code>#competition_distance              2642
df1['competition_distance'] = df1['competition_distance'].apply(lambda x: 200000.0 if math.isnan(x) else x )

#competition_open_since_month    323348
df1['competition_open_since_month'] =  df1.apply(lambda x: x['date'].month if math.isnan(x['competition_open_since_month']) else x['competition_open_since_month'], axis=1)

#competition_open_since_year     323348
df1['competition_open_since_year'] =  df1.apply(lambda x: x['date'].year if math.isnan(x['competition_open_since_year']) else x['competition_open_since_year'], axis=1)

#promo2_since_week               508031
df1['promo2_since_week'] =  df1.apply(lambda x: x['date'].week if math.isnan(x['promo2_since_week']) else x['promo2_since_week'], axis=1)

#promo2_since_year               508031
df1['promo2_since_year'] =  df1.apply(lambda x: x['date'].year if math.isnan(x['promo2_since_year']) else x['promo2_since_year'], axis=1)

#promo_interval                  508031
month_map = {1: 'Jan', 2: 'Fev', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov', 12: 'Dec' }

df1['promo_interval'].fillna(0, inplace=True)

df1['month_map'] = df1['date'].dt.month.map(month_map)

df1['is_promo'] =  df1[['promo_interval', 'month_map']].apply(lambda x: 0 if x['promo_interval'] == 0 else 1 if x['month_map'] in  x['promo_interval'].split(',') else 0, axis=1)<jupyter_output><empty_output><jupyter_text>1.6. Change Types<jupyter_code>df1['competition_open_since_month'] = df1['competition_open_since_month'].astype('int64')
df1['competition_open_since_year'] = df1['competition_open_since_year'].astype('int64')

df1['promo2_since_week'] = df1['promo2_since_week'].astype('int64')
df1['promo2_since_year'] = df1['promo2_since_year'].astype('int64')<jupyter_output><empty_output><jupyter_text>1.7. Descriptive Statistical<jupyter_code>#Separate the numerics and categorics 

#numerics
num_attributes = df1.select_dtypes( include=['int64', 'float64'] )

#categorics
cat_attributes = df1.select_dtypes( exclude=['int64', 'float64', 'datetime64[ns]'] )<jupyter_output><empty_output><jupyter_text>1.7.1 Numerical Attributes<jupyter_code># Central Tendency - mean, median

ct1 = pd.DataFrame(num_attributes.apply( np.mean )).T
ct2 = pd.DataFrame(num_attributes.apply( np.median )).T

# Dispersion - std, min, max, range, skew, kurtosis

d1 = pd.DataFrame(num_attributes.apply(np.std)).T
d2 = pd.DataFrame(num_attributes.apply(min)).T
d3 = pd.DataFrame(num_attributes.apply(max)).T
d4 = pd.DataFrame(num_attributes.apply( lambda x: x.max() - x.min() ) ).T
d5 = pd.DataFrame(num_attributes.apply( lambda x: x.skew() ) ).T
d6 = pd.DataFrame(num_attributes.apply( lambda x: x.kurtosis() ) ).T

#concatenate the dispersions

metrics = pd.concat( [d2, d3, d4, ct1, ct2, d1, d5, d6] ).T.reset_index()
metrics.columns = [['Attributes','Min', 'Max', 'Range', 'Media', 'Mediana', 'STD', 'Skew', 'Kurtosis']]
metrics
sns.distplot( df1['competition_distance'] )<jupyter_output><empty_output><jupyter_text>1.7.2 Categorical Attributes<jupyter_code>#Types of categorical
cat_attributes.apply( lambda x: x.unique().shape[0] )
#Create BOX PLOT
aux1 = df1[(df1['state_holiday'] != '0') & (df1['sales'] > 0 )]


plt.subplot( 1, 3 ,1)
sns.boxplot(x='state_holiday' , y='sales', data=aux1)

plt.subplot( 1, 3 ,2)
sns.boxplot(x='store_type' , y='sales', data=aux1)

plt.subplot( 1, 3 ,3)
sns.boxplot(x='assortment' , y='sales', data=aux1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16195856873822975, 'fraction_numerical': 0.03879472693032015, 'mean_word_length': 7.010558069381599, 'pattern_counts': {'":': 0, '<': 48, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '219798', 'license': 'FTL', 'n_tokens_mistral': 2186, 'n_tokens_neox': 2081, 'n_words': 376, 'path': 'notebooks/m02_v01_store_sales_prediction.ipynb', 'repo_name': 'felipetmota/DsEmProducao', 'size': '5310'}