<jupyter_start><jupyter_text>0.0 Imports<jupyter_code>import math
import numpy  as np
import pandas as pd
import inflection

import seaborn as sns

from matplotlib import pyplot as plt
from IPython.core.display import HTML<jupyter_output><empty_output><jupyter_text>0.1 Helper functions<jupyter_code>def jupyter_settings():
    %matplotlib inline
    %pylab inline
    
    plt.style.use( 'bmh' )
    plt.rcParams['figure.figsize'] = [25, 12]
    plt.rcParams['font.size'] = 24
    
    display( HTML( '<style>.container { width:100% !important; }</style>') )
    pd.options.display.max_columns = None
    pd.options.display.max_rows = None
    pd.set_option( 'display.expand_frame_repr', False )
    
    sns.set()
jupyter_settings()<jupyter_output>Populating the interactive namespace from numpy and matplotlib<jupyter_text>0.2 Loading data<jupyter_code>df_sales_raw = pd.read_csv('train.csv', low_memory=False)
df_store_raw = pd.read_csv('store.csv', low_memory=False)
# merge
df_raw = pd.merge(df_sales_raw,df_store_raw, how ='left', on='Store')<jupyter_output><empty_output><jupyter_text>1.0 Descrição dos dados<jupyter_code>df_raw.columns<jupyter_output><empty_output><jupyter_text>1.1 Rename columns<jupyter_code>df1 = df_raw.copy()
cols_old = ['Store', 'DayOfWeek', 'Date', 'Sales', 'Customers', 'Open', 'Promo',
           'StateHoliday', 'SchoolHoliday', 'StoreType', 'Assortment',
           'CompetitionDistance', 'CompetitionOpenSinceMonth',
           'CompetitionOpenSinceYear', 'Promo2', 'Promo2SinceWeek',
           'Promo2SinceYear', 'PromoInterval']

snakecase = lambda x: inflection.underscore (x)

cols_new = list (map (snakecase, cols_old))
df1.columns = cols_new<jupyter_output><empty_output><jupyter_text>1.2 Data dimensions<jupyter_code>print ('Number of rows: {}'.format(df1.shape[0]))
print ('Number of columns: {}'.format(df1.shape[1]))<jupyter_output>Number of rows: 1017209
Number of columns: 18<jupyter_text>1.3 Data types<jupyter_code>df1.dtypes
# Convertendo coluna 'date' para tipo 'datetime'
df1['date'] = pd.to_datetime( df1['date'] )<jupyter_output><empty_output><jupyter_text>1.4 Check NA's<jupyter_code>df1.isna().sum()<jupyter_output><empty_output><jupyter_text>1.5 Fillout NA<jupyter_code>df1['competition_distance'].max()
#competition_distance 
df1['competition_distance'] = df1['competition_distance'].apply( lambda x: 200000.0 if math.isnan( x ) else x )

#competition_open_since_month  
df1['competition_open_since_month'] = df1.apply( lambda x: x['date'].month if math.isnan( x['competition_open_since_month'] ) else x['competition_open_since_month'], axis=1 )

#competition_open_since_year  
df1['competition_open_since_year'] = df1.apply( lambda x: x['date'].year if math.isnan( x['competition_open_since_year'] ) else x['competition_open_since_year'], axis=1 )

#promo2_since_week  
df1['promo2_since_week'] = df1.apply( lambda x: x['date'].week if math.isnan( x['promo2_since_week'] ) else x['promo2_since_week'], axis=1 )

#promo2_since_year  
df1['promo2_since_year'] = df1.apply( lambda x: x['date'].year if math.isnan( x['promo2_since_year'] ) else x['promo2_since_year'], axis=1 )
#promo_interval              
month_map = {1: 'Jan',  2: 'Fev',  3: 'Mar',  4: 'Apr',  5: 'May',  6: 'Jun',  7: 'Jul',  8: 'Aug',  9: 'Sep',  10: 'Oct', 11: 'Nov', 12: 'Dec'}

df1['promo_interval'].fillna(0, inplace=True )

df1['month_map'] = df1['date'].dt.month.map( month_map )

df1['is_promo'] = df1[['promo_interval', 'month_map']].apply( lambda x: 0 if x['promo_interval'] == 0 else 1 if x['month_map'] in x['promo_interval'].split( ',' ) else 0, axis=1 )
df1.sample(5).T
df1.isna().sum()<jupyter_output><empty_output><jupyter_text>1.6 Change types<jupyter_code>df1.dtypes
# competiton
df1['competition_open_since_month'] = df1['competition_open_since_month'].astype( int )
df1['competition_open_since_year'] = df1['competition_open_since_year'].astype( int )
    
# promo2
df1['promo2_since_week'] = df1['promo2_since_week'].astype( int )
df1['promo2_since_year'] = df1['promo2_since_year'].astype( int )<jupyter_output><empty_output><jupyter_text>1.7 Estatística descritiva dos dados<jupyter_code># Separação de variáveis numéricas e categóricas
num_attributes = df1.select_dtypes( include=['int64', 'float64'] )
cat_attributes = df1.select_dtypes( exclude=['int64', 'float64', 'datetime64[ns]'] )
num_attributes.sample(5)<jupyter_output><empty_output><jupyter_text>1.7.1 Variáveis numéricas<jupyter_code># Tendência central - média, mediana 
ct1 = pd.DataFrame(num_attributes.apply(np.std)).T 
ct2 =  pd.DataFrame(num_attributes.apply(np.median)).T 
# Dispersion - std, min, max, range, skew, kurtosis
d1 = pd.DataFrame(num_attributes.apply(np.std)).T 
d2 = pd.DataFrame(num_attributes.apply(min)).T 
d3 = pd.DataFrame(num_attributes.apply(max)).T
d4 = pd.DataFrame( num_attributes.apply( lambda x: x.max() - x.min() ) ).T 
d5 = pd.DataFrame( num_attributes.apply( lambda x: x.skew() ) ).T 
d6 = pd.DataFrame( num_attributes.apply( lambda x: x.kurtosis() ) ).T 
# Concatenar
df = pd.concat( [d2, d3, d4, ct1, ct2,d1, d5, d6] ).T.reset_index()
df
# Muda nome colunas
df.columns = ['attributes', 'min', 'max', 'range', 'mean', 'median', 'std', 'skew', 'kurtosis']
df
# Plota a distribuição da variável competition_distance
sns.distplot( df1['competition_distance'])<jupyter_output><empty_output><jupyter_text>1.7.2 Variáveis categóricas<jupyter_code># Seleciona os valores por variável categórica
cat_attributes.apply(lambda x: x.unique().shape[0])
# Gráfico boxplot
aux = df1[(df1['state_holiday'] != '0') & (df1['sales'] > 0)]

plt.subplot( 1, 3, 1 )
sns.boxplot( x='state_holiday', y='sales', data=aux )

plt.subplot( 1, 3, 2 )
sns.boxplot( x='store_type', y='sales', data=aux )

plt.subplot( 1, 3, 3 )
sns.boxplot( x='assortment', y='sales', data=aux )<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15831903945111492, 'fraction_numerical': 0.030360205831903946, 'mean_word_length': 6.334591194968554, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 54, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '334817', 'license': 'MIT', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2279, 'n_words': 416, 'path': 'm02_video1_store_sales_prediction.ipynb', 'repo_name': 'luanipiva/ds_producao_mar21', 'size': '5830'}