<jupyter_start><jupyter_text>0.0. IMPORTS<jupyter_code>import pandas as pd
import inflection
import math
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt<jupyter_output><empty_output><jupyter_text>0.1. Helper Functions  0.2. Loading Data<jupyter_code>df_sales_raw = pd.read_csv( 'data/train.csv', low_memory=False)
df_store_raw = pd.read_csv( 'data/store.csv', low_memory=False )

#merge
df_raw = pd.merge(df_sales_raw, df_store_raw, how='left', on='Store')
df_raw.sample()<jupyter_output><empty_output><jupyter_text>1.0. DESCRICAO DOS DADOS<jupyter_code>df1 = df_raw.copy()<jupyter_output><empty_output><jupyter_text>1.1. Rename Columns<jupyter_code>cols_old = ['Store', 'DayOfWeek', 'Date', 'Sales', 'Customers', 'Open', 'Promo','StateHoliday', 'SchoolHoliday', 
            'StoreType', 'Assortment','CompetitionDistance', 'CompetitionOpenSinceMonth','CompetitionOpenSinceYear', 
            'Promo2', 'Promo2SinceWeek','Promo2SinceYear', 'PromoInterval']

snakecase = lambda x: inflection.underscore( x )

cols_new = list(map(snakecase, cols_old))

#rename
df1.columns = cols_new<jupyter_output><empty_output><jupyter_text>1.2. Data Dimensions<jupyter_code>print( 'Number of Rowns: {}'.format(df1.shape[0]))
print( 'Number of Columns: {}'.format(df1.shape[1]))<jupyter_output>Number of Rowns: 1017209
Number of Columns: 18<jupyter_text>1.3. Data Types<jupyter_code>df1['date'] = pd.to_datetime(df1['date'])
df1.dtypes<jupyter_output><empty_output><jupyter_text>1.4. Check NA<jupyter_code>df1.isna().sum()<jupyter_output><empty_output><jupyter_text>1.5. Fillout NA<jupyter_code>df1['competition_distance'].max()
#competition_distance    
df1['competition_distance'] = df1['competition_distance'].apply(lambda x:200000.0 if math.isnan( x ) else x)

#competition_open_since_month  
df1['competition_open_since_month'] = df1.apply(lambda x: x['date'].month if math.isnan(x['competition_open_since_month'])
                                                else x['competition_open_since_month'], axis=1)
#competition_open_since_year 
df1['competition_open_since_year'] = df1.apply(lambda x: x['date'].year if math.isnan(x['competition_open_since_year'])
                                                else x['competition_open_since_year'], axis=1)                              
#promo2_since_week    
df1['promo2_since_week'] = df1.apply(lambda x: x['date'].week if math.isnan(x['promo2_since_week'])
                                                else x['promo2_since_week'], axis=1)
#promo2_since_year 
df1['promo2_since_year'] = df1.apply(lambda x: x['date'].year if math.isnan(x['promo2_since_year'])
                                                else x['promo2_since_year'], axis=1)
#promo_interval 
month_map = {1: 'Jan', 2: 'Feb', 3: 'Mar', 4: 'Apr', 5: 'May', 6: 'Jun', 7: 'Jul', 8: 'Aug', 9: 'Sep', 10: 'Oct', 11: 'Nov',
             12: 'Dec'}

df1['promo_interval'].fillna(0, inplace=True )

df1['month_map'] = df1['date'].dt.month.map(month_map)

df1['is_promo'] = df1[['promo_interval', 'month_map']].apply(lambda x: 0 if x['promo_interval'] == 0 else 1 if x['month_map'] 
                                                           in x['promo_interval'].split(',') else 0, axis=1)<jupyter_output><empty_output><jupyter_text>1.6. Change Types<jupyter_code>df1['competition_open_since_month'] = df1['competition_open_since_month'].astype(np.int64)
df1['competition_open_since_year'] = df1['competition_open_since_year'].astype(np.int64)
df1['promo2_since_week'] = df1['promo2_since_week'].astype(np.int64)
df1['promo2_since_year'] = df1['promo2_since_year'].astype(np.int64)<jupyter_output><empty_output><jupyter_text>1.7. Descriptive Statistical<jupyter_code>num_attributes = df1.select_dtypes(include=['int64', 'float64'])
cat_attributes = df1.select_dtypes(exclude=['int64', 'float64', 'datetime64[ns]'])<jupyter_output><empty_output><jupyter_text>1.7.1. Numerical Attributes<jupyter_code># Central Tendency - mean, median
ct1 = pd.DataFrame(num_attributes.apply(np.mean)).T
ct2 = pd.DataFrame(num_attributes.apply(np.median)).T

# Dispersion -std, min, max, range, skew, kurtosis
d1 = pd.DataFrame(num_attributes.apply(np.std)).T
d2 = pd.DataFrame(num_attributes.apply(min)).T
d3 = pd.DataFrame(num_attributes.apply(max)).T
d4 = pd.DataFrame(num_attributes.apply(lambda x: x.max() - x.min())).T
d5 = pd.DataFrame(num_attributes.apply(lambda x: x.skew())).T
d6 = pd.DataFrame(num_attributes.apply(lambda x: x.kurtosis())).T

#Concatenate
m = pd.concat([d2, d3, d4, ct1, ct2, d1,d5, d6 ]).T.reset_index()
m.columns = ['attributes', 'min', 'max', 'range','mean', 'median', 'std', 'skew', 'kurtosis' ]
m
sns.distplot(df1['competition_distance'])<jupyter_output>C:\Users\a00203953\Anaconda3\envs\Project_Rossmann\lib\site-packages\seaborn\distributions.py:2557: FutureWarning: `distplot` is a deprecated function and will be removed in a future version. Please adapt your code to use either `displot` (a figure-level function with similar flexibility) or `histplot` (an axes-level function for histograms).
  warnings.warn(msg, FutureWarning)<jupyter_text>1.7.2. Categorical Attributes<jupyter_code>cat_attributes.apply(lambda x: x.unique().shape[0])
aux1 = df1[(df1['state_holiday'] != '0') & (df1['sales']>0)]

plt.subplot(1,3,1)
sns.boxplot(x='state_holiday', y='sales',data=aux1)

plt.subplot(1,3,2)
sns.boxplot(x='store_type', y='sales',data=aux1)

plt.subplot(1,3,3)
sns.boxplot(x='assortment', y='sales',data=aux1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15962099125364432, 'fraction_numerical': 0.03407434402332361, 'mean_word_length': 5.382558139534884, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '422782', 'license': 'MIT', 'n_tokens_mistral': 2148, 'n_tokens_neox': 2060, 'n_words': 321, 'path': 'm02_v01_store_prediction.ipynb', 'repo_name': 'diegohbrs/Rossmann_Store_Sales', 'size': '5488'}