<jupyter_start><jupyter_text>Problem Statement: The data scientists at BigMart have collected 2013 sales data for 1559 products across 10 stores in different cities. Also, certain attributes of each product and store have been defined. The aim is to build a predictive model and find out the sales of each product at a particular store. Using this model, BigMart will try to understand the properties of products and stores which play a key role in increasing sales.<jupyter_code>import pandas as pd
import numpy as np
import seaborn as sb
import matplotlib.pyplot as plt
import math
train_data='D:/My Personal Documents/Learnings/Data Science/Data Sets/Big Mart Sale/Train.csv'
test_data='D:/My Personal Documents/Learnings/Data Science/Data Sets/Big Mart Sale/Test.csv'
train=pd.read_csv(train_data)
test=pd.read_csv(test_data)
print(train.shape)
print(test.shape)
train['source']='train'
test['source']='test'
data = pd.concat([train, test],ignore_index=True)
print (train.shape, test.shape, data.shape)
data.describe()
train=data
train.info()
train.apply(lambda x: sum(x.isnull()))
train.apply(lambda x: len(x.unique()))
sb.barplot(train.Outlet_Size,train.Item_Outlet_Sales)
Outlet_Type_Sales=train.groupby('Outlet_Type').Item_Outlet_Sales.sum().reset_index()
Outlet_Type_Sales
plt.figure(figsize=(10,6))
sb.barplot(x='Outlet_Type',y='Item_Outlet_Sales', data=Outlet_Type_Sales)
Outlet_Size_Sales=train.groupby('Outlet_Size').Item_Outlet_Sales.sum().reset_index()
Outlet_Size_Sales
sb.barplot(x='Outlet_Size',y='Item_Outlet_Sales', data=Outlet_Size_Sales)
Item_Fat_Content_Sales=train.groupby('Item_Fat_Content').Item_Outlet_Sales.sum().reset_index()
Item_Fat_Content_Sales
sb.barplot(x='Item_Fat_Content', y='Item_Outlet_Sales', data=Item_Fat_Content_Sales)
Outlet_Location_Type_Sales=train.groupby('Outlet_Location_Type').Item_Outlet_Sales.sum().reset_index()
Outlet_Location_Type_Sales
sb.barplot(x='Outlet_Location_Type',y='Item_Outlet_Sales', data=Outlet_Location_Type_Sales)
Item_Type_Sales=train.groupby('Item_Type').Item_Outlet_Sales.sum().reset_index()
Item_Type_Sales
plt.figure(figsize=(24,6))
sb.barplot(x='Item_Type',y='Item_Outlet_Sales',data=Item_Type_Sales)
train['Item_Type_Derived']=train['Item_Identifier'].apply(lambda x: x[0:2])
train['Item_Type_Derived']=train['Item_Type_Derived'].map({'FD':'Food','DR':'Drink','NC':'Non Consumable'})
train.head()
type(train['Item_Identifier'].map(lambda x: x[0:2]))
train['Item_Fat_Content_Derived']=train['Item_Fat_Content'].map({'LF':'Low Fat','low fat':'Low Fat','reg':'Regular','Low Fat':'Low Fat','Regular':'Regular'})
train.head()
train['Store_Age']=train['Outlet_Establishment_Year'].apply(lambda x :2013-x)
train.head()
train.isnull().sum()
train.apply(lambda x: len(x.unique()))
sb.distplot(train.Item_Weight[train.Item_Weight.isnull()==False])
def MRP_decode(mrp):
    if(mrp <=50):
        return 'Low'
    elif(mrp >50 and mrp <100):
        return 'medium'
    elif(mrp >100 and mrp <200):
        return 'high'
    else:
        return 'very high'
train.head()
train['Price_Cat']=train.Item_MRP.apply(MRP_decode)
train['Sales']=math.ceil(train.Item_Outlet_Sales/train.Item_MRP)
train['Item_Weight'][train.Item_Weight.isnull()==True]=train.Item_Weight.mean()
df1=pd.get_dummies(train['Outlet_Type'])
df2=pd.get_dummies(train['Outlet_Size'])
outlet_sixe_mode=train.groupby('Outlet_Type').Outlet_Size.apply(lambda x: x.mode())
missing_bool=train.Outlet_Size.isnull()
train.loc[missing_bool,'Outlet_Size']=train.loc[missing_bool,'Outlet_Size'].apply(lambda x:outlet_sixe_mode[1] )
train.apply(lambda x: sum(x.isnull()))
item_mean_visibility=train.groupby('Item_Identifier').Item_Visibility.mean()
zero_visibility=train.Item_Visibility==0
train.loc[zero_visibility,'Item_Visibility']=train.loc[zero_visibility,'Item_Visibility'].apply(lambda x:item_mean_visibility[1] )
#Import library:
from sklearn.preprocessing import LabelEncoder
le = LabelEncoder()
#New variable for outlet
train['Outlet'] = le.fit_transform(train['Outlet_Identifier'])
var_mod = ['Item_Fat_Content','Outlet_Location_Type','Outlet_Size','Item_Type_Derived','Outlet_Type','Outlet']
le = LabelEncoder()
for i in var_mod:
    train[i] = le.fit_transform(train[i])
train=pd.get_dummies(train,columns=['Price_Cat','Item_Fat_Content_Derived','Item_Type_Derived','Outlet_Location_Type','Outlet_Size','Outlet_Type','Outlet'])
train.drop(['Item_Fat_Content','Item_Type','Outlet_Establishment_Year'],axis=1,inplace=True)
test=train[train.source=='test']
train=train[train.source=='train']

test.drop(['source','Item_Outlet_Sales'], axis=1,inplace=True)
train.drop(['source'], axis=1,inplace=True)
train.head()
#Define target and ID columns:
target = 'Item_Outlet_Sales'
IDcol = ['Item_Identifier','Outlet_Identifier']
from sklearn import model_selection, metrics
def modelfit(alg, dtrain, dtest, predictors, target, IDcol, filename):
    #Fit the algorithm on the data
    alg.fit(dtrain[predictors], dtrain[target])
        
    #Predict training set:
    dtrain_predictions = alg.predict(dtrain[predictors])

    #Perform cross-validation:
    cv_score = model_selection.cross_val_score(alg, dtrain[predictors], dtrain[target], cv=20, scoring='neg_mean_squared_error')
    cv_score = np.sqrt(np.abs(cv_score))
    
    #Print model report:
    print ("\nModel Report")
    print ("RMSE : %.4g" % np.sqrt(metrics.mean_squared_error(dtrain[target].values, dtrain_predictions)))
    print ("CV Score : Mean - %.4g | Std - %.4g | Min - %.4g | Max - %.4g" % (np.mean(cv_score),np.std(cv_score),np.min(cv_score),np.max(cv_score)))
    
    #Predict on testing data:
    dtest[target] = alg.predict(dtest[predictors])
    
    #Export submission file:
    IDcol.append(target)
    submission = pd.DataFrame({ x: dtest[x] for x in IDcol})
    submission.to_csv(filename, index=False)
from sklearn.linear_model import LinearRegression, Ridge, Lasso
predictors = [x for x in train.columns if x not in [target]+IDcol]
# print predictors
alg1 = LinearRegression(normalize=True)
modelfit(alg1, train, test, predictors, target, IDcol, 'alg1.csv')
coef1 = pd.Series(alg1.coef_, predictors).sort_values()
coef1.plot(kind='bar', title='Model Coefficients',figsize=(10,6))
predictors = [x for x in train.columns if x not in [target]+IDcol]
alg2 = Ridge(alpha=0.05,normalize=True)
modelfit(alg2, train, test, predictors, target, IDcol, 'alg2.csv')
coef2 = pd.Series(alg2.coef_, predictors).sort_values()
coef2.plot(kind='bar', title='Model Coefficients')
from sklearn.tree import DecisionTreeRegressor
predictors = [x for x in train.columns if x not in [target]+IDcol]
alg3 = DecisionTreeRegressor(max_depth=15, min_samples_leaf=100)
modelfit(alg3, train, test, predictors, target, IDcol, 'alg3.csv')
coef3 = pd.Series(alg3.feature_importances_, predictors).sort_values(ascending=False)
coef3.plot(kind='bar', title='Feature Importances')
predictors = ['Item_MRP','Outlet_Type_0','Store_Age']
alg4 = DecisionTreeRegressor(max_depth=8, min_samples_leaf=150)
modelfit(alg4, train, test, predictors, target, IDcol, 'alg4.csv')
coef4 = pd.Series(alg4.feature_importances_, predictors).sort_values(ascending=False)
coef4.plot(kind='bar', title='Feature Importances')
from sklearn.ensemble import RandomForestRegressor
predictors = [x for x in train.columns if x not in [target]+IDcol]
alg5 = RandomForestRegressor(n_estimators=200,max_depth=8,max_features=5,min_samples_leaf=20,n_jobs=4)
modelfit(alg5, train, test, predictors, target, IDcol, 'alg5.csv')
coef5 = pd.Series(alg5.feature_importances_, predictors).sort_values(ascending=False)
coef5.plot(kind='bar', title='Feature Importances')
from sklearn.ensemble import RandomForestRegressor
#predictors = ['Item_MRP','Outlet_Type_0','Store_Age','Outlet_Type_3','Outlet_5','Item_Visibility','Item_Weight']
predictors = [x for x in train.columns if x not in [target]+IDcol]
#predictors = ['Item_MRP','Outlet_Type_0','Store_Age','Outlet_Type_3','Outlet_5','Outlet_Type_1','Outlet_4','Outlet_Size_1','Item_Visibility']
alg5 = RandomForestRegressor(n_estimators=200,max_depth=8,max_features=None,min_samples_split=80, min_samples_leaf=50,n_jobs=4,bootstrap=True)
modelfit(alg5, train, test, predictors, target, IDcol, 'alg5.csv')
coef5 = pd.Series(alg5.feature_importances_, predictors).sort_values(ascending=False)
coef5.plot(kind='bar', title='Feature Importances')
from sklearn.ensemble import RandomForestRegressor
predictors = ['Item_MRP','Outlet_Type_0','Store_Age','Outlet_Type_3','Outlet_5']
alg6 = RandomForestRegressor(n_estimators=200,max_depth=6, min_samples_leaf=100,n_jobs=4)
modelfit(alg6, train, test, predictors, target, IDcol, 'alg6.csv')
coef6 = pd.Series(alg6.feature_importances_, predictors).sort_values(ascending=False)
coef6.plot(kind='bar', title='Feature Importances')
from xgboost.sklearn import XGBRegressor
#predictors = ['Item_MRP','Outlet_Type_0','Store_Age','Outlet_Type_3','Outlet_5','Outlet_Type_1','Outlet_4','Outlet_Size_1','Item_Visibility','Item_Weight','Outlet_Location_Type_2','Outlet_3']
predictors = ['Item_MRP','Outlet_Type_0','Store_Age','Outlet_Type_3','Outlet_5','Outlet_Type_1','Outlet_4','Outlet_Size_1','Item_Visibility']
alg7 = XGBRegressor(max_depth=3, learning_rate=0.1, n_estimators=45, reg_alpha=0.1,colsample_bytree=0.9, silent=True, min_child_weight=1,objective='reg:linear', booster='gbtree', n_jobs=1, nthread=None, gamma=0)
modelfit(alg7, train, test, predictors, target, IDcol, 'alg7.csv')
coef7 = pd.Series(alg7.feature_importances_, predictors).sort_values(ascending=False)
coef7.plot(kind='bar', title='Feature Importances')
#Grid seach on subsample and max_features
#Choose all predictors except target & IDcols
from sklearn.grid_search import GridSearchCV
param_test2 = {
    'max_depth':[3,4,5,6],
    'n_estimators':[50,100,200,300,500]
}
gsearch2 = GridSearchCV(estimator = XGBRegressor(max_depth=6, 
                                                 learning_rate=0.1, 
                                                 n_estimators=200, 
                                                 silent=True, 
                                                 objective='reg:linear', 
                                                 booster='gbtree', 
                                                 n_jobs=1, 
                                                 nthread=None, 
                                                 gamma=0, 
                                                 min_child_weight=1, 
                                                 max_delta_step=0, 
                                                 subsample=1, 
                                                 reg_alpha=0,
                       param_grid = param_test2, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch2.fit(train[predictors],train[target])
from sklearn.grid_search import GridSearchCV
param_test2 = {
    'max_depth':[4,5,6],
    'min_child_weight':[4,5,6]
}
gsearch2 = GridSearchCV(estimator = XGBRegressor( max_depth=6, 
                                                 learning_rate=0.1, 
                                                 n_estimators=200, 
                                                 silent=True, 
                                                 objective='reg:linear', 
                                                 booster='gbtree', 
                                                 n_jobs=1, 
                                                 nthread=None, 
                                                 gamma=0, 
                                                 min_child_weight=1, 
                                                 max_delta_step=0, 
                                                 subsample=1, 
                                                 reg_alpha=0,seed=27), 
                       param_grid = param_test2, scoring='roc_auc',n_jobs=4,iid=False, cv=5)
gsearch2.fit(train[predictors],train[target])<jupyter_output>C:\Amrendra\Anaconda\lib\site-packages\sklearn\cross_validation.py:41: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. Also note that the interface of the new CV iterators are different from that of this module. This module will be removed in 0.20.
  "This module will be removed in 0.20.", DeprecationWarning)
C:\Amrendra\Anaconda\lib\site-packages\sklearn\grid_search.py:42: DeprecationWarning: This module was deprecated in version 0.18 in favor of the model_selection module into which all the refactored classes and functions are moved. This module will be removed in 0.20.
  DeprecationWarning)
{'fraction_non_alphanumeric': 0.12731022447846435, 'fraction_numerical': 0.020623463155389862, 'mean_word_length': 4.169331693316933, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '196530', 'license': 'FTL', 'n_tokens_mistral': 4206, 'n_tokens_neox': 3975, 'n_words': 723, 'path': 'notebooks/Big Mart Sell Prediction.ipynb', 'repo_name': 'amrendra1002/ML-DL', 'size': '12607'}