<jupyter_start><jupyter_text>Data Preparation of Macro_economic data<jupyter_code>macro_economic.insert(loc=0,column='Year',value=macro_economic['Year-Month'].str.split(expand=True)[0].astype('int64'))
macro_economic.insert(loc=1,column='Month',value=macro_economic['Year-Month'].str.split(expand=True)[2])
macro_economic.drop('Year-Month',axis=1,inplace=True)
macro_economic.drop('PartyInPower',axis=1,inplace=True)
macro_economic.drop('AdvertisingExpenses (in Thousand Dollars)',axis=1, inplace=True)
macro_economic
macro_economic['Month'].unique()
macro_economic['Month'].replace({'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8, 'Sep':9,
       'Oct':10, 'Nov':11, 'Dec':12},inplace=True)
# 5 years
economic09_13 = macro_economic[macro_economic['Year'].between(2009,2013)]
print(economic09_13.head(2)['Year'])
print(economic09_13.tail(2)['Year'])


# 1 year 2014 (Output)
economic14 = macro_economic[macro_economic['Year']==2014]<jupyter_output>0    2009
1    2009
Name: Year, dtype: int64
58    2013
59    2013
Name: Year, dtype: int64<jupyter_text>Data Preparation for weather patterns dataset<jupyter_code>weather09['Month'].replace({'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8, 'Sep':9,
       'Oct':10, 'Nov':11, 'Dec':12}, inplace=True)
temp = weather09.groupby(['Year','Month']).median().reset_index()
we09 = pd.merge(left=temp,right=weather09.groupby(['Month']).count()['WeatherEvent'].reset_index(), how='inner',on='Month')
for i in weather12.columns:
    weather12.loc[:,i].replace({'-':-1},inplace=True)
for i in weather13.columns:
    weather13.loc[:,i].replace({'-':-1},inplace=True)
for i in weather14.columns:
    weather14.loc[:,i].replace({'-':-1},inplace=True)
for i in weather14.columns:
    weather14.loc[:,i].replace({'-':-1},inplace=True)
weather09.iloc[:,-2].replace({'T':np.nan},inplace=True)
weather09.iloc[:,-2].fillna(weather09.iloc[:,-2].mean(),inplace=True)

weather10.iloc[:,-2].replace({'T':np.nan},inplace=True)
weather10.iloc[:,-2].fillna(weather10.iloc[:,-2].mean(),inplace=True)

weather11.iloc[:,-2].replace({'T':np.nan},inplace=True)
weather11.iloc[:,-2].fillna(weather11.iloc[:,-2].mean(),inplace=True)

weather12.iloc[:,-2].replace({'T':np.nan},inplace=True)
weather12.iloc[:,-2].fillna(weather12.iloc[:,-2].mean(),inplace=True)

weather13.iloc[:,-2].replace({'T':np.nan},inplace=True)
weather13.iloc[:,-2].fillna(weather13.iloc[:,-2].mean(),inplace=True)

weather14.iloc[:,-2].replace({'T':np.nan},inplace=True)
weather14.iloc[:,-2].fillna(weather14.iloc[:,-2].mean(),inplace=True)
weather14['Month'].replace({'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8, 'Sep':9,
       'Oct':10, 'Nov':11, 'Dec':12}, inplace=True)
yt= 2010
for number in range(10,15):
    vars()['weather'+str(number)]['Year'] = yt
    yt += 1
    vars()['weather'+str(number)]['Month'].replace({'Jan':1, 'Feb':2, 'Mar':3, 'Apr':4, 'May':5, 'Jun':6, 'Jul':7, 'Aug':8, 'Sep':9,
       'Oct':10, 'Nov':11, 'Dec':12}, inplace=True)
    temp = vars()['weather'+str(number)].groupby(['Year','Month']).median().reset_index()
    vars()['we'+str(number)] = pd.merge(left=temp,right=vars()['weather'+str(number)].groupby(['Month']).count()['WeatherEvent'].reset_index(), how='inner',on='Month')
we09.shape,we10.shape,we11.shape,we12.shape,we13.shape,
we14.iloc[:,[0,1,2,4,7,10,16,-1]].columns
#Using these columns only
w09 = we09.iloc[:,[0,1,2,4,7,10,16,-1]]
for i in range(10,14):
    vars()['w'+str(i)] = vars()['we'+str(i)].iloc[:,[0,1,2,4,7,10,16,-1]]
weather5yrs = w09.append([w10,w11,w12,w13],ignore_index=True)
weather5yrs.head()
weather5yrs.shape
weather1yr = we14.iloc[:,[0,1,2,4,7,10,16,-1]]
weather1yr.shape<jupyter_output><empty_output><jupyter_text>Data Preparation of holidays data<jupyter_code>holidays
holidays['Month'] = holidays['MonthDate'].dt.month
holidays
holidays['Event'].unique()
temp = holidays.groupby(['Year','Month']).count()['Event'].reset_index()
temp914 = temp[temp['Year'].between(2009,2014)]
temp914.head(2)
len(temp914)
#Encoding missing Months holidays
for i in range(2009,2015):
    temp914=temp914.append(other={'Year':i, 'Month':3,'Event': 0},ignore_index=True)
    temp914=temp914.append(other={'Year':i, 'Month':8,'Event': 0},ignore_index=True)
temp914.sort_values(by=['Year','Month'],ignore_index=True,inplace=True)
len(temp914)
holiday9_13 = temp914[temp914['Year'].between(2009,2013)]
holiday14 = temp914[temp914['Year']==2014]
holiday9_13.shape, holiday14.shape<jupyter_output><empty_output><jupyter_text>Merging of DATAFRAMES<jupyter_code>#Matching Granularity
sales.shape
sales.head(2)
train = pd.read_csv("submission.csv")
train.shape
#train['ProductCategory'].replace({'MenClothing':1,'OtherClothing':2,'WomenClothing':3},inplace=True)
test = train.drop(['Unnamed: 3','Sales(In ThousandDollars)'],axis=1)
test.head()
holiday14.shape,weather1yr.shape,economic14.shape
#YEAR 14

#holidays and sales
merge14_1 = pd.merge(left=test,right=holiday14,how='inner',left_on=['Year','Month'],right_on=['Year','Month'])

#holiday,sales and Weather
merge14_2 = pd.merge(left=merge14_1, right=weather1yr, how='inner')

#holiday,sales, Weather and macroeconomic
merge14_3 = pd.merge(left=merge14_2, right=economic14, how='inner')
sales_data_14 = merge14_3.copy()
sales_data_14['Year'].replace({2014:6},inplace=True)
#s14 = pd.get_dummies(data=sales_data_14, columns=['ProductCategory'])
holiday9_13.shape,weather5yrs.shape,economic09_13.shape
sales_data_14.shape
#YEAR 09 - 13


#holidays and sales
merge1 = pd.merge(left=sales,right=holiday9_13,how='inner',left_on=['Year','Month'],right_on=['Year','Month'])

#holiday,sales and Weather
merge2 = pd.merge(left=merge1, right=weather5yrs, how='inner')

#holiday,sales, Weather and macroeconomic
merge3 = pd.merge(left=merge2, right=economic09_13, how='inner')
sales_data_09_13 = merge3.copy()
sales_data_09_13.isnull().sum()
#Dropping nan Values---Nan in sales (Output feature itself)
sales_data_09_13.dropna(inplace=True)
sales_data_09_13.columns
sales_data_09_13['Year'].replace({2009:1,2010:2,2011:3, 2012:4, 2013:5},inplace=True)
sales['ProductCategory'].unique()<jupyter_output><empty_output><jupyter_text>MODEL II ---> Creating separate Models for Product Categories<jupyter_code>#5yrs data
sales_913_Women = sales_data_09_13[sales_data_09_13['ProductCategory']=='WomenClothing']
sales_913_Men = sales_data_09_13[sales_data_09_13['ProductCategory']=='MenClothing']
sales_913_Other = sales_data_09_13[sales_data_09_13['ProductCategory']=='OtherClothing']

#1yrs data
sales_14_Women = sales_data_14[sales_data_14['ProductCategory']=='WomenClothing']
sales_14_Men = sales_data_14[sales_data_14['ProductCategory']=='MenClothing']
sales_14_Other = sales_data_14[sales_data_14['ProductCategory']=='OtherClothing']
#The column dimensionality must be same
print(sales_913_Men.shape,sales_913_Other.shape,sales_913_Women.shape)
print(sales_14_Men.shape,sales_14_Other.shape,sales_14_Women.shape)
#5yr ----
Xmen = sales_913_Men.drop(['Sales(In ThousandDollars)','ProductCategory','Day'],axis=1)
ymen = sales_913_Men['Sales(In ThousandDollars)']

Xwomen = sales_913_Women.drop(['Sales(In ThousandDollars)','ProductCategory','Day'],axis=1)
ywomen = sales_913_Women['Sales(In ThousandDollars)']

Xother = sales_913_Other.drop(['Sales(In ThousandDollars)','ProductCategory','Day'],axis=1)
yother = sales_913_Other['Sales(In ThousandDollars)']
#men
xtrain_men, xtest_men, ytrain_men, ytest_men = train_test_split(Xmen,ymen, test_size=0.3, random_state=50)
#women
xtrain_women, xtest_women, ytrain_women, ytest_women = train_test_split(Xwomen,ywomen, test_size=0.3, random_state=50)
#other
xtrain_other, xtest_other, ytrain_other, ytest_other = train_test_split(Xother,yother, test_size=0.3, random_state=50)
GradientBoostingRegressor()
GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 66, 
                                 learning_rate=0.5099999999999998,subsample= 0.9,random_state=208)
kf = KFold(n_splits=2,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=GBoost, X=xtrain_men, y=ytrain_men, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=GBoost, X=xtrain_men, y=ytrain_men, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=GBoost, X=xtest_men, y=ytest_men, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=GBoost, X=xtest_men, y=ytest_men, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
#This LAbel encoding and Labelling ProductCategory as Ordinal won't help.
#sales_data_09_13['ProductCategory'].replace({'MenClothing':1,'OtherClothing':2,'WomenClothing':3},inplace=True)
#s913 = pd.get_dummies(data=sales_data_09_13, columns=['ProductCategory'])
plt.figure(figsize=(20,10))
sns.heatmap(s913.corr(),annot=True,vmin=-1,vmax=1, cmap='Dark2')
s913.to_csv("s913.csv",index=True)
s14.to_csv('s14.csv',index=True)
X = s913.drop('Sales(In ThousandDollars)',axis=1)
y = s913['Sales(In ThousandDollars)']
#X = sales_data_09_13.drop('Sales(In ThousandDollars)',axis=1)
#y = sales_data_09_13['Sales(In ThousandDollars)']
xtrain, xtest, ytrain, ytest = train_test_split(X,y, test_size=0.20)
xsc = sm.add_constant(X)
sm_model = sm.OLS(y, xsc).fit()
sm_model.summary()
lr = RandomForestRegressor()
lr_model = lr.fit(xtrain,ytrain)
print(lr_model.score(xtrain,ytrain))

ypred_test = lr_model.predict(xtest)
np.sqrt(mean_squared_error(ytest,ypred_test))
lr_model.predict(s14)
# pca.explained_variance_ratio_
# pca =PCA(8)
# xtrain_scpca = pca.fit_transform(xtrain)
# xtest_scpca  = pca.transform(xtest)
# s14_scpa = pca.transform(s14)
# pca.fit(xtrain,ytrain)

# xtrain, xtest, ytrain, ytest = train_test_split(X,y, test_size=0.2)
# GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 66, 
#                                  learning_rate=0.5099999999999998,subsample= 0.9)


# GB_model_Tunned_v5 = GBoost.fit(xtrain_scpca, ytrain)
# kf = KFold(n_splits=10,shuffle=True)
# trainRMSEscore = cross_val_score(estimator=GBoost, X=xtrain_scpca, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
# trainr2score = cross_val_score(estimator=GBoost, X=xtrain_scpca, y=ytrain, scoring='r2', cv=kf)

# print("Average Train R2   ",np.mean(trainr2score))
# print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
# print("-"*50)
# testRMSEscore = cross_val_score(estimator=GBoost, X=xtest_scpca, y=ytest, scoring='neg_mean_squared_error', cv=kf)
# testr2score = cross_val_score(estimator=GBoost, X=xtest_scpca, y=ytest, scoring='r2', cv=kf)

# print("Average Test R2   ",np.mean(testr2score))
# print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
# GB_model_Tunned_v5.predict(s14_scpa)
xte = xtrain.iloc[:,[21,19,9,10,11,20,5,4,15,12,13,6,16,23,18,14,7,22,17]]
vif = pd.DataFrame()
vif['Features'] = xte.columns
vif['VIF'] = [round(variance_inflation_factor(xte.values,i)) for i in range(xte.shape[1])]
vif.sort_values(by='VIF',ascending=False)
xtc = xte.drop(xte.iloc[:,[3,4,8,0,10,2,15,5,17,14,]].columns,axis=1)
vif = pd.DataFrame()
vif['Features'] = xtc.columns
vif['VIF'] = [round(variance_inflation_factor(xtc.values,i)) for i in range(xtc.shape[1])]
vif.sort_values(by='VIF',ascending=False)
lr = LinearRegression()
lr_model = lr.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=lr, X=xtrain, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=lr, X=xtrain, y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=lr, X=xtest, y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=lr, X=xtest, y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
#DATA IS NOT SCALED !


# sg = SGDRegressor()
# sg_model = lr.fit(xtrain,ytrain)

# kf = KFold(n_splits=10,shuffle=True,random_state=10)
# trainRMSEscore = cross_val_score(estimator=sg, X=xtrain, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
# trainr2score = cross_val_score(estimator=sg, X=xtrain, y=ytrain, scoring='r2', cv=kf)

# print("Average Train R2   ",np.mean(trainr2score))
# print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
# print("-"*50)
# testRMSEscore = cross_val_score(estimator=sg, X=xtest, y=ytest, scoring='neg_mean_squared_error', cv=kf)
# testr2score = cross_val_score(estimator=sg, X=xtest, y=ytest, scoring='r2', cv=kf)

# print("Average Test R2   ",np.mean(testr2score))
# print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
gb = GradientBoostingRegressor(n_estimators=300,random_state=10)
gb_model = gb.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=gb, X=xtrain, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=gb, X=xtrain, y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=gb, X=xtest, y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=gb, X=xtest, y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = gb_model.predict(s14)
te.to_csv("basic_model_3.csv",index=False)
lr = XGBRFRegressor(n_estimators=30,learning_rate=0.99)
lr_model = lr.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=lr, X=xtrain, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=lr, X=xtrain, y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=lr, X=xtest, y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=lr, X=xtest, y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
rf = RandomForestRegressor(n_estimators=300)
rf_model = rf.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=rf, X=xtrain, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=rf, X=xtrain, y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=rf, X=xtest, y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=rf, X=xtest, y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
sklearn.metrics.SCORERS.keys()<jupyter_output><empty_output><jupyter_text>Feature Selection<jupyter_code>len(X.columns)
features_set = X.iloc[:,[1,2,3,4,8,11,12,24,25,26]].columns
index = pd.DataFrame()
index['Features']=X.columns
index
manual_features_1 = X.iloc[:,[0,1,2,3,4,7,12,15,24,25,26]].columns
print(manual_features_1)
sns.scatterplot(s913['Finance Rate on Personal Loans at Commercial Banks, 24 Month Loan'],s913['Sales(In ThousandDollars)'])
manual_features = X.iloc[:,[0,1,2,3,4,5,6,7,8,11,12,13,15,19,21,22,23,24,25,26]].columns
print(manual_features)
# Manual Feature selection
#manual_features = ['Year', 'Month', 'Event', 'Day', 'Temp avg (°C)', 'Dew Point avg (°C)',
       'Humidity (%) avg', 'Visibility (km) avg', 'WeatherEvent','CPI', 'unemployment rate',
       'CommercialBankInterestRateonCreditCardPlans',
       'Earnings or wages  in dollars per hour',
       'ProductCategory_MenClothing', 'ProductCategory_OtherClothing',
       'ProductCategory_WomenClothing']
sfs = SequentialFeatureSelector(estimator=LinearRegression(),k_features='best',forward=True, scoring='neg_mean_squared_error',cv=kf,verbose=True)
sfs_select_lr = sfs.fit(X,y)
sfs_forward_features = list(sfs_select_lr.k_feature_names_)
sfs_forward_features
# sfs = SequentialFeatureSelector(estimator=GradientBoostingRegressor(),k_features='best',forward=True, scoring='neg_mean_squared_error',cv=kf,verbose=True)
# sfs_select_f_gb = sfs.fit(X,y)
# sfs_forward_features_gb = list(sfs_select_f_gb.k_feature_names_)
# sfs_forward_features_gb

#Without tuned Estimator
#BAckward features
#['Year', 'Month', 'Temp avg (°C)', 'Monthly Real GDP Index (inMillion$)', 'unemployment rate', 'Average upland planted(million acres)', 'Average upland harvested(million acres)', 'Production (in  480-lb netweright in million bales)', 'Mill use  (in  480-lb netweright in million bales)', 'Exports', 'ProductCategory_OtherClothing', 'ProductCategory_WomenClothing']
--------------------------------------------------


#Forward Features
#['Year', 'Month', 'Temp avg (°C)', 'Monthly Real GDP Index (inMillion$)', 'unemployment rate', 'Average upland harvested(million acres)', 'ProductCategory_MenClothing', 'ProductCategory_OtherClothing', 'ProductCategory_WomenClothing']
sfs = SequentialFeatureSelector(estimator=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 200, 
                                 learning_rate=0.19,random_state=208),k_features='best',forward=True, scoring='neg_mean_squared_error',cv=kf,verbose=True)
sfs_select_f_gb = sfs.fit(X,y)
sfs_forward_features_gb = list(sfs_select_f_gb.k_feature_names_)
sfs_forward_features_gb
#['Year', 'Month', 'Temp avg (°C)', 'Monthly Real GDP Index (inMillion$)', 'unemployment rate', 'Average upland harvested(million acres)', 'ProductCategory_MenClothing', 'ProductCategory_OtherClothing', 'ProductCategory_WomenClothing']
sfs = SequentialFeatureSelector(estimator=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 200, 
                                 learning_rate=0.19,random_state=208),k_features='best',forward=False, scoring='neg_mean_squared_error',cv=kf,verbose=True)
sfs_select_b_gb = sfs.fit(X,y)
sfs_backward_features_gb = list(sfs_select_b_gb.k_feature_names_)
sfs_backward_features_gb
rfe_gb = RFECV(estimator=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 200, 
                                 learning_rate=0.19,random_state=208) ,cv= kf, scoring='neg_mean_squared_error')
refe_gb_select = rfe_lr.fit(X,y)
rfegb=pd.DataFrame()
rfegb['Features'] = X.columns
rfegb['Rank'] = refe_gb_select.ranking_
rfe_gb_features = list(rfegb[rfegb['Rank']==1]['Features'])
rfe_gb_features
sfs = SequentialFeatureSelector(estimator=LinearRegression(),k_features='best',forward=False, scoring='neg_mean_squared_error',cv=kf,verbose=True)
sfs_select_b_lr = sfs.fit(X,y)
sfs_backward_features = list(sfs_select_b_lr.k_feature_names_)
sfs_backward_features
rfe_lr = RFECV(estimator=LinearRegression() ,cv= kf, scoring='neg_mean_squared_error')
refe_lr_select = rfe_lr.fit(X,y)
rfe=pd.DataFrame()
rfe['Features'] = X.columns
rfe['Rank'] = refe_lr_select.ranking_
rfe_lr_features = list(rfe[rfe['Rank']==1]['Features'])
rfe_lr_features
lr = LinearRegression()
lr_model = lr.fit(xtrain[sfs_forward_features],ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=lr, X=xtrain[sfs_forward_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=lr, X=xtrain[sfs_forward_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=lr, X=xtest[sfs_forward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=lr, X=xtest[sfs_forward_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
dt = DecisionTreeRegressor(max_depth=12)
dt_model = dt.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=dt, X=xtrain[rfe_lr_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=dt, X=xtrain[rfe_lr_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=dt, X=xtest[rfe_lr_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=dt, X=xtest[rfe_lr_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = dt_model.predict(s14)
te.to_csv("basic_model_4.csv",index=False)
# Extremely high variation in Model...not recommended


dt = DecisionTreeRegressor(max_depth=12,random_state=None)
#dt_model = dt.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=dt, X=xtrain[sfs_forward_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=dt, X=xtrain[sfs_forward_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=dt, X=xtest[sfs_forward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=dt, X=xtest[sfs_forward_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
# Good for tunning

rf = RandomForestRegressor(n_estimators=300)
rf_model = rf.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=rf, X=xtrain[sfs_forward_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=rf, X=xtrain[sfs_forward_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=rf, X=xtest[sfs_forward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=rf, X=xtest[sfs_forward_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
#GoOD  Model further for tunning

gb = GradientBoostingRegressor(n_estimators=300,random_state=10)
gb_model = gb.fit(xtrain[rfe_lr_features],ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features_gb], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features_gb], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=gb, X=xtest[sfs_backward_features_gb], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=gb, X=xtest[sfs_backward_features_gb], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
gb_model.predict(s14[rfe_lr_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = gb_model.predict(s14[rfe_lr_features])
te.to_csv("basic_model_7.csv",index=False)
#Good model ----Further for tunning


gb = GradientBoostingRegressor(n_estimators=300,random_state=10)
gb_model = gb.fit(xtrain[sfs_backward_features],ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=gb, X=xtest[sfs_backward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=gb, X=xtest[sfs_backward_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
gb_model.predict(s14[sfs_backward_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = gb_model.predict(s14[sfs_backward_features])
te.to_csv("basic_model_5.csv",index=False)


#KAGGLE
#just now
#1 seconds
#1 seconds
#295.1187
gb = GradientBoostingRegressor(n_estimators=300,random_state=10)
gb_model = gb.fit(xtrain[manual_features],ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=gb, X=xtrain[manual_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=gb, X=xtrain[manual_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=gb, X=xtest[manual_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=gb, X=xtest[manual_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
gb_model.predict(s14[manual_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = gb_model.predict(s14[manual_features])
te.to_csv("basic_model_6.csv",index=False)

#KAGGLE
#297.32912
gb = GradientBoostingRegressor(n_estimators=30,learning_rate=0.19,min_samples_leaf=1)
gb_model = gb.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=gb, X=xtrain, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=gb, X=xtrain, y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=gb, X=xtest, y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=gb, X=xtest, y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
#br = BaggingRegressor(base_estimator=GradientBoostingRegressor(n_estimators=30,random_state=10,learning_rate=0.19,min_samples_leaf=1),n_estimators=30)
gb=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 200, 
                                 learning_rate=0.19,random_state=208)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=gb, X=xtest[sfs_backward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=gb, X=xtest[sfs_backward_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
np.sqrt(abs(testRMSEscore))<jupyter_output><empty_output><jupyter_text>Grid Search CV  Random Forest<jupyter_code>param_grid = {'criterion':['mse'],'max_depth':[2,4,6,8,10,12,13,14,15],'min_samples_split':[2,3,4,5,6,7],
             'n_estimators':[50,100,150,200]}
kf = KFold(n_splits=10,shuffle=True,random_state=10)
searchrf = GridSearchCV(estimator=RandomForestRegressor(),param_grid=param_grid, scoring='neg_mean_squared_error', cv=kf, verbose=1)
searchrf.fit(xtrain,ytrain)
searchrf.best_params_
rf_tuned = RandomForestRegressor(criterion= 'mse',max_depth= 8,min_samples_split= 4,n_estimators= 150)
rf_tuned_model = rf.fit(xtrain,ytrain)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=rf_tuned, X=xtrain, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=rf_tuned, X=xtrain, y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=rf_tuned, X=xtest, y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=rf_tuned, X=xtest, y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))<jupyter_output>Average Train R2    0.9380133570861234
Average Train RMSE  265.58767237855074
--------------------------------------------------
Average Test R2    0.6024818252589288
Average Test RMSE  385.81737824299887<jupyter_text>Gradient Boost<jupyter_code># {'criterion': 'friedman_mse',
#  'max_depth': 2,
#  'min_samples_split': 3,
#  'n_estimators': 200}
param_grid = {'criterion':['mse','friedman_mse'],'n_estimators':[100,150,200,250,300],
              'max_depth':[2,3,4,5,10],'min_samples_split':[2,3,4,5],'learning_rate':np.arange(0.1,0.3,0.01),
             'min_samples_leaf':[1,2,3]}
kf = KFold(n_splits=10,shuffle=True,random_state=10)
searchgb = GridSearchCV(estimator=GradientBoostingRegressor(),param_grid=param_grid, scoring='neg_mean_squared_error', cv=kf, verbose=True)
searchgb.fit(xtrain,ytrain)
searchgb.best_params_
param_grid = {'criterion':['mse','friedman_mse'],'n_estimators':[100,150,200,250,300],
              'max_depth':[2,3,4,5,10],'min_samples_split':[2,3,4,5],'learning_rate':np.arange(0.1,0.3,0.01)}
kf = KFold(n_splits=10,shuffle=True,random_state=10)
searchgb = GridSearchCV(estimator=GradientBoostingRegressor(),param_grid=param_grid, scoring='neg_mean_squared_error', cv=kf, verbose=True)
searchgb.fit(xtrain,ytrain)
searchgb.best_params_<jupyter_output>Fitting 10 folds for each of 4000 candidates, totalling 40000 fits<jupyter_text>Model Building using tunned params and feature engeering<jupyter_code>LR = LinearRegression()
LR_bag = BaggingRegressor(base_estimator=LR,n_estimators=30)
LR_boost = AdaBoostRegressor(base_estimator=LR,n_estimators=50,learning_rate=0.19)


RF=RandomForestRegressor(criterion= 'mse',max_depth= 8,min_samples_split= 4,n_estimators= 150)
RF_boost=AdaBoostRegressor(base_estimator=RF,n_estimators=4,random_state=0)


XGB = XGBRegressor(n_estimators =150,learning_rate = 0.19, gamma= 3,max_depth= 2)


GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 66, 
                                 learning_rate=0.5099999999999998,subsample= 0.9)

models = []
models.append(('Linear', LR))
models.append(('BaggedLR', LR_bag))
models.append(('BoostedLR',LR_boost))

models.append(('RF',RF))
models.append(('BoostedRF',RF_boost))

models.append(('GBoost',GBoost))
#Train Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=xtrain, y=ytrain, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=xtrain, y=ytrain, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()
#Test Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    testRMSEscore = cross_val_score(estimator=model, X=xtest, y=ytest, scoring='neg_mean_squared_error', cv=kf)
    testr2score = cross_val_score(estimator=model, X=xtest, y=ytest, scoring='r2', cv=kf)
    
    
    #RMSE Plot
    results.append(np.sqrt(abs(testRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(testr2score),np.mean(np.sqrt(abs(testRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-testr2score),np.std(testr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()
#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=X, y=y, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=X, y=y, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score 0.8866981893494887,  RMSE 360.7884146897849
Linear :  Bias Error = 0.11330181065051129,  Variance Error=0.04594888892849369
--------------------------------
BaggedLR :  R2 Score 0.8880221381776663,  RMSE 359.106887894779
BaggedLR :  Bias Error = 0.11197786182233382,  Variance Error=0.04606409897572384
--------------------------------
BoostedLR :  R2 Score 0.8691631286998213,  RMSE 384.6927501813653
BoostedLR :  Bias Error = 0.13083687130017868,  Variance Error=0.04827898508311901
--------------------------------
RF :  R2 Score 0.9629077565974328,  RMSE 210.75972697756634
RF :  Bias Error = 0.03709224340256726,  Variance Error=0.018351739343131637
--------------------------------
BoostedRF :  R2 Score 0.947732678874452,  RMSE 235.9220108991423
BoostedRF :  Bias Error = 0.052267321125547904,  Variance Error=0.03848722982704759
--------------------------------
GBoost :  R2 Score 0.9751689650545229,  RMSE 170.83881465403593
GBoost :  Bias Error = 0.024831034945476983,  V[...]<jupyter_text>Trials with different features<jupyter_code>print(sfs_backward_features)
print("-"*50)
print(sfs_forward_features)
print("-"*50)
print(rfe_lr_features)
print("-"*50)
print(sfs_backward_features_gb)
print("-"*50)
print(sfs_forward_features_gb)
print("-"*50)
print(manual_features_1)
print("-"*50)
print(manual_features)<jupyter_output>['Month', 'Day', 'Dew Point avg (°C)', 'Visibility\xa0(km) avg', 'unemployment rate', 'CommercialBankInterestRateonCreditCardPlans', 'Change(in%)', 'Average upland planted(million acres)', 'Production (in  480-lb netweright in million bales)', 'Mill use  (in  480-lb netweright in million bales)', 'Exports', 'ProductCategory_MenClothing', 'ProductCategory_OtherClothing', 'ProductCategory_WomenClothing']
--------------------------------------------------
['Year', 'Month', 'Day', 'Temp avg (°C)', 'Monthly Real GDP Index (inMillion$)', 'Change(in%)', 'Average upland planted(million acres)', 'Production (in  480-lb netweright in million bales)', 'Exports', 'ProductCategory_MenClothing', 'ProductCategory_OtherClothing', 'ProductCategory_WomenClothing']
--------------------------------------------------
['Year', 'Month', 'Event', 'Day', 'Temp avg (°C)', 'Visibility\xa0(km) avg', 'CPI', 'unemployment rate', 'CommercialBankInterestRateonCreditCardPlans', 'Finance Rate on Personal Loans at Comme[...]<jupyter_text>[sfs_backward_features]<jupyter_code>#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=xtest[sfs_backward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=xtest[sfs_backward_features], y=ytest, scoring='r2', cv=kf)
    
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score 0.5682100191414585,  RMSE 328.397526072794
Linear :  Bias Error = 0.43178998085854153,  Variance Error=0.5935388647831432
--------------------------------
BaggedLR :  R2 Score 0.5285557602231328,  RMSE 351.4081582059568
BaggedLR :  Bias Error = 0.4714442397768671,  Variance Error=0.61883219988732
--------------------------------
BoostedLR :  R2 Score 0.41426269867546905,  RMSE 372.61175137226405
BoostedLR :  Bias Error = 0.585737301324531,  Variance Error=0.9093405882297045
--------------------------------
RF :  R2 Score 0.7220437950302537,  RMSE 353.853340084023
RF :  Bias Error = 0.27795620496974627,  Variance Error=0.3006807657426803
--------------------------------
BoostedRF :  R2 Score 0.7595137933278727,  RMSE 346.9292315581468
BoostedRF :  Bias Error = 0.2404862066721273,  Variance Error=0.19087354811136212
--------------------------------
GBoost :  R2 Score 0.9073242115646012,  RMSE 240.6054477455603
GBoost :  Bias Error = 0.0926757884353988,  Variance Error=[...]<jupyter_text>sfs_forward_features<jupyter_code>#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=xtest[sfs_forward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=xtest[sfs_forward_features], y=ytest, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score 0.23994235740581718,  RMSE 416.8582491546379
Linear :  Bias Error = 0.7600576425941828,  Variance Error=0.9103960029236224
--------------------------------
BaggedLR :  R2 Score 0.30268461905669086,  RMSE 404.8266945537869
BaggedLR :  Bias Error = 0.6973153809433091,  Variance Error=0.8155281240583634
--------------------------------
BoostedLR :  R2 Score 0.2639917531663835,  RMSE 421.5809206507009
BoostedLR :  Bias Error = 0.7360082468336164,  Variance Error=0.8784538482321035
--------------------------------
RF :  R2 Score 0.6326510160782449,  RMSE 369.51239118117144
RF :  Bias Error = 0.3673489839217551,  Variance Error=0.476375411787334
--------------------------------
BoostedRF :  R2 Score 0.7024393207366904,  RMSE 383.83020788765765
BoostedRF :  Bias Error = 0.2975606792633097,  Variance Error=0.27160919558101604
--------------------------------
GBoost :  R2 Score 0.8088700006678181,  RMSE 327.2509963977904
GBoost :  Bias Error = 0.19112999933218194,  Variance E[...]<jupyter_text>rfe_lr_features<jupyter_code>#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=xtest[rfe_lr_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=xtest[rfe_lr_features], y=ytest, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score 0.16506188974881394,  RMSE 483.02240531954703
Linear :  Bias Error = 0.8349381102511861,  Variance Error=1.0255971907498933
--------------------------------
BaggedLR :  R2 Score -4.947464500297511e+17,  RMSE 134263012970.39473
BaggedLR :  Bias Error = 4.947464500297511e+17,  Variance Error=9.811850099165449e+17
--------------------------------
BoostedLR :  R2 Score -0.06685141501329721,  RMSE 515.8250576526392
BoostedLR :  Bias Error = 1.0668514150132973,  Variance Error=1.3331322825623821
--------------------------------
RF :  R2 Score 0.623091171798585,  RMSE 369.20292671634314
RF :  Bias Error = 0.37690882820141497,  Variance Error=0.5032440023246745
--------------------------------
BoostedRF :  R2 Score 0.6790694244499615,  RMSE 365.99281067714344
BoostedRF :  Bias Error = 0.32093057555003845,  Variance Error=0.3666922932895443
--------------------------------
GBoost :  R2 Score 0.8347492642443968,  RMSE 289.62566693042993
GBoost :  Bias Error = 0.165250735755603[...]<jupyter_text>sfs_backward_features_gb<jupyter_code>sfs_bck_gb_untunned_features=['Year', 'Month', 'Temp avg (°C)', 'Monthly Real GDP Index (inMillion$)', 'unemployment rate', 'Average upland planted(million acres)', 'Average upland harvested(million acres)', 'Production (in  480-lb netweright in million bales)', 'Mill use  (in  480-lb netweright in million bales)', 'Exports', 'ProductCategory_OtherClothing', 'ProductCategory_WomenClothing']
#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=xtest[sfs_bck_gb_untunned_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=xtest[sfs_bck_gb_untunned_features], y=ytest, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score 0.49064197042556595,  RMSE 385.526364946972
Linear :  Bias Error = 0.509358029574434,  Variance Error=0.6295472206250756
--------------------------------
BaggedLR :  R2 Score 0.4821030689730745,  RMSE 393.09914059178675
BaggedLR :  Bias Error = 0.5178969310269255,  Variance Error=0.6072594431561785
--------------------------------
BoostedLR :  R2 Score 0.412380352199348,  RMSE 387.89774125968734
BoostedLR :  Bias Error = 0.5876196478006519,  Variance Error=0.8069919406236353
--------------------------------
RF :  R2 Score 0.6568579858584863,  RMSE 373.61415174914987
RF :  Bias Error = 0.34314201414151363,  Variance Error=0.424711136728151
--------------------------------
BoostedRF :  R2 Score 0.7172215101993855,  RMSE 365.6958349690615
BoostedRF :  Bias Error = 0.2827784898006144,  Variance Error=0.2837057134717779
--------------------------------
GBoost :  R2 Score 0.8602263954937325,  RMSE 283.5577109514643
GBoost :  Bias Error = 0.13977360450626755,  Variance Erro[...]<jupyter_text>sfs_forward_features_gb<jupyter_code>sfs_fwd_gb_untunned_features = ['Year', 'Month', 'Temp avg (°C)', 'Monthly Real GDP Index (inMillion$)', 'unemployment rate', 'Average upland harvested(million acres)', 'ProductCategory_MenClothing', 'ProductCategory_OtherClothing', 'ProductCategory_WomenClothing']
#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=xtest[sfs_fwd_gb_untunned_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=xtest[sfs_fwd_gb_untunned_features], y=ytest, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score 0.5302725663936273,  RMSE 358.8403587002725
Linear :  Bias Error = 0.4697274336063727,  Variance Error=0.6278762655142559
--------------------------------
BaggedLR :  R2 Score 0.4810245296800959,  RMSE 368.4958892102266
BaggedLR :  Bias Error = 0.5189754703199041,  Variance Error=0.6571191688505603
--------------------------------
BoostedLR :  R2 Score 0.6529810003633184,  RMSE 359.944489675581
BoostedLR :  Bias Error = 0.3470189996366816,  Variance Error=0.4510922082910718
--------------------------------
RF :  R2 Score 0.71238843956317,  RMSE 346.1866813441662
RF :  Bias Error = 0.2876115604368301,  Variance Error=0.38855310844771146
--------------------------------
BoostedRF :  R2 Score 0.6776731984542397,  RMSE 364.6797934566565
BoostedRF :  Bias Error = 0.32232680154576043,  Variance Error=0.4506262809074202
--------------------------------
GBoost :  R2 Score 0.8046012188446209,  RMSE 279.66872393005167
GBoost :  Bias Error = 0.19539878115537893,  Variance Error[...]<jupyter_text>manual_features_1<jupyter_code>#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=xtest[manual_features_1], y=ytest, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=xtest[manual_features_1], y=ytest, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score 0.3213542748966801,  RMSE 442.4797792457815
Linear :  Bias Error = 0.6786457251033198,  Variance Error=0.8499795299356221
--------------------------------
BaggedLR :  R2 Score 0.18692823183592555,  RMSE 435.509843828436
BaggedLR :  Bias Error = 0.8130717681640744,  Variance Error=1.0870353720599195
--------------------------------
BoostedLR :  R2 Score 0.23441614490386523,  RMSE 454.85024729436043
BoostedLR :  Bias Error = 0.7655838550961348,  Variance Error=1.022384654096843
--------------------------------
RF :  R2 Score 0.6361289806672548,  RMSE 350.1144942700054
RF :  Bias Error = 0.36387101933274524,  Variance Error=0.589073541148362
--------------------------------
BoostedRF :  R2 Score 0.6381411302519845,  RMSE 356.90833206149966
BoostedRF :  Bias Error = 0.3618588697480155,  Variance Error=0.5187165950487506
--------------------------------
GBoost :  R2 Score 0.8116123033099221,  RMSE 264.3367238847108
GBoost :  Bias Error = 0.18838769669007793,  Variance Err[...]<jupyter_text>manual_features<jupyter_code>#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=xtest[manual_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=xtest[manual_features], y=ytest, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score -0.10802561960225518,  RMSE 533.2947535907928
Linear :  Bias Error = 1.1080256196022549,  Variance Error=1.6158649345987937
--------------------------------
BaggedLR :  R2 Score -5.174847459849473e+18,  RMSE 454495937645.2887
BaggedLR :  Bias Error = 5.174847459849473e+18,  Variance Error=1.0848120784246577e+19
--------------------------------
BoostedLR :  R2 Score -0.2440404922489951,  RMSE 560.7508297880753
BoostedLR :  Bias Error = 1.2440404922489952,  Variance Error=1.8815890080385573
--------------------------------
RF :  R2 Score 0.6116468006722438,  RMSE 362.2507484676882
RF :  Bias Error = 0.3883531993277562,  Variance Error=0.5041155171747397
--------------------------------
BoostedRF :  R2 Score 0.6759143300568333,  RMSE 390.8341637578388
BoostedRF :  Bias Error = 0.32408566994316673,  Variance Error=0.3297928397549683
--------------------------------
GBoost :  R2 Score 0.7797035562527528,  RMSE 285.28499115674765
GBoost :  Bias Error = 0.22029644374724722,[...]<jupyter_text>Feature selection from hypertunned Gradient Boost Model<jupyter_code>print(sfs_backward_features_gb)
print("-"*50)
print(sfs_forward_features_gb)
print("-"*50)
print(rfe_gb_features)
#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=X[sfs_backward_features_gb], y=y, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=X[sfs_backward_features_gb], y=y, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()
#Entire Dataset-------------------------So far the best
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=X[sfs_forward_features_gb], y=y, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=X[sfs_forward_features_gb], y=y, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()
#Entire Dataset
results = []
names = []
for name, model in models:
    kf = KFold(n_splits=10,shuffle=True,random_state=10)
    
    trainRMSEscore = cross_val_score(estimator=model, X=X[rfe_gb_features], y=y, scoring='neg_mean_squared_error', cv=kf)
    trainr2score = cross_val_score(estimator=model, X=X[rfe_gb_features], y=y, scoring='r2', cv=kf)
    
    #RMSE Plot
    results.append(np.sqrt(abs(trainRMSEscore)))
    names.append(name)
    print("{} :  R2 Score {},  RMSE {}".format(name, np.mean(trainr2score),np.mean(np.sqrt(abs(trainRMSEscore)))))
    print("{} :  Bias Error = {},  Variance Error={}".format(name, np.mean(1-trainr2score),np.std(trainr2score,ddof=1)))
    print("-"*32)
   # boxplot algorithm comparison
fig = plt.figure(figsize=(20,8))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names)
plt.show()<jupyter_output>Linear :  R2 Score 0.8996225842862764,  RMSE 342.1106633587003
Linear :  Bias Error = 0.10037741571372347,  Variance Error=0.037910976117589784
--------------------------------
BaggedLR :  R2 Score 0.9020043576448273,  RMSE 339.5663251412394
BaggedLR :  Bias Error = 0.09799564235517272,  Variance Error=0.034727897158463854
--------------------------------
BoostedLR :  R2 Score 0.8861772621956658,  RMSE 358.27870152214183
BoostedLR :  Bias Error = 0.11382273780433408,  Variance Error=0.042018761551727964
--------------------------------
RF :  R2 Score 0.9650505209593023,  RMSE 205.24444283718168
RF :  Bias Error = 0.034949479040697715,  Variance Error=0.01880119562759777
--------------------------------
BoostedRF :  R2 Score 0.9452118190559882,  RMSE 251.074081058461
BoostedRF :  Bias Error = 0.05478818094401174,  Variance Error=0.026362091466723452
--------------------------------
GBoost :  R2 Score 0.9755340717351034,  RMSE 169.46871527817103
GBoost :  Bias Error = 0.02446592826489676[...]<jupyter_text>Final Model Building  1. GB + sfs_forward_features<jupyter_code>GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 200, 
                                 learning_rate=0.19,random_state=208)
GB_model_Tunned_v1 = GBoost.fit(X[sfs_forward_features],y)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=GBoost, X=X[sfs_forward_features], y=y, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=GBoost, X=X[sfs_forward_features], y=y, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))

GB_model_Tunned_v1.predict(s14[sfs_forward_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = GB_model_Tunned_v1.predict(s14[sfs_forward_features])
te.to_csv("GB_tunned_FeatureSet_v1.csv",index=False)

#Kaggle
313.78568<jupyter_output><empty_output><jupyter_text>2. GB + sfs_forward_features_gb<jupyter_code>sfs_fwd_gb_untunned_features = ['Year', 'Month', 'Temp avg (°C)', 'Monthly Real GDP Index (inMillion$)', 'unemployment rate', 'Average upland harvested(million acres)', 'ProductCategory_MenClothing', 'ProductCategory_OtherClothing', 'ProductCategory_WomenClothing']
GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 200, 
                                 learning_rate=0.19,random_state=208)
GB_model_Tunned_v2 = GBoost.fit(X[sfs_fwd_gb_untunned_features],y)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=GBoost, X=X[sfs_fwd_gb_untunned_features], y=y, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=GBoost, X=X[sfs_fwd_gb_untunned_features], y=y, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))

GB_model_Tunned_v2.predict(s14[sfs_fwd_gb_untunned_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = GB_model_Tunned_v2.predict(s14[sfs_fwd_gb_untunned_features])
te.to_csv("GB_tunned_FeatureSet_v2.csv",index=False)

#Kaggle
271.54052<jupyter_output><empty_output><jupyter_text>3.GB + sfs_forward_features_gb<jupyter_code>sfs_forward_features_gb
GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 200, 
                                 learning_rate=0.19,random_state=208)
GB_model_Tunned_v3 = GBoost.fit(X[sfs_forward_features_gb],y)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=GBoost, X=X[sfs_forward_features_gb], y=y, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=GBoost, X=X[sfs_forward_features_gb], y=y, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))

GB_model_Tunned_v3.predict(s14[sfs_forward_features_gb])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = GB_model_Tunned_v3.predict(s14[sfs_forward_features_gb])
te.to_csv("GB_tunned_FeatureSet_v3.csv",index=False)

#KAggle
324.23764<jupyter_output><empty_output><jupyter_text>4.GB + sfs_backward_features_gb<jupyter_code>sfs_backward_features_gb
0.2899999999999999
GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 270, 
                                 learning_rate=0.5099999999999998,subsample= 0.9)
GB_model_Tunned_v4 = GBoost.fit(X[sfs_fwd_gb_untunned_features],y)

kf = KFold(n_splits=10,shuffle=True)
trainRMSEscore = cross_val_score(estimator=GBoost, X=X[sfs_fwd_gb_untunned_features], y=y, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=GBoost, X=X[sfs_fwd_gb_untunned_features], y=y, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))

GB_model_Tunned_v4.predict(s14[sfs_fwd_gb_untunned_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = GB_model_Tunned_v4.predict(s14[sfs_fwd_gb_untunned_features])
te.to_csv("GB_tunned_FeatureSet_v4.csv",index=False)

#Kaggle
#387.66145<jupyter_output><empty_output><jupyter_text>5. GB and sfsBackward Feature<jupyter_code>#br = BaggingRegressor(base_estimator=GradientBoostingRegressor(n_estimators=30,random_state=10,learning_rate=0.19,min_samples_leaf=1),n_estimators=30)
gb=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 300, 
                                 learning_rate=0.2899999999999999,subsample= 0.9)

GB_model_Tunned_v5 = gb.fit(xtrain[sfs_backward_features], ytrain)
kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=gb, X=xtest[sfs_backward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=gb, X=xtest[sfs_backward_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
GB_model_Tunned_v5.predict(s14[sfs_backward_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = GB_model_Tunned_v5.predict(s14[sfs_backward_features])
te.to_csv("GB_tunned_FeatureSet_v5.csv",index=False)
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = GB_model_Tunned_v5.predict(s14[sfs_backward_features])
te.to_csv("GB_tunned_FeatureSet_v6.csv",index=False)
GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 200, 
                                 learning_rate=0.19)
GB_model_Tunned_v2 = GBoost.fit(X[sfs_fwd_gb_untunned_features],y)

kf = KFold(n_splits=10,shuffle=True,random_state=10)
trainRMSEscore = cross_val_score(estimator=GBoost, X=X[sfs_fwd_gb_untunned_features], y=y, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=GBoost, X=X[sfs_fwd_gb_untunned_features], y=y, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))

GB_model_Tunned_v2.predict(s14[sfs_fwd_gb_untunned_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = GB_model_Tunned_v2.predict(s14[sfs_fwd_gb_untunned_features])
te.to_csv("GB_tunned_FeatureSet_v11.csv",index=False)
features_set

xtrain, xtest, ytrain, ytest = train_test_split(X,y, test_size=0.2)
GBoost=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 66, 
                                 learning_rate=0.5099999999999998,subsample= 0.9)
lr = LinearRegression()

GB_model_Tunned_v5 = lr.fit(xtrain[sfs_fwd_gb_untunned_features], ytrain)
kf = KFold(n_splits=10,shuffle=True)
trainRMSEscore = cross_val_score(estimator=lr, X=xtrain[sfs_fwd_gb_untunned_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=lr, X=xtrain[sfs_fwd_gb_untunned_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=lr, X=xtest[sfs_fwd_gb_untunned_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=lr, X=xtest[sfs_fwd_gb_untunned_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
GB_model_Tunned_v5.predict(s14[sfs_fwd_gb_untunned_features])
te = pd.DataFrame()
te['Year'] = np.arange(1,37)
te['Sales(In ThousandDollars)'] = GB_model_Tunned_v5.predict(s14[sfs_fwd_gb_untunned_features])
te.to_csv("GB_tunned_FeatureSet_v12.csv",index=False)
'learning_rate': 0.2899999999999999
xtrain, xtest, ytrain, ytest = train_test_split(X,y, test_size=0.25,random_state=208)
gb=GradientBoostingRegressor(criterion= 'friedman_mse',max_depth= 2,min_samples_split= 3,n_estimators= 300, 
                                 learning_rate=0.2899999999999999,subsample= 0.9)

GB_model_Tunned_v6 = gb.fit(xtrain[sfs_backward_features], ytrain)
kf = KFold(n_splits=10,shuffle=True)
trainRMSEscore = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features], y=ytrain, scoring='neg_mean_squared_error', cv=kf)
trainr2score = cross_val_score(estimator=gb, X=xtrain[sfs_backward_features], y=ytrain, scoring='r2', cv=kf)

print("Average Train R2   ",np.mean(trainr2score))
print("Average Train RMSE ",np.mean(np.sqrt(abs(trainRMSEscore))))
print("-"*50)
testRMSEscore = cross_val_score(estimator=gb, X=xtest[sfs_backward_features], y=ytest, scoring='neg_mean_squared_error', cv=kf)
testr2score = cross_val_score(estimator=gb, X=xtest[sfs_backward_features], y=ytest, scoring='r2', cv=kf)

print("Average Test R2   ",np.mean(testr2score))
print("Average Test RMSE ",np.mean(np.sqrt(abs(testRMSEscore))))
GB_model_Tunned_v6.predict(s14[sfs_backward_features])
{'max_depth': 3, 'min_samples_split': 2, 'n_estimators': 300, 'subsample': 0.9}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.170858563519058, 'fraction_numerical': 0.08777816756402239, 'mean_word_length': 6.298675275520427, 'pattern_counts': {'":': 0, '<': 80, '<?xml version=': 0, '>': 81, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 9, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '23', 'dir': 'jupyter-structured-clean-dedup', 'id': '661944', 'license': 'Apache-2.0', 'n_tokens_mistral': 29530, 'n_tokens_neox': 25645, 'n_words': 3612, 'path': 'Walmart Sales Forecast - Time Series and regression/WSF - Feature Engineering and Regression.ipynb', 'repo_name': 'nandanloya/GenNL', 'size': '65563'}