<jupyter_start><jupyter_text>Data Understanding  Data Collection<jupyter_code># Import data
product_quantity = pd.read_csv(r"C:\Users\devfano1\OneDrive - EY\Documents\VF_Privat\PartA.csv", low_memory = False, encoding = 'utf-8')

product_ad = pd.read_csv(r"C:\Users\devfano1\OneDrive - EY\Documents\VF_Privat\PartB.csv", low_memory = False, encoding = 'utf-8')

#Structure of table product_quantity
print(product_quantity.dtypes)
print(product_quantity.shape)
product_quantity.head()
#Structure of table product_ad
print(product_ad.dtypes)
print(product_ad.shape)
product_ad.head()
product_ad.dtypes<jupyter_output><empty_output><jupyter_text>Change StoresAvailability in float<jupyter_code>product_ad['StoresAvailability'] = product_ad['StoresAvailability']/100<jupyter_output><empty_output><jupyter_text>Duplicates and missing values in both tables<jupyter_code>print("product_quantity duplicates (row): "+str(product_quantity.duplicated().sum()))
print("product_ad duplicates (row): "+str(product_ad.duplicated().sum()))
duplicates_Product = product_quantity[product_quantity.duplicated()]
print("Duplicate are made for these variables:")
for i in {'Product', 'Channel', 'WeekStart'}:
    print (duplicates_Product[i].name+":")
    print (duplicates_Product[i].sort_values().unique())
    print ("\n")<jupyter_output>Duplicate are made for these variables:
Product:
['TXQ0T' 'TXQQX' 'TXQT7' 'TXQTZ' 'TXX00' 'TXX06' 'TXX5Q' 'TXXX0' 'TXZ06']


Channel:
['A' 'B' 'C' 'D' 'E' 'F']


WeekStart:
['2015-01-26' '2015-02-02' '2015-02-09' '2015-02-16' '2015-02-23'
 '2015-03-02' '2015-03-09' '2015-03-16' '2015-03-23' '2015-03-30'
 '2015-04-06' '2015-04-13' '2015-04-20' '2015-04-27' '2015-05-04'
 '2015-05-11' '2015-05-18' '2015-05-25']<jupyter_text>Duplicates have been made in the first half of 2015<jupyter_code>#Drop duplicates
product_quantity.drop_duplicates(inplace=True)
#Missing values in both tables:
print("product_quantity Table Null values")
print(product_quantity.isnull().sum())
print ("\n")
print("product_ad Table Null values")
print(product_ad.isnull().sum())
print ("\n")
print("product_quantity Table NA values")
print(product_quantity.isna().sum())
print ("\n")
print("product_ad Table NA values")
print(product_ad.isna().sum())<jupyter_output>product_quantity Table Null values
Product          0
Channel          0
WeekStart        0
QuantitySold     0
Price            0
isPromoPeriod    0
dtype: int64


product_ad Table Null values
Product               0
Channel               0
WeekStart             0
TV                    0
Online                0
StoresAvailability    0
dtype: int64


product_quantity Table NA values
Product          0
Channel          0
WeekStart        0
QuantitySold     0
Price            0
isPromoPeriod    0
dtype: int64


product_ad Table NA values
Product               0
Channel               0
WeekStart             0
TV                    0
Online                0
StoresAvailability    0
dtype: int64<jupyter_text>Merge both tables<jupyter_code>products = pd.merge(product_quantity, product_ad,how = 'inner', on=["Product", "Channel", "WeekStart"])
products.head()<jupyter_output><empty_output><jupyter_text>Data Description  Shape and data types of new dataset<jupyter_code>print("Products dataset rows: {}".format(products.shape[0]))
print("Products dataset columns: {}".format(products.shape[1]))
print("Products duplicates (row): "+str(products.duplicated().sum()))
products.dtypes<jupyter_output><empty_output><jupyter_text>Unique values<jupyter_code>for i in products:
    print (products[i].name+":")
    print (products[i].sort_values().unique())
    print ("\n")<jupyter_output>Product:
['TXQ0T' 'TXQQX' 'TXQT7' 'TXQTZ' 'TXX00' 'TXX06' 'TXX5Q' 'TXXX0' 'TXZ06'
 'TZQAQ']


Channel:
['A' 'B' 'C' 'D' 'E' 'F']


WeekStart:
['2015-01-05' '2015-01-12' '2015-01-19' '2015-01-26' '2015-02-02'
 '2015-02-09' '2015-02-16' '2015-02-23' '2015-03-02' '2015-03-09'
 '2015-03-16' '2015-03-23' '2015-03-30' '2015-04-06' '2015-04-13'
 '2015-04-20' '2015-04-27' '2015-05-04' '2015-05-11' '2015-05-18'
 '2015-05-25' '2015-06-01' '2015-06-08' '2015-06-15' '2015-06-22'
 '2015-06-29' '2015-07-06' '2015-07-13' '2015-07-20' '2015-07-27'
 '2015-08-03' '2015-08-10' '2015-08-17' '2015-08-24' '2015-08-31'
 '2015-09-07' '2015-09-14' '2015-09-21' '2015-09-28' '2015-10-05'
 '2015-10-12' '2015-10-19' '2015-10-26' '2015-11-02' '2015-11-09'
 '2015-11-16' '2015-11-23' '2015-11-30' '2015-12-07' '2015-12-14'
 '2015-12-21' '2015-12-28' '2016-01-04' '2016-01-11' '2016-01-18'
 '2016-01-25' '2016-02-01' '2016-02-08' '2016-02-15' '2016-02-22'
 '2016-02-29' '2016-03-07' '2016-03-14' '2016-03-21' '2016-03-28'
[...]<jupyter_text>Detection of missing values and errors in new table<jupyter_code>print( products.isnull().sum())<jupyter_output>Product               0
Channel               0
WeekStart             0
QuantitySold          0
Price                 0
isPromoPeriod         0
TV                    0
Online                0
StoresAvailability    0
dtype: int64<jupyter_text>-> No missing values in merged product table  Export to CSV<jupyter_code>products.to_csv(r"C:\Users\devfano1\Documents\Privat\export_products_clean.csv", index=False, encoding='utf-8')<jupyter_output><empty_output><jupyter_text>Data Exploration Analysis of historical data  Analysis of average quantity sold per product<jupyter_code>#Add Month & year column
products['Month'] = pd.DatetimeIndex(products['WeekStart']).month
products['Year'] = pd.DatetimeIndex(products['WeekStart']).year

#Drop WeekStart
products.drop(['WeekStart'], axis =1, inplace = True)
products.columns
ax = sns.barplot(x="Product", y="QuantitySold", data = products, ci=None, estimator = np.average, palette = 'Blues')
ax.set_title("Average quantity sold per product")
plt.show()<jupyter_output><empty_output><jupyter_text>Most sold product is TXQAQ.  Price over the years<jupyter_code>plt.figure()
ax = sns.lineplot(x="Year", y="Price", hue = 'Product', data = products, estimator = np.average, ci=None)
ax.legend(loc = 'upper right')
ax.set_title("Price over the years")
plt.show()<jupyter_output><empty_output><jupyter_text>Price is decreasing for TXQT7 and TXQTZ (maybe related products as they have similiar names), but still both are the most expensive products still.The price of TXQT7 was decreased a lot during 2016-2017. On the other hand you can see that the quantity decreased in 2016, but then increased again in 2017 and 2018. --> Due to decreasing quantity sold maybe the price was adjusted. --> Quantity sold increased again.  The lowest price has product TXQQX. This is also one of the most sold products. The quantity sold was increasing constantly. Therefore it was possible to increase the price slowly and increase so the revenue.   Revenue<jupyter_code>products['Revenue'] = (products['QuantitySold']*products['Price']) 
ax = sns.barplot(x="Product", y="Revenue", data = products, ci=None, estimator = np.average, palette = 'Blues')
ax.set_title("Average quantity sold per product")
plt.show()<jupyter_output><empty_output><jupyter_text>The highest revenue (price * quantity sold) is bringing in average Product TXQTZ.  Influence of TV or Online advertisment<jupyter_code>products['TVimpact'] = False
for i in range(len(products['TV'])):
    if products['TV'][i] > 0:
        products['TVimpact'][i] = True
        
products['OnlineImpact'] = False
for i in range(len(products['Online'])):
    if products['Online'][i] > 0:
        products['OnlineImpact'][i] = True

fig, axes = plt.subplots(1,2, figsize = (12,6),  constrained_layout = True)
fig.suptitle("Average quantity sold impacted by additional advertisments")

sns.barplot(x="Product", y="QuantitySold",hue='TVimpact', data = products, estimator = np.average,
                 ci=None, palette = 'Blues', ax = axes[0])
ax = axes[0].set_title("Average quantity sold impacted by TV advertisment")

sns.barplot(x="Product", y="QuantitySold",hue='OnlineImpact', data = products, estimator = np.average,
                 ci=None, palette = 'Blues', ax = axes[1])
ax = axes[1].set_title("Average quantity sold impacted by Online advertisment")

plt.show()<jupyter_output><empty_output><jupyter_text>Quantity sold is higher if online advertisment is spent.   Revenue per Product including cost of advertisment<jupyter_code>products['Revenue2'] = (products['QuantitySold']*products['Price']) - (products['TV']+products['Online'])
ax = sns.barplot(x="Product", y="Revenue2", data = products, ci=None, estimator = np.average, palette = 'Blues')
ax.set_title("Revenue sold per product")
plt.show()<jupyter_output><empty_output><jupyter_text>For Product TXQQX the extra advertisment was more expensive in average then the revenue.<jupyter_code>#Drop 
products.drop(['TVimpact', 'OnlineImpact', 'Revenue', 'Revenue2'], axis =1, inplace = True)<jupyter_output><empty_output><jupyter_text>Analysis of seasonality<jupyter_code>#Month 
ax = sns.barplot(x="Month", y="QuantitySold", data = products, ci=None, estimator = np.average, palette = 'Blues')
ax.set_title("Average quantity sold per Month")
plt.show()<jupyter_output><empty_output><jupyter_text>- November is most sold. - During summer period is a bit less.  Quantity sold per month per product<jupyter_code>mqs= sns.catplot(x="Month", y="QuantitySold", col = 'Product', data = products, kind="bar",ci=None, estimator = np.average,
            palette = 'Blues', height = 5, aspect=.8, col_wrap = 4, margin_titles=True)

mqs.fig.subplots_adjust(top=0.9) # adjust the Figure in rp
mqs.fig.suptitle("Average quantity sold per Month per product")
plt.show()<jupyter_output><empty_output><jupyter_text>Top Products: TXQQX (especially in November) and TZQAQ (esp. in August, Sept)  Quantity sold per year<jupyter_code>#year
plt.figure()
ax = sns.barplot(x="Year", y="QuantitySold", data = products, ci=None, palette = 'Blues')
ax.set_title("Quantity sold per year")
plt.show()<jupyter_output><empty_output><jupyter_text>Constantly increasing of sold products --> 2018 has most quantities, even if the data is provided only for the first 4 month.   Quantity sold per product on years<jupyter_code>sns.catplot(x="Year", y="QuantitySold", col = 'Product', data = products, kind="bar", ci=None, palette = 'Blues', 
            estimator = np.average, height = 3, aspect=.8, col_wrap=5,)
mqs.fig.subplots_adjust(top=0.9) 
mqs.fig.suptitle("Average quantity sold per year per product")
plt.show()<jupyter_output><empty_output><jupyter_text>Product TXX06 and TXQQX are most sold in 2018. Product TXQAQ was released in 2016 and most sold product 2017.  Quantity sold per distribution channel<jupyter_code>plt.figure(figsize = (7,5))
ax = sns.barplot(x="Channel", y="QuantitySold", data = products, ci=None, estimator = np.average,
                 palette = 'Blues', order = products['Channel'].sort_values().unique())
ax.set_title("Quantity sold per channel")
plt.show()
channel_Product= pd.pivot_table(products, values= 'QuantitySold',
                                index =  ['Channel'], columns = ['Product'], aggfunc =np.average,
                                margins = True, margins_name = "Total")
channel_Product.style.background_gradient(cmap='Blues')<jupyter_output><empty_output><jupyter_text>Multiple products are only sold on specific channels.   Does the promo period influence the quantity sold?<jupyter_code>plt.figure(figsize = (7,5))
ax = sns.barplot(x="Channel", y="QuantitySold",hue='isPromoPeriod', data = products, estimator = np.average,
                 ci=None, palette = 'Blues', order = products['Channel'].sort_values().unique())
ax.set_title("Average quantity sold during the promo period")
plt.show()
channel_PromoPeriod= pd.pivot_table(products, values = 'QuantitySold', columns = ['isPromoPeriod'],
                                index =  ['Channel'], aggfunc =np.average, margins = True, margins_name = "Total")

channel_PromoPeriod['Percentage'] = np.round(((channel_PromoPeriod[True]-channel_PromoPeriod[False])/channel_PromoPeriod[False])*100,2)

channel_Product.style.background_gradient(cmap='Blues')
channel_PromoPeriod<jupyter_output><empty_output><jupyter_text>The promo periode increases the quantity sold, especially on channel C. However, Channel A is not that much fluenced percentage wise by promo period.  Store Availability<jupyter_code>plt.figure()
ax = sns.barplot(x="Product", y="StoresAvailability", data = products, hue ='isPromoPeriod', ci=None, palette = 'Blues')
plt.show()<jupyter_output><empty_output><jupyter_text>Stores Availability increases during promo period.   Correlation matrix<jupyter_code>products_corr = products.drop(['Year'], axis =1, inplace = True)
products_corr = pd.get_dummies(products, columns=['Channel', 'Product'])
#Correlation between Products and Channels:
plt.figure(figsize = (17,15), constrained_layout = True)
sns.set(font_scale=1)
corr=products_corr.corr()

hmap = sns.heatmap(data = corr, annot=True, linecolor="white",  cmap= 'Blues', fmt='.1g')
hmap.set_xticklabels(hmap.get_xmajorticklabels(), fontsize = 14)
hmap.set_yticklabels(hmap.get_ymajorticklabels(), fontsize = 14)
plt.show()<jupyter_output><empty_output><jupyter_text>- no Correlation between the products- no Correlation between products and channels- Correlation between:     - Quantity Sold and Stores Availability, Channel C, Promo Period    - Price and Product TXQT7 and Product TXQTZ    - Promo Period and Channel C, Stores Availability    - Channel C and Stores Availability--> It is likely that Channel C is (physical) stores.  Normality check of the data<jupyter_code>from statsmodels.graphics.gofplots import qqplot<jupyter_output><empty_output><jupyter_text>Visual normality check<jupyter_code>#QQ-Plot and Normaldistribution plot

fig, axes = plt.subplots(1,2, figsize = (9,5),  constrained_layout = True)
fig.suptitle("Visual normality checks")

#stats.probplot(products['QuantitySold'], dist="norm", plot=axes[0])
qqplot(products['QuantitySold'], line = 's', ax = axes[0])
ax=axes[0].set_title("QQ-Plot", fontsize = 16)
sns.distplot(products['QuantitySold'], fit=stats.norm, kde=False, ax= axes[1])
ax =axes[1].set_title("Normality distribution", fontsize = 16)
#plt.xlim(-1000, 5000)<jupyter_output><empty_output><jupyter_text>Normality tests<jupyter_code>#Shapiro-Wilk-Test -> not so reliable since data set might be too big.
stat, p = stats.shapiro(products['QuantitySold'])
print("Statistics: " + str(stat))
print("p-value: " + str(p))
#D'Agostino's-K^2-Test
stat, p = normaltest(products['QuantitySold'])
print('Statistics= %.3f, p= %.3f' % (stat, p))
#Anderson-Darling-Test
result = anderson(products['QuantitySold'])
print('Statistics: %.3f' % result.statistic)
p = 0
for i in range(len(result.critical_values)):
    sl, cv = result.significance_level[i], result.critical_values[i]
    if result.statistic < result.critical_values[i]:
        print('%.3f: %.3f, data looks normal (fail to reject H0)' % (sl, cv))
    else:
        print('%.3f: %.3f, data does not look normal (reject H0)' % (sl, cv))<jupyter_output>Statistics: 562.101
15.000: 0.576, data does not look normal (reject H0)
10.000: 0.656, data does not look normal (reject H0)
5.000: 0.786, data does not look normal (reject H0)
2.500: 0.917, data does not look normal (reject H0)
1.000: 1.091, data does not look normal (reject H0)<jupyter_text>--> The data is not normal distributed.  Data Preparation<jupyter_code>#import of Packages
from sklearn.preprocessing import MinMaxScaler
from sklearn.model_selection import train_test_split, KFold, cross_val_score, GridSearchCV
from sklearn.metrics import mean_squared_error, r2_score, mean_absolute_error, explained_variance_score

#Regression-Algorithms
from sklearn.neighbors import KNeighborsRegressor
from sklearn.svm import SVR
from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor
from sklearn import linear_model
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.tree import DecisionTreeRegressor

import pickle<jupyter_output><empty_output><jupyter_text>Get dummy variables<jupyter_code>products = pd.get_dummies(products, columns=['Product', 'Channel', 'Month'], drop_first = True)<jupyter_output><empty_output><jupyter_text>Export to CSV<jupyter_code>products.to_csv(r"C:\Users\devfano1\Documents\Privat\export_products_notNorm.csv", index=False, encoding='utf-8')<jupyter_output><empty_output><jupyter_text>Normalization<jupyter_code>scaler = MinMaxScaler()
products[['Price','TV','Online', 'StoresAvailability']] = scaler.fit_transform(products[['Price', 'TV',
       'Online', 'StoresAvailability']])

products.head()

products.dtypes
#### Export to CSV

products.to_csv(r"C:\Users\devfano1\Documents\Privat\export_products_Norm.csv", index=False, encoding='utf-8')<jupyter_output><empty_output><jupyter_text>Modeling  Regression model  Split into train and test data<jupyter_code>Y = products['QuantitySold'].values
X = products.drop(['QuantitySold'], axis = 1)
X_train, X_test, Y_train, Y_test = train_test_split( X, Y, test_size = 0.25)

print(X_train.shape)
print(X_test.shape)
print(Y_train.shape)
print(Y_test.shape)<jupyter_output>(4473, 30)
(1492, 30)
(4473,)
(1492,)<jupyter_text>Used regression methods<jupyter_code>#Used Models
models = []
models.append(('LR', LinearRegression()))
models.append(('RandomForest', RandomForestRegressor(n_estimators = 100)))
models.append(('RidgeRegression', Ridge()))
models.append(('LASSO', Lasso()))
models.append(('KNN', KNeighborsRegressor()))
models.append(('dtr', DecisionTreeRegressor()))
models.append(('ExtraTrees', ExtraTreesRegressor(n_estimators=100, random_state=0)))
models.append(('SVR', SVR()))<jupyter_output><empty_output><jupyter_text>Apply models<jupyter_code>names = []
scores = []
train_errors = []
test_errors = []
train_errors_absolute = []
test_errors_absolute = []
variance_scores =[]
results = []
RMS = 'neg_mean_squared_error'
for name, model in models:
    model.fit(X_train,Y_train)
    score = model.score(X_train,Y_train)
    train_error = mean_squared_error(Y_train, model.predict(X_train))
    y_pred = model.predict(X_test)
    test_error = mean_squared_error(Y_test, y_pred)
    train_error_absolute = mean_absolute_error(Y_train, model.predict(X_train))
    test_error_absolute = mean_absolute_error(Y_test, y_pred)
    variance_score=explained_variance_score(Y_test,y_pred)
    kfold = KFold()
    cv_results = cross_val_score(model, X_train, Y_train, cv=kfold, scoring=RMS)
    
    names.append(name)
    results.append(cv_results)
    scores.append(score)
    train_errors.append(train_error)
    test_errors.append(test_error)
    train_errors_absolute.append(train_error_absolute)
    test_errors_absolute.append(test_error_absolute)         
    variance_scores.append(variance_score)<jupyter_output><empty_output><jupyter_text>Evaluation<jupyter_code>results_table = pd.concat([pd.DataFrame([[names[i], scores[i], variance_scores[i], train_errors[i], test_errors[i], 
                                         train_errors_absolute[i], test_errors_absolute[i]]], 
                              columns = ["Models","Score", "Variance Score", "Train set mse", 
                                         "Test set mse", "Train set mae", "Test set mae"]) for i in range(len(models))],ignore_index = True)

results_table = results_table.sort_values(by=('Score'),ascending=False)

results_table.style.background_gradient(cmap='Blues')
fig = plt.figure(figsize=(5,5))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results)
ax.set_xticklabels(names, rotation ='vertical', fontsize = 8)
plt.show()

results2 = [results[1], results[6], results[5]]
names2  = [names[1], names[6], names[5]]

fig = plt.figure(figsize=(5,5))
fig.suptitle('Algorithm Comparison')
ax = fig.add_subplot(111)
plt.boxplot(results2)
ax.set_xticklabels(names2, rotation ='vertical', fontsize = 8)
plt.show()<jupyter_output><empty_output><jupyter_text>-> Decision Tree is most accurate, Random Forest best cross validation results. Extra Trees has also appr accuracy score 1 and a good cv result.  Analyze 3 best regression algorithms: Decision Tree, Extra Trees, Random Forest<jupyter_code>#Decision Tree
dtr = DecisionTreeRegressor()
dtr.fit(X_train,Y_train)
y_train_pred_dtr = dtr.predict(X_train)
res_dtr = (Y_train - y_train_pred_dtr)

#Extra Tree
et = ExtraTreesRegressor(n_estimators=100, random_state=0)
et.fit(X_train,Y_train)
y_train_pred_et = et.predict(X_train)
res_et = (Y_train - y_train_pred_et)

#Random Forest
rf = RandomForestRegressor(n_estimators=100, random_state=0)
rf.fit(X_train,Y_train)
y_train_pred_rf = rf.predict(X_train)
res_rf = (Y_train - y_train_pred_rf)<jupyter_output><empty_output><jupyter_text>Distribution of error terms<jupyter_code>fig, axes = plt.subplots(3,1, figsize=(9,9), constrained_layout = True)
fig.suptitle("Error Terms")

sns.distplot(res_dtr, ax = axes[0])
sns.distplot(res_et, ax = axes[1])
sns.distplot(res_rf, ax = axes[2])

plt.show()<jupyter_output><empty_output><jupyter_text>Key influencer<jupyter_code>fig, axes = plt.subplots(3,1, figsize=(10,10))
fig.suptitle("Key influencer")

feat_importances_rf = pd.Series(rf.feature_importances_, index=X.columns)
feat_importances_rf.nlargest(15).plot(kind='barh', ax = axes[0],fontsize = 9)
ax=axes[0].set_title("Random Forest", fontsize = 14)

feat_importances_dtr = pd.Series(dtr.feature_importances_, index=X.columns)
feat_importances_dtr.nlargest(15).plot(kind='barh', ax = axes[1], fontsize = 9)
ax=axes[1].set_title("Decision Tree", fontsize = 14)

feat_importances_et = pd.Series(et.feature_importances_, index=X.columns)
feat_importances_et.nlargest(15).plot(kind='barh', ax = axes[2], fontsize = 9)
ax=axes[2].set_title("Extra Trees", fontsize = 14)

plt.show()<jupyter_output><empty_output><jupyter_text>Key Influencer is Channel C, Store Availability and Price for the quantity sold.   Export files as .pkl<jupyter_code>pickle.dump(rf,open('rf_model.pkl','wb'))
pickle.dump(dtr,open('dtr_model.pkl','wb'))
pickle.dump(et,open('et_model.pkl','wb'))
products.columns
#Example prediction
input = np.matrix([[5,False, 0, 0, 0.10, 1, 0, 0, 0, 0, 0, 0, 0,0,0,1,0,0,0, 0,0,0,0,0,0,0,0,0,0,1]])
dtr.predict(input)<jupyter_output><empty_output><jupyter_text>Tuning Parameters<jupyter_code># Run Randomized Search CV
from sklearn.model_selection import RandomizedSearchCV
#Decision Tree Parameters
parameters_dtr = {'ccp_alpha': np.linspace(0, 5.0, endpoint=True),
              'criterion': ['mse', 'friedman_mse', 'mae', 'poission'], 
              'splitter': ['best', 'random'], 
              'min_samples_split':np.linspace(0.1, 2, 10, endpoint=True),
              'max_depth': [None,30,50,100,110,150],
               'max_features': ['auto', 'sqrt', 'log2'], 
               'max_leaf_nodes': [None,30,50,100,110,150,300],
              'min_impurity_decrease':[0, 0.5, 1],
              'min_impurity_split':[None,1,5],
              'min_samples_leaf':np.linspace(0.1, 0.5, 5, endpoint=True),
            'min_weight_fraction_leaf': [0, 0.25, 0.5],
              'random_state':[None, 16, 32]
             }
# Optimize Parameters for Decision Tree
random_dtr = RandomizedSearchCV(DecisionTreeRegressor(), parameters_dtr,  refit = True)

#fitting the model for grid search
random_dtr.fit(X_train,Y_train)

#print best parameter after tuning
random_dtr_best_param = random_dtr.best_params_
print(random_dtr_best_param)
random_dtr_predictions = random_dtr.predict(X_test)
#parameters for Extra Trees and Random Forest
parameters_et_rf = {'bootstrap':[True, False],
                    'ccp_alpha': np.linspace(0.1, 5.0, endpoint=True),
                    'criterion': ['mse', 'mae'],
                    'max_depth': [None,30,50,100,110,150], 
                    'max_features': ['auto', 'sqrt', 'log2'], 
                    'max_leaf_nodes': [None,50,100,150,300],
                    'min_impurity_decrease':[0, 0.5, 1],
                    'min_impurity_split':[None,1,5],
                    'min_samples_leaf':np.linspace(0.1, 0.5, 5, endpoint=True),
                    'min_weight_fraction_leaf': [0, 0.25, 0.5],
                    'n_estimators':[100, 200, 300],
                    'oob_score': [True, False],
                    'random_state':[None, 16, 32]
             }
# Optimize Parameters for Random Forest
random_rf = RandomizedSearchCV(RandomForestRegressor(), parameters_et_rf,cv=3)

#fitting the model for random search
random_rf.fit(X_train,Y_train)

#print best parameter after tuning
random_rf_best_param = random_rf.best_params_
print(random_rf_best_param)
random_rf_predictions = random_rf.predict(X_test)
# Optimize Parameters for Extra Trees
random_et = RandomizedSearchCV(ExtraTreesRegressor(), parameters_et_rf, cv = 3, refit = True)

#fitting the model for grid search
random_et.fit(X_train,Y_train)

#print best parameter after tuning
random_et_best_param = random_et.best_params_
print(random_et_best_param)
random_et_predictions = random_et.predict(X_test)
# grid_dtr = GridSearchCV(DecisionTreeRegressor(), parameters_dtr,  refit = True, verbose = 3, n_jobs = -1)

# #fitting the model for grid search
# grid_dtr.fit(X_train,Y_train)

# #print best parameter after tuning
# print(grid_dtr.best_params_)
# grid_dtr_predictions = grid_dtr.predict(X_test)

# grid_rf = GridSearchCV(RandomForestRegressor(), parameters_et_rf,cv=3, refit = True)

# #fitting the model for grid search
# grid_rf.fit(X_train,Y_train)

# #print best parameter after tuning
# print(grid_rf.best_params_)
# grid_rf_predictions = grid.predict(X_test)


# grid_et = GridSearchCV(ExtraTreesRegressor(), parameters_et_rf, cv = 3, refit = True)

# #fitting the model for grid search
# grid_et.fit(X_train,Y_train)

# #print best parameter after tuning
# print(grid_et.best_params_)
# grid_et_predictions = grid.predict(X_test)<jupyter_output><empty_output><jupyter_text>Analze model with tuned parameters<jupyter_code>#Used Models
models = []
models.append(('Random Forest tuned', RandomForestRegressor(random_state=0,
                                                 bootstrap = random_rf_best_param['bootstrap'], 
                                                 max_leaf_nodes =random_rf_best_param['max_leaf_nodes'], 
                                                 max_features =random_rf_best_param['max_features'],
                                                 max_depth=random_rf_best_param['max_depth'],
                                                 criterion=random_rf_best_param['criterion'],
                                                 ccp_alpha=random_rf_best_param['ccp_alpha']
                                                 ,n_estimators=random_rf_best_param['n_estimators'])))

models.append(('DTR tuned', DecisionTreeRegressor(splitter = random_dtr_best_param['splitter'] ,
                                                 random_state=random_dtr_best_param['random_state'],
                                                 min_samples_split =random_dtr_best_param['min_samples_split'], 
                                                 max_leaf_nodes =random_dtr_best_param['max_leaf_nodes'], 
                                                 max_features =random_dtr_best_param['max_features'],
                                                 max_depth=random_dtr_best_param['max_depth'],
                                                 criterion=random_dtr_best_param['criterion'],
                                                 ccp_alpha=random_dtr_best_param['ccp_alpha'])))



models.append(('Extra Trees tuned', ExtraTreesRegressor(random_state=0,
                                                 bootstrap = random_et_best_param['bootstrap'], 
                                                 max_leaf_nodes =random_et_best_param['max_leaf_nodes'], 
                                                 max_features =random_et_best_param['max_features'],
                                                 max_depth=random_et_best_param['max_depth'],
                                                 criterion=random_et_best_param['criterion'],
                                                 ccp_alpha=random_et_best_param['ccp_alpha']
                                                 ,n_estimators=random_et_best_param['n_estimators'])))
#Apply Model
results = []
names = []
scores = []
train_errors = []
test_errors = []
r_squareds =[]
train_errors_absolute = []
test_errors_absolute = []
variance_scores =[]
RMS = 'neg_mean_squared_error'
for name, model in models:
    kfold = KFold()
    cv_results = cross_val_score(model, X_train, Y_train, cv=kfold, scoring=RMS)
    model.fit(X_train,Y_train)
    score = model.score(X_train,Y_train)
    train_error = mean_squared_error(Y_train, model.predict(X_train))
    y_pred = model.predict(X_test)
    test_error = mean_squared_error(Y_test, y_pred)
    r_squared = r2_score(Y_test, y_pred)
    train_error_absolute = mean_absolute_error(Y_train, model.predict(X_train))
    test_error_absolute = mean_absolute_error(Y_test, y_pred)
    variance_score=explained_variance_score(Y_test,y_pred)

    results.append(cv_results)
    r_squareds.append(r_squared)
    names.append(name)
    scores.append(score)
    train_errors.append(train_error)
    test_errors.append(test_error)
    train_errors_absolute.append(train_error_absolute)
    test_errors_absolute.append(test_error_absolute)         
    variance_scores.append(variance_score)
results_table_tuned = pd.concat([pd.DataFrame([[names[i], scores[i], variance_scores[i], train_errors[i], test_errors[i], 
                                         train_errors_absolute[i], test_errors_absolute[i]]], 
                              columns = ["Models","Score", "Variance Score", "Train set mse", 
                                         "Test set mse", "Train set mae", "Test set mae"]) for i in range(len(models))],ignore_index = True)

results_table_tuned = results_table.sort_values(by=('Score'),ascending=False)

results_table_tuned.style.background_gradient(cmap='Blues')
results_table<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13021163139476755, 'fraction_numerical': 0.03912681219796701, 'mean_word_length': 4.335348506401138, 'pattern_counts': {'":': 2, '<': 156, '<?xml version=': 0, '>': 165, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 50, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '40', 'dir': 'jupyter-structured-clean-dedup', 'id': '158585', 'license': 'MIT', 'n_tokens_mistral': 10980, 'n_tokens_neox': 9875, 'n_words': 2066, 'path': 'CS_BDF_VF.ipynb', 'repo_name': 'VeronikaFanous/BDF_Model', 'size': '30005'}