<jupyter_start><jupyter_text>Binning / Discretization<jupyter_code>import mglearn
X, y = mglearn.datasets.make_wave(n_samples=40)   #This data has only one independent column. Easier to explain concepts
print(X.shape)
print(y.shape)
print('X values :\n',pd.DataFrame(X).head())
print('y values :\n',pd.DataFrame(y).head())
from sklearn.linear_model import LinearRegression   #We will compare the linear model with decision model on this data
from sklearn.tree import DecisionTreeRegressor
X , y = mglearn.datasets.make_wave(n_samples=120)

line = np.linspace(-3, 3 , 1000, endpoint = False) # generate list of 1000 numbers from -3 upto 3 but not including 3 
line = line.reshape(-1,1) # converts to array of numbers
Dtreg = DecisionTreeRegressor(min_samples_leaf = 3).fit(X,y)
predicted = Dtreg.predict(line)
plt.plot(line, predicted, label="Decision Tree")

linreg = LinearRegression().fit(X,y)
linpredicted = linreg.predict(line)
plt.plot(line, linpredicted, label = "Linear Regression" )

plt.plot(X[:,0], y, 'o', c='k')
plt.ylabel("Regression Output")
plt.xlabel("Input Features")
plt.legend(loc = 'best')
print(linreg.score(X, y))
print(Dtreg.score(X, y))
# Linear models (orange line) can build only simple models but decision trees can build simple to complex models
# In case we have to use linear models, we can make them more powerful by changing the data representation 
# one way to make linear models more powerful is binning the data


from sklearn.preprocessing import KBinsDiscretizer

kb = KBinsDiscretizer(n_bins=10, strategy='uniform')

kb.fit(X)

print("bin edges \n", kb.bin_edges_)


# With the bins defined, we can transform each data point X into a bin using the transform function

X_binned = kb.transform(X)
X_binned
print(X_binned.toarray()[0,:])
print(pd.DataFrame(X).head(1))   #First data point -0.752759 is stored in bin 4
# To prevent sparse matrix and create dense matrix, let us re-do the binning with onehot coding. A given input belongs to
# one bin and not others. Hence, we can onehot code them

kb = KBinsDiscretizer( n_bins = 10, strategy ='uniform', encode ='onehot-dense') 
kb.fit(X) 
X_binned = kb.transform(X)
X_binned[0:10]
# bin the input line data too which is our test data

line_binned = kb.transform(line)

linreg = LinearRegression().fit(X_binned, y)
predicted = linreg.predict(line_binned)
plt.plot(line, predicted, label = 'Linear Regressor combined')

#Dtreg = DecisionTreeRegressor(min_samples_split = 3).fit(X_binned,y)
#predicted = Dtreg.predict(line_binned)
#plt.plot(line, predicted, label="Decision Tree combined")


plt.plot(X[:,0], y, 'o', c='k')       # plotting origial X Vs y. The slope of line may not match coeff
plt.vlines(kb.bin_edges_[0], -3,3, linewidth=1, alpha=.2)
plt.ylabel("Regression Output")
plt.xlabel("Input Features")
plt.legend(loc = 'best')
print(linreg.score(X_binned, y))
print(Dtreg.score(X_binned, y))   # Performance of DT has gone down because now it is built on bins not original data
# The decision tree boundaries and linear regression boundaries are exactly same. They are overlapping. For each bin they
# are predicting a constant value. To convince, comment out the plot for Decision Tree... You will see the regression line
# Comparing the linear models before binning and after binning, we can see that the linear model has become more flexible.
print("coefficients for each bin \n", linreg.coef_ ) #coefficients for each bin.
print("")
print("intercept\n", linreg.intercept_)
# compare the linear regression scores before and after binning<jupyter_output><empty_output><jupyter_text>Adding slope to the linear model in a bin<jupyter_code>X_combined = np.hstack([X, X_binned])
line_combined = np.hstack([line, line_binned])
X_combined[0:10]
print('X_Combined shape :', X_combined.shape)  #from 10 onehotcoded columns, it has become 11
linreg = LinearRegression().fit(X_combined, y)
predicted = linreg.predict(line_combined)
plt.plot(line, predicted, label = 'Linear Regressor combined')

Dtreg = DecisionTreeRegressor(min_samples_leaf = 3).fit(X_combined,y)
predicted = Dtreg.predict(line_combined)
plt.plot(line, predicted, label="Decision Tree combined")


plt.plot(X[:,0], y, 'o', c='k')
plt.vlines(kb.bin_edges_[0], -3,3, linewidth=1, alpha=.2)
plt.ylabel("Regression Output")
plt.xlabel("Input Features")
plt.legend(loc = 'best')
# linear regression model now has learnt a slope but the slope is same in all the bins (blue line)
print("coefficients for each bin \n", linreg.coef_ ) #coefficients for each bin.
print("")
print("intercept\n", linreg.intercept_)
print(linreg.score(X_combined, y))
print(Dtreg.score(X_combined, y)) 
# Let us create an interaction feature by using simple multiple operation 
X_product = np.hstack([X_binned, X_binned * X])


print(X_product.shape)

line_product = np.hstack([line_binned, line_binned * line])


print(X_binned[1])
print(X[1])
print(X_binned[1] * X[1])
X_product[0]
linreg = LinearRegression().fit(X_product, y)
predicted = linreg.predict(line_product)
plt.plot(line, predicted, label = 'Linear Regressor product of features')

Dtreg = DecisionTreeRegressor(min_samples_split = 5).fit(X_product,y)
predicted = Dtreg.predict(line_product)
plt.plot(line, predicted, label="Decision Tree product")


plt.plot(X[:,0], y, 'o', c='k')    # Note: we are plotting original X Vs Y, hence the model slope may not reflect coeff
plt.vlines(kb.bin_edges_[0], -3,3, linewidth=1, alpha=.2)
plt.ylabel("Regression Output")
plt.xlabel("Input Features")
plt.legend(loc = 'best')
# Each bin has it's own offset and its own coefficient.
# Each bin has it's own offset and coefficient

print("coefficients for each bin \n", linreg.coef_ ) #coefficients for each bin.
print("")
print("intercept\n", linreg.intercept_)
print(linreg.score(X_product, y))
print(Dtreg.score(X_product, y))<jupyter_output>0.787287089329761
0.9359711056059232<jupyter_text>Polynomial feature transformation<jupyter_code>from sklearn.preprocessing import PolynomialFeatures

poly = PolynomialFeatures(degree=15, include_bias=False)
poly.fit(X)
X_poly = poly.transform(X)

line_poly = poly.transform(line)
print("X_poly shape: ", X_poly.shape)
# Let us see the difference in the X and X_poly entries

print("Records in X \n", X[0])
print("Records in X_poly \n",X_poly[0])
# the columns are x, x^2, x ^3

print("X_Poly feature names : " , poly.get_feature_names())
linreg = LinearRegression().fit(X_poly, y)
predicted = linreg.predict(line_poly)
plt.plot(line, predicted, label = 'Linear Regressor product of features')

Dtreg = DecisionTreeRegressor(min_samples_split = 5).fit(X_poly,y)
predicted = Dtreg.predict(line_poly)
plt.plot(line, predicted, label="Decision Tree product")


plt.plot(X[:,0], y, 'o', c='k')    # Note: we are plotting original X Vs Y, hence the model slope may not reflect coeff
plt.vlines(kb.bin_edges_[0], -3,3, linewidth=1, alpha=.2)
plt.ylabel("Regression Output")
plt.xlabel("Input Features")
plt.legend(loc = 'best')
print(linreg.score(X_poly, y))
print(Dtreg.score(X_poly, y))
# Though polynomial features give a smoother and fitting curve, the problem with such features is the behaviour of the model 
# at the extreme values or in regions where data is scant. This could lead to variance errors. If we have to generate polynomial 
# features, SVM /SVR may be a natural choice as SVR generates features using kernels automatically<jupyter_output><empty_output><jupyter_text>SVM generated features<jupyter_code>from sklearn.svm import SVR
for gamma in [1,10]:
    svr = SVR(gamma = gamma).fit(X, y)   # default degree = 3 
    predicted = svr.predict(line)
    plt.plot(line, predicted, label = 'SVR gamma={}'.format(gamma ))
    
plt.plot(X[:,0], y, 'o', c='k')    # Note: we are plotting original X Vs Y, hence the model slope may not reflect coeff
plt.vlines(kb.bin_edges_[0], -3,3, linewidth=1, alpha=.2)
plt.ylabel("Regression Output")
plt.xlabel("Input Features")
plt.legend(loc = 'best')
print(svr.score(X, y))  #SVR gives same peformance at degree 3 gamma 5<jupyter_output>0.7872376732593508<jupyter_text>Regularizing linear models to select features<jupyter_code>svr.get_params()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1195238679592588, 'fraction_numerical': 0.01803902319302982, 'mean_word_length': 5.735537190082645, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '468278', 'license': 'MIT', 'n_tokens_mistral': 2912, 'n_tokens_neox': 2726, 'n_words': 845, 'path': 'Practice_Notebooks/.ipynb_checkpoints/Feature_Transform-checkpoint.ipynb', 'repo_name': 'Aujasvi-Moudgil/Ensemble-Learning', 'size': '8149'}