<jupyter_start><jupyter_text>Linear Regression Theory<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline

X = 2* np.random.rand(100,1)
y = 4+3 * X + np.random.randn(100,1)
plt.scatter(x=X, y=y )
X_b = np.c_[np.ones((100,1)), X]
theta_best = np.linalg.inv(X_b.T.dot(X_b)).dot(X_b.T).dot(y)
theta_best
X_new = np.array([[0],[2]])
X_new_b = np.c_[np.ones((2,1)), X_new]
y_predict = X_new_b.dot(theta_best)
y_predict
plt.plot(X_new,y_predict, "r-")
plt.plot(X,y,"b.")
plt.axis([0,2,0,15])
from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()

lin_reg.fit(X,y)
lin_reg.predict(X_new)<jupyter_output><empty_output><jupyter_text>Gradient Descent Theory  Learning Curves<jupyter_code>m = 100
X = 6 * np.random.rand(m,1) -3
y = 0.5 * X**2 + X + 2 + np.random.randn(m,1)
plt.scatter(X,y)
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import train_test_split

def plot_learning_curves(model, X, y):
    X_train, X_val, y_train, y_val = train_test_split(X,y,test_size=0.2)
    train_errors, val_errors = [], []
    for m in range(1,len(X_train)):
        model.fit(X_train[:m], y_train [:m])
        
        #Train on incrementally increasing training set
        y_train_predict = model.predict(X_train[:m])
        
        #Always test on entire val set
        y_val_predict = model.predict(X_val)
        
        train_errors.append(mean_squared_error(y_train_predict, y_train[:m]))
        val_errors.append(mean_squared_error(y_val_predict, y_val))
        
    plt.plot(np.sqrt(train_errors), "r-+", linewidth=2, label="train" )
    plt.plot(np.sqrt(val_errors), "b-", linewidth=3, label="val")
    plt.legend()
from sklearn.linear_model import LinearRegression

lin_reg = LinearRegression()
plot_learning_curves(lin_reg, X,y)<jupyter_output><empty_output><jupyter_text>This curve illustrates what an underfitting model looks like. Adding additional data would not reduce errors.  Use a 10th degree polynomial this time<jupyter_code>from sklearn.pipeline import Pipeline
from sklearn.preprocessing import PolynomialFeatures

polynomial_regression = Pipeline([
    ("poly_features", PolynomialFeatures(degree=10, include_bias=False)),
    ("lin_reg", LinearRegression())
])

plot_learning_curves(polynomial_regression, X, y)<jupyter_output><empty_output><jupyter_text>Regularized linear models Ridge Regression using closed form solution<jupyter_code>from sklearn.linear_model import Ridge

ridge_reg = Ridge(alpha=1, solver="cholesky")
ridge_reg.fit(X,y)
ridge_reg.predict([[1.5]])
from sklearn.linear_model import SGDRegressor

sgd_reg = SGDRegressor(penalty="l2")
sgd_reg.fit(X,y)
sgd_reg.predict([[1.5]])<jupyter_output>/home/edwardchaos/.local/share/virtualenvs/TF_py3/lib/python3.5/site-packages/sklearn/linear_model/stochastic_gradient.py:128: FutureWarning: max_iter and tol parameters have been added in <class 'sklearn.linear_model.stochastic_gradient.SGDRegressor'> in 0.19. If both are left unset, they default to max_iter=5 and tol=None. If tol is not None, max_iter defaults to max_iter=1000. From 0.21, default max_iter will be 1000, and default tol will be 1e-3.
  "and default tol will be 1e-3." % type(self), FutureWarning)
/home/edwardchaos/.local/share/virtualenvs/TF_py3/lib/python3.5/site-packages/sklearn/utils/validation.py:578: DataConversionWarning: A column-vector y was passed when a 1d array was expected. Please change the shape of y to (n_samples, ), for example using ravel().
  y = column_or_1d(y, warn=True)<jupyter_text>Lasso Regression<jupyter_code>from sklearn.linear_model import Lasso
lasso_reg = Lasso(alpha=0.1)
lasso_reg.fit(X,y)
lasso_reg.predict([[1.5]])
from sklearn.linear_model import ElasticNet
elastic_net = ElasticNet(alpha=0.1, l1_ratio = 0.5)
elastic_net.fit(X,y)
elastic_net.predict([[1.5]])<jupyter_output><empty_output><jupyter_text>Early Stopping; A Regularization Technique<jupyter_code>from sklearn.base import clone
from sklearn.preprocessing import StandardScaler

#prepare the data
poly_scaler = Pipeline([
    ("poly_features", PolynomialFeatures(degree = 90, include_bias=False)),
    ("std_scaler", StandardScaler())
])

X_train, X_val, y_train, y_val = train_test_split(X,y,test_size=0.2)

X_train_poly_scaled = poly_scaler.fit_transform(X_train)
X_val_poly_scaled = poly_scaler.fit_transform(X_val)

sgd_reg = SGDRegressor(n_iter=1, warm_start = True, penalty=None,
                      learning_rate = "constant", eta0=0.0005)

minimum_val_error = float('inf')
best_epoch = None
best_model = None
for epoch in range(1000):
    #Fit continues training from where it left off since 'warm_start'
    #is set to true
    sgd_reg.fit(X_train_poly_scaled,y_train)
    y_predictions = sgd_reg.predict(X_val_poly_scaled)
    val_error = mean_squared_error(y_val, y_predictions)
    if val_error < minimum_val_error:
        best_epoch = epoch
        best_model = clone(sgd_reg)
    

best_model<jupyter_output><empty_output><jupyter_text>Decision Boundaries with IRIS EXAMPLE<jupyter_code>from sklearn import datasets

iris = datasets.load_iris()
list(iris.keys())
X = iris['data'][:,3:] #Only use pedal width to keep it simple
y = (iris['target'] == 2) #Only classify Virginica or not Virginica
from sklearn.linear_model import LogisticRegression

log_reg = LogisticRegression()
log_reg.fit(X,y)
X_new = np.linspace(0,3,1000).reshape(-1,1)
y_proba = log_reg.predict_proba(X_new)
print(y_proba[1])
print(log_reg.classes_)
#plt.plot(X_new,y_proba)
plt.plot(X_new, y_proba[:,0], "b--", label="Not Virginica")
plt.plot(X_new, y_proba[:,1], "g-", label="Virginica")
plt.legend("center right")
plt.show()
log_reg.predict([[1.7], [1.5]])<jupyter_output><empty_output><jupyter_text>Softmax Regression<jupyter_code>X = iris["data"][:,(2,3)] #using pedal length and width
y = iris["target"]

softmax_reg = LogisticRegression(multi_class="multinomial", solver="lbfgs", C=10)
softmax_reg.fit(X,y)
softmax_reg.predict([[5,2]])
softmax_reg.classes_
softmax_reg.predict_proba([[5,2]])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12683975525053745, 'fraction_numerical': 0.022159748635687117, 'mean_word_length': 5.8805460750853245, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '235357', 'license': 'Apache-2.0', 'n_tokens_mistral': 2370, 'n_tokens_neox': 2272, 'n_words': 440, 'path': 'Ch4_Training_models_code_along/Linear Regression Theory.ipynb', 'repo_name': 'edwardchaos/Hands-on-ML-with-Sklearn-and-TensorFlow', 'size': '6047'}