<jupyter_start><jupyter_text>Stochastic Gradient Descent<jupyter_code>import numpy as np
m = 100000

x = np.random.normal(size=m)
X = x.reshape(-1, 1)
y = 4. * x + 3. + np.random.normal(0, 3, size=m)<jupyter_output><empty_output><jupyter_text>Batch Gradient Descent<jupyter_code>def J(theta, X_b, y):
    try:
        return np.sum((y - X_b.dot(theta)) ** 2) / len(X_b)
    except:
        return float('inf')
    
def dJ(theta, X_b, y):
    return X_b.T.dot(X_b.dot(theta) - y) * 2 /len(X_b)


def gradient_descent(X_b, y, initial_theta, eta, n_iters=1e4, epsilon=1e-8):
    theta = initial_theta
    i_ters = 0

    while i_ters < n_iters:
        gradient = dJ(theta, X_b, y)
        last_theta = theta

        theta = theta - eta * gradient
        
        if (abs(J(theta, X_b, y) - J(last_theta, X_b, y)) < epsilon):
            break
        
        i_ters += 1
        
    return theta
%%time
X_b = np.hstack([np.ones((len(X), 1)), X])
initial_theta = np.zeros(X_b.shape[1])
eta = 0.01

theta = gradient_descent(X_b, y, initial_theta, eta)
theta<jupyter_output><empty_output><jupyter_text>Stochastic Gradient Descent<jupyter_code>def dJ_sgd(theta, X_b_i, y_i):
    return X_b_i.T.dot(X_b_i.dot(theta) - y_i) * 2.
def sgd(X_b, y, initial_theta, n_iters=1e4):
    t0 = 5
    t1 = 50
    
    def learning_rate(t):
        return t0 / (t + t1)
    
    theta = initial_theta
    for cur_iter in range(n_iters):
        rand_i = np.random.randint(len(X_b))
        grandient = dJ_sgd(theta, X_b[rand_i], y[rand_i])
        theta = theta - learning_rate(cur_iter) * grandient
        
    return theta
%%time
X_b = np.hstack([np.ones((len(X), 1)), X])
initial_theta = np.zeros(X_b.shape[1])

theta = sgd(X_b, y, initial_theta, n_iters=len(X_b)//3)
theta<jupyter_output><empty_output><jupyter_text>Implement Stochastic Gradient Descent in our Linear Regression<jupyter_code>from LinearRegression import LinearRegression
lin_reg = LinearRegression()
lin_reg.fit_sgd(X, y, n_iters=2)
lin_reg.coef_
lin_reg.interception_
from sklearn import datasets
boston = datasets.load_boston()
X = boston.data
y = boston.target

X = X[y < 50]
y = y[y < 50]
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=666)
from sklearn.preprocessing import StandardScaler
standardScaler = StandardScaler()
standardScaler.fit(X_train)
X_train_standard = standardScaler.transform(X_train)
X_test_standard = standardScaler.transform(X_test)
lin_reg1 = LinearRegression()
%time lin_reg1.fit_sgd(X_train_standard, y_train, n_iters=2)
lin_reg1.score(X_test_standard, y_test)
lin_reg2 = LinearRegression()
%time lin_reg2.fit_sgd(X_train_standard, y_train, n_iters=50)
lin_reg2.score(X_test_standard, y_test)
lin_reg3 = LinearRegression()
%time lin_reg3.fit_sgd(X_train_standard, y_train, n_iters=200)
lin_reg3.score(X_test_standard, y_test)<jupyter_output>CPU times: user 347 ms, sys: 3.66 ms, total: 351 ms
Wall time: 355 ms<jupyter_text>Scikit Learn 中的 SGD<jupyter_code>from sklearn.linear_model import SGDRegressor
sgd_reg = SGDRegressor()
%time sgd_reg.fit(X_train_standard, y_train)
sgd_reg.score(X_test_standard, y_test)
sgd_reg = SGDRegressor(n_iter=100)
%time sgd_reg.fit(X_train_standard, y_train)
sgd_reg.score(X_test_standard, y_test)<jupyter_output>CPU times: user 6.94 ms, sys: 1.69 ms, total: 8.63 ms
Wall time: 6.65 ms
{'fraction_non_alphanumeric': 0.12041420118343195, 'fraction_numerical': 0.02603550295857988, 'mean_word_length': 4.470873786407767, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '637994', 'license': 'MIT', 'n_tokens_mistral': 1471, 'n_tokens_neox': 1391, 'n_words': 277, 'path': '04Gradient-Descent-and-Linear-Regressiion/03Stochastic-Gradient-Descent.ipynb', 'repo_name': 'ovtenng/ML-Algorithms-Action', 'size': '3380'}