<jupyter_start><jupyter_text>Семинар 4 - Линейные модели<jupyter_code>import pandas as pd
import numpy as np
%matplotlib inline 
import seaborn as sns
import matplotlib.pyplot as plt
sns.set(palette='deep', style='darkgrid', rc={'figure.figsize': (15, 4)})
import scipy.stats as st 

import warnings 
warnings.simplefilter('ignore')<jupyter_output><empty_output><jupyter_text>Сгенерируем исскуственные данные<jupyter_code>def function(x):
    return 3*x + 4

x_true = np.array([-2, 2])
y_true = function(x_true)
plt.plot(x_true, y_true, linewidth=1);
n = 100
x = np.random.rand(n, 1)*4-2
noise = np.random.rand(n, 1)*4-2
y = function(x) + noise
plt.scatter(x, y, color='r')
plt.plot(x_true, y_true, color='g', linewidth=1)
plt.show()<jupyter_output><empty_output><jupyter_text>Аналитический метод $$\hat \theta = \bigl(X^T \cdot X  \bigr)^{-1} \cdot X^T \cdot y $$<jupyter_code>x_matrix = np.c_[np.ones((n, 1)), x]
%%time
thetha_matrix = np.linalg.inv(x_matrix.T.dot(x_matrix)).dot(x_matrix.T).dot(y)
thetha_matrix.T[0].tolist()
print("Свободный член: {[0][0]:.7}".format(thetha_matrix.T))
print("Коэфициент: {[0][1]:.7}".format(thetha_matrix.T))
from sklearn.metrics import mean_squared_error
from sklearn.linear_model import LinearRegression
%%time 
lr = LinearRegression()
lr.fit(x, y)
print("Свободный член: {:.7}".format(lr.intercept_[0]))
print("Коэфициент: {:.7}".format(lr.coef_[0][0]))
plt.scatter(x, y, color='g')
plt.scatter(x, lr.predict(x), color='r')
plt.plot(x_true, y_true, linewidth=1)
plt.show()<jupyter_output><empty_output><jupyter_text>Пакетный градиентный спуск $$\nabla MSE(\theta)= \frac{2}{l} X^T \cdot \bigl(X \cdot \theta - y \bigr) $$ Реализация в numpy<jupyter_code>from tqdm import tqdm
learning_rate = 0.1
n_iterations = 100

theta = np.random.randn(2, 1)

plt.scatter(x, y, color='g')
for iteration in tqdm(range(n_iterations)):
    if iteration < 10:
        plt.plot(x_true, x_true*theta[1]+theta[0], linewidth=1, color='r')
    gradients = 2/n * x_matrix.T.dot(x_matrix.dot(theta) - y)
    theta -= learning_rate * gradients
    
plt.plot(x_true, y_true, linewidth=1)
plt.show()

print(theta.T[0].tolist())<jupyter_output>100%|██████████| 100/100 [00:00<00:00, 3994.84it/s]<jupyter_text>Слишком маленький шаг обучения (learning rate)<jupyter_code>learning_rate = 0.01
n_iterations = 100

theta = np.random.randn(2, 1)

plt.scatter(x, y, color='g')
for iteration in tqdm(range(n_iterations)):
    if iteration < 10:
        plt.plot(x_true, x_true*theta[1]+theta[0], linewidth=1, color='r')
    gradients = 2/n * x_matrix.T.dot(x_matrix.dot(theta) - y)
    theta -= learning_rate * gradients
    
plt.plot(x_true, y_true, linewidth=1)
plt.show()

print(theta.T[0].tolist())<jupyter_output>100%|██████████| 100/100 [00:00<00:00, 3468.37it/s]<jupyter_text>Слишком большой шаг обучения (learning rate)<jupyter_code>learning_rate = 1.01
n_iterations = 100

theta = np.random.randn(2, 1)

plt.scatter(x, y, color='g')
for iteration in tqdm(range(n_iterations)):
    if iteration < 10:
        plt.plot(x_true, x_true*theta[1]+theta[0], linewidth=1, color='r')
    gradients = 2/n * x_matrix.T.dot(x_matrix.dot(theta) - y)
    theta -= learning_rate * gradients
    
plt.plot(x_true, y_true, linewidth=1)
plt.show()

print(theta.T[0].tolist())<jupyter_output>100%|██████████| 100/100 [00:00<00:00, 3580.50it/s]<jupyter_text>Уменьшение шага на каждой итерации<jupyter_code>learning_rate = 1
n_iterations = 1000

theta = np.random.randn(2, 1)

for iteration in range(n_iterations):
    gradients = 2/n * x_matrix.T.dot(x_matrix.dot(theta) - y)
    theta -= (learning_rate/(iteration + 1)) * gradients
    
print(theta.T.tolist())<jupyter_output>[[4.110530909743645, 3.160412789435898]]<jupyter_text>Learning rate - гипперпараметр, и можно воспользоваться GridSearchCV, однако чтобы не учить каждый раз такое кол-во итераций, мы можем измерять норму градиента, и прекращать спуск, когда он "затух"<jupyter_code>learning_rate = 0.1
n_iterations = 1000
tol = 0.01

theta = np.random.randn(2, 1)

for iteration in range(n_iterations):
    gradients = 2/n * x_matrix.T.dot(x_matrix.dot(theta) - y)
    if np.linalg.norm(gradients) < tol:
        break
    theta -= learning_rate * gradients
    
print('Градиент затух на {} итерации '.format(iteration))
print(theta.T.tolist())<jupyter_output>Градиент затух на 30 итерации 
[[4.10641211182658, 3.160227709989504]]<jupyter_text>__Реализация в Scikit-Learn отсутствует__ Cтохастический градиентный спуск<jupyter_code>n_epochs = 1000

def learning_schedule(t):
    return t0 / (t + t1)

t0 = 5
t1 = 100  

theta = np.random.randn(2,1)  # random initialization

for epoch in range(n_epochs):
    for i in range(n):
        random_index = np.random.randint(n)
        xi = x_matrix[random_index:random_index+1]
        yi = y[random_index:random_index+1]
        gradients = 2 * xi.T.dot(xi.dot(theta) - yi)
        eta = learning_schedule(epoch * n + i)
        theta = theta - eta * gradients
print(theta)
from sklearn.linear_model import SGDRegressor
sgd = SGDRegressor(tol=0.0001)
sgd.fit(x, y)
sgd.intercept_, sgd.coef_<jupyter_output><empty_output><jupyter_text>Полиномиальные модели$y = \cos(x)$, $x \in [-5,5]  $     $y = \cos(x) + \varepsilon$, где  $\varepsilon = \mathscr{N} \bigl(0,\frac{1}{2} \bigr)$, $x \in [-5,5]$<jupyter_code>n = 100
n2 = 20
x = np.linspace(-5, 5, num=n)
x_dt = np.linspace(-5, 5, num=n2)
y_true = np.cos(x)
noise = np.random.sample(n2)/2-0.25
y_noised = np.cos(x_dt) + noise
noise
plt.scatter(x_dt, y_noised, label="Прецеденты (Зашумленный Cos)")
plt.plot(x, y_true, color='red', label='Cos(x)')
plt.legend()
plt.show()
from sklearn.preprocessing import PolynomialFeatures
LR = LinearRegression()
poly = PolynomialFeatures(12)
x_poly2 = poly.fit_transform(x_dt.reshape(-1, 1))
LR.fit(x_poly2, y_noised)
pred = LR.predict(poly.transform(np.linspace(-5, 5, num=n).reshape(-1, 1)))
pred_poly_11 = LR.predict(x_poly2)
plt.scatter(x_dt,y_noised,  label="Прецеденты (Зашумленный Cos)")
plt.plot(x,y_true, color='red', label="Cos(x)")
plt.plot(np.linspace(-5,5,num=n),pred, color='green', label="Восстановленая функция")
plt.ylim([-1,1])
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text>Регуляризация моделей<jupyter_code>from sklearn.linear_model import Lasso, Ridge, ElasticNet
LR = Lasso()
poly = PolynomialFeatures(15)
x_poly2 = poly.fit_transform(x_dt.reshape(-1, 1))
LR.fit(x_poly2, y_noised)
pred = LR.predict(poly.transform(np.linspace(-5,5,num=n).reshape(-1, 1)))
pred_poly_11 = LR.predict(x_poly2)
plt.scatter(x_dt,y_noised,  label="Прецеденты (Зашумленный Cos)")
plt.plot(x,y_true, color='red', label="Cos(x)")
plt.plot(np.linspace(-5,5,num=n),pred, color='green', label="Восстановленая функция")
plt.ylim([-1.5,1.5])
plt.legend()
plt.show()
pd.DataFrame(np.round(LR.coef_, decimals=2), columns=['Feature Coef']).T<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16790447065676423, 'fraction_numerical': 0.046454055628367553, 'mean_word_length': 5.412698412698413, 'pattern_counts': {'":': 0, '<': 47, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '398125', 'license': 'MIT', 'n_tokens_mistral': 3182, 'n_tokens_neox': 2989, 'n_words': 513, 'path': 'Course_1/Seminar_4_MIPT_DAFE.ipynb', 'repo_name': 'LinarAbdrazakov/LearningML', 'size': '6867'}