<jupyter_start><jupyter_text>Let's start with Exponentially Weighted Averages ![image.png](attachment:image.png)  Generate dataset<jupyter_code>np.random.seed(10)
x = np.linspace(0,10,500)
S = np.sin(x) + np.random.normal(0,0.5,500)
fig, ax = plt.subplots(figsize=(10,8))
ax.scatter(x,S)
ax.set_title("Original data");<jupyter_output><empty_output><jupyter_text>Transforme dataset<jupyter_code>def EWA_transform(S,betta):
    """
    EWA transforme
    """
    V = [0]
    for i in range(1,len(S)):
        V.append((betta*V[-1] + (1-betta)*S[i]))
    return V

V_1 = EWA_transform(S,0.5)
V_2 = EWA_transform(S,0.9)
V_3 = EWA_transform(S,0.99)
fig, ax = plt.subplots(figsize=(10,8))
ax.plot(x, S, '-g', label='Original data')
ax.plot(x, V_1, '-b', label='betta = 0.5')
ax.plot(x, V_2, 'r', label='betta = 0.9')
ax.plot(x, V_3, '-y', label='betta = 0.99')
ax.axis('equal')
leg = ax.legend();<jupyter_output><empty_output><jupyter_text>Select function for optimization ![image.png](attachment:image.png) ![image.png](attachment:image.png) ![image.png](attachment:image.png)<jupyter_code>A = np.array([[1,0],[0,1]])
b = np.array([50,25]).reshape(2,1)

def cost(w,A,b):
    return np.asscalar(1/2*np.dot(np.dot(w.T,A),w) - np.dot(b.T,w)) + np.random.normal(0,0.5)

def cost_grad(w,A,b):
    return (np.dot(A,w) - b)
w_0 = np.dot(np.linalg.inv(A),b)
print("Real Minimum point:",w_0.reshape(1,2))
print("Min value:",cost(w_0,A,b))
X = np.random.randn(1000,2)*1000
Z = np.array([cost(x.reshape(2,1),A,b) for x in X]).reshape(-1,1)
Z_grad = np.array([cost_grad(x.reshape(2,1),A,b) for x in X]).reshape(-1,1)

fig = plt.figure(figsize=(10,8))
ax = fig.gca(projection='3d')

ax.scatter(X[:,0], X[:,1], Z, cmap=cm.coolwarm,
                       linewidth=0, antialiased=False)

ax.set_xticks(w_0[0])
ax.set_yticks(w_0[1]);<jupyter_output><empty_output><jupyter_text>Gradient Descent  ![image.png](attachment:image.png)<jupyter_code>class Gradient_Descent():
    """
    Simple Gradient Descent
    """
    def __init__(self,**kwargs):
        self.cost = kwargs["cost"]
        self.cost_grad = kwargs["cost_grad"]
        self.lr = kwargs["lr"]
        self.epochs = kwargs["epochs"]
        self.A = kwargs["A"]
        self.b = kwargs["b"]
        self.w_0 = kwargs["w_0"]
        self.tau = kwargs["tau"]
        self.history = []
    def optimize(self,X):
        w = np.ones((2,1))
        converg = False
        for i in range(self.epochs):
            if not converg:
                for j in range(X.shape[1]):
                    grad = self.cost_grad(X[:,j].reshape(-1,1),self.A,self.b)
                    w -= self.lr*grad
                    self.history.append(cost(w,self.A,self.b))
                    if np.linalg.norm(w-self.w_0)<self.tau:
                        converg = True
                        print("converge in {0} steps".format(len(self.history)))
                        break
            else:
                break
        return w
def plot_results(optimizer,X,**params):
    opt = optimizer(**params)
    w_min = opt.optimize(X)
    print("Real min point:",params["w_0"].reshape(1,2))
    print("Estimated min point:",w_min.reshape(1,2))
    print("Distance:",np.linalg.norm(w_min-w_0))
    steps = np.linspace(0,len(opt.history),len(opt.history))
    plt.plot(steps,opt.history)
    plt.title("History")
    plt.xlabel("Steps")
    plt.ylabel("Cost")
    plt.show();
def shuffle_along_axis(a, axis):
    idx = np.random.rand(*a.shape).argsort(axis=axis)
    return np.take_along_axis(a,idx,axis=axis)
params = {"cost":cost,
          "cost_grad":cost_grad,
          "lr":0.09,
          "epochs":50,
          "w_0":w_0,
          "A":A,
          "b":b,
          "tau":4
}

# np.random.seed(1)
X = np.arange(-25,25,0.5)
Y = np.arange(-25,25,0.5)
X,Y = np.meshgrid(X,Y)
data = np.vstack([X.ravel(), Y.ravel()])
data = shuffle_along_axis(data,axis=1)

plot_results(Gradient_Descent,data,**params)<jupyter_output>converge in 11 steps
Real min point: [[50. 25.]]
Estimated min point: [[49.015 28.135]]
Distance: 3.286099511579037<jupyter_text>Mini-Batch Stochastic Gradient Descent<jupyter_code>class SGD():
    """
    Stochastic Gradient Descenet 
    
    """
    def __init__(self,**kwargs):
        self.cost = kwargs["cost"]
        self.cost_grad = kwargs["cost_grad"]
        self.lr = kwargs["lr"]
        self.epochs = kwargs["epochs"]
        self.A = kwargs["A"]
        self.b = kwargs["b"]
        self.w_0 = kwargs["w_0"]
        self.tau = kwargs["tau"]
        self.batch_size = kwargs["batch_size"]
        self.step_size = kwargs["step_size"]
        self.history = []
        
        
    def optimize(self,X):
        w = np.zeros((2,1))
        converg = False
        for i in range(self.epochs):
            if not converg:
                for j in range(X.shape[1]):
                    batch = X[:,np.random.randint(0,X.shape[1],self.batch_size)]
                    grad = np.zeros((2,1))
                    for t in range(self.batch_size):
                        grad += self.cost_grad(batch[:,t].reshape(-1,1),self.A,self.b)
                    grad = grad/self.batch_size
                    w -= self.lr*grad
                    self.history.append(cost(w,self.A,self.b))
                    if np.linalg.norm(w-self.w_0)<self.tau:
                        converg = True
                        print("converge in {0} steps".format(len(self.history)))
                        break
            else:
                break
        return w
    

sgd_params = {"cost":cost,
              "cost_grad":cost_grad,
              "lr":0.1,
              "epochs":1000,
              "w_0":w_0,
              "A":A,
              "b":b,
              "tau":1,
              "batch_size":200,
              "step_size":100
}

X = np.arange(-25,25,0.5)
Y = np.arange(-25,25,0.5)
X,Y = np.meshgrid(X,Y)
data = np.vstack([X.ravel(), Y.ravel()])
data = shuffle_along_axis(data,axis=1)

plot_results(SGD,data,**sgd_params)<jupyter_output>converge in 10 steps
Real min point: [[50. 25.]]
Estimated min point: [[49.312   24.87625]]
Distance: 0.6990408160472482<jupyter_text>SGD with momentum ![image.png](attachment:image.png)<jupyter_code>class SGD_momentum():
    """
    Stochastic Gradient Descenet whit momentum
    
    """
    def __init__(self,**kwargs):
        self.cost = kwargs["cost"]
        self.cost_grad = kwargs["cost_grad"]
        self.lr = kwargs["lr"]
        self.betta = kwargs["betta"]
        self.epochs = kwargs["epochs"]
        self.A = kwargs["A"]
        self.b = kwargs["b"]
        self.w_0 = kwargs["w_0"]
        self.tau = kwargs["tau"]
        self.batch_size = kwargs["batch_size"]
        self.step_size = kwargs["step_size"]
        self.history = []
        
        
    def optimize(self,X):
        w = np.zeros((2,1))
        z = np.zeros((2,1))
        converg = False
        for i in range(self.epochs):
            if not converg:
                for j in range(X.shape[1]):
                    batch = X[:,np.random.randint(0,X.shape[1],self.batch_size)]
                    grad = np.zeros((2,1))
                    for t in range(self.batch_size):
                        grad += self.cost_grad(batch[:,t].reshape(-1,1),self.A,self.b)
                    grad = grad/self.batch_size
                    z = self.betta*z + (1-self.betta)*grad
                    w -= self.lr*z
                    self.history.append(cost(w,self.A,self.b))
                    if np.linalg.norm(w-self.w_0)<self.tau:
                        converg = True
                        print("converge in {0} steps".format(len(self.history)))
                        break
            else:
                break
        return w
    

momentum_params = {"cost":cost,
              "cost_grad":cost_grad,
              "lr":0.09,
              "betta":0.8,
              "epochs":1000,
              "w_0":w_0,
              "A":A,
              "b":b,
              "tau":2,
              "batch_size":200,
              "step_size":100
}

X = np.arange(-25,25,0.5)
Y = np.arange(-25,25,0.5)
X,Y = np.meshgrid(X,Y)
data = np.vstack([X.ravel(), Y.ravel()])
data = shuffle_along_axis(data,axis=1)

plot_results(SGD_momentum,data,**momentum_params)<jupyter_output>converge in 15 steps
Real min point: [[50. 25.]]
Estimated min point: [[50.55530241 25.33661064]]
Distance: 0.6493592943988709<jupyter_text>RMS_prop ![image.png](attachment:image.png)<jupyter_code>class RMS_prop():
    """
    RMS propogation
    """
    def __init__(self,**kwargs):
        """
        Init function
        """
        self.cost = kwargs["cost"]
        self.cost_grad = kwargs["cost_grad"]
        self.lr = kwargs["lr"]
        self.betta = kwargs["betta"]
        self.epsilon = kwargs["epsilon"]
        self.epochs = kwargs["epochs"]
        self.A = kwargs["A"]
        self.b = kwargs["b"]
        self.w_0 = kwargs["w_0"]
        self.tau = kwargs["tau"]
        self.batch_size = kwargs["batch_size"]
        self.step_size = kwargs["step_size"]
        self.history = []
        
        
    def optimize(self,X):
        
        w = np.zeros((2,1))
        z = np.zeros((2,1))
        
        converg = False
        for i in range(self.epochs):
            if not converg:
                for j in range(X.shape[1]):
                    batch = X[:,np.random.randint(0,X.shape[1],self.batch_size)]
                    grad = np.zeros((2,1))
                    for t in range(self.batch_size):
                        grad += self.cost_grad(batch[:,t].reshape(-1,1),self.A,self.b)
                    grad = grad/self.batch_size
                    z = self.betta*z + (1-self.betta)*(grad**2)
                    w -= self.lr/np.sqrt(np.mean(z)+self.epsilon)*grad
                    self.history.append(cost(w,self.A,self.b))
                    if np.linalg.norm(w-self.w_0)<self.tau:
                        converg = True
                        print("converge in {0} steps".format(len(self.history)))
                        break
            else:
                break
        return w
rms_prop = {"cost":cost,
              "cost_grad":cost_grad,
              "lr":0.1,
              "betta":0.9,
              "epochs":1000,
              "epsilon":1e-8,
              "w_0":w_0,
              "A":A,
              "b":b,
              "tau":0.2,
              "batch_size":200,
              "step_size":100
}

X = np.arange(-25,25,0.5)
Y = np.arange(-25,25,0.5)
X,Y = np.meshgrid(X,Y)
data = np.vstack([X.ravel(), Y.ravel()])
data = shuffle_along_axis(data,axis=1)

plot_results(RMS_prop,data,**rms_prop)<jupyter_output>converge in 386 steps
Real min point: [[50. 25.]]
Estimated min point: [[49.91693491 25.11229324]]
Distance: 0.13967669494443194<jupyter_text>Adam  Momentum and RMS prop updates ![image.png](attachment:image.png)  Bias correction ![image.png](attachment:image.png)  Update gradients ![image.png](attachment:image.png)<jupyter_code>class Adam():
    """
    Adam
    
    """
    def __init__(self,**kwargs):
        """
        Init function
        """
        self.cost = kwargs["cost"]
        self.cost_grad = kwargs["cost_grad"]
        self.lr = kwargs["lr"]
        self.betta1 = kwargs["betta1"]
        self.betta2 = kwargs["betta2"]
        self.epsilon = kwargs["epsilon"]
        self.epochs = kwargs["epochs"]
        self.A = kwargs["A"]
        self.b = kwargs["b"]
        self.w_0 = kwargs["w_0"]
        self.tau = kwargs["tau"]
        self.batch_size = kwargs["batch_size"]
        self.step_size = kwargs["step_size"]
        self.history = []
        
        
    def optimize(self,X):
        
        w = np.zeros((2,1),dtype = np.float64)
        v = np.zeros((2,1),dtype = np.float64)
        m = np.zeros((2,1),dtype = np.float64)
        converg = False
        for i in range(self.epochs):
            if not converg:
                for j in range(X.shape[1]):
                    
                    batch = X[:,np.random.randint(0,X.shape[1],self.batch_size)]
                    grad = np.zeros((2,1))
                    
                    for t in range(self.batch_size):
                        grad += self.cost_grad(batch[:,t].reshape(-1,1),self.A,self.b)
                    grad = grad/self.batch_size
                    
                    m = self.betta1*m + (1-self.betta1)*(grad)
                    v = self.betta2*v + (1-self.betta2)*(grad**2)

                    m_hat = m/(1-np.power(self.betta1,j+1))
                    v_hat = v/(1-np.power(self.betta2,j+1))

                    w -= self.lr*m_hat/np.sqrt(np.mean(v_hat)+self.epsilon)
                    
                    self.history.append(cost(w,self.A,self.b))
                    if np.linalg.norm(w-self.w_0)<self.tau:
                        converg = True
                        print("converge in {0} steps".format(len(self.history)))
                        break
            else:
                break
        return w
adam_conf = {"cost":cost,
              "cost_grad":cost_grad,
              "lr":0.5,
              "betta1":0.9,
              "betta2":0.9,
              "epochs":1000,
              "epsilon":1e-8,
              "w_0":w_0,
              "A":A,
              "b":b,
              "tau":0.2,
              "batch_size":200,
              "step_size":100
}

X = np.arange(-25,25,0.5)
Y = np.arange(-25,25,0.5)
X,Y = np.meshgrid(X,Y)
data = np.vstack([X.ravel(), Y.ravel()])
data = shuffle_along_axis(data,axis=1)

plot_results(Adam,data,**adam_conf)<jupyter_output>converge in 79 steps
Real min point: [[50. 25.]]
Estimated min point: [[49.94567069 24.97641449]]
Distance: 0.05922794988286252
{'fraction_non_alphanumeric': 0.16198783258814045, 'fraction_numerical': 0.039873928021696106, 'mean_word_length': 1.9073087577242702, 'pattern_counts': {'":': 54, '<': 33, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '452675', 'license': 'MIT', 'n_tokens_mistral': 5140, 'n_tokens_neox': 4866, 'n_words': 750, 'path': 'Deep_Learning/Neural_Networks_Basics/Main_Optimizers.ipynb', 'repo_name': 'GarikAdamyan/ISTC_ML_course', 'size': '13643'}