coding utf8 basic class for implementing continuous regression estimators which are trained with gradient descent optimization on their particular loss function parameters lr float default 0 001 learning rate penalty str l1 l2 none default none regularization function name c float default 0 01 the regularization coefficient tolerance float default 0 0001 if the gradient descent updates are smaller than tolerance then stop optimization process maxiters int default 10000 the maximum number of iterations apply regularization to the loss if self penalty l1 loss self c np absw1 sum elif self penalty l2 loss 0 5 self c w1 2 sum return loss def costself x y theta prediction x dottheta error self costfuncy prediction return error def fitself x ynone self setupinputx y self initcost self nsamples self nfeatures x shape initialize weights bias term self theta np random normalsizeself nfeatures 1 scale0 5 add an intercept column self x self addinterceptself x self train staticmethod def addinterceptx b np onesx shape0 1 return np concatenateb x axis1 def trainself self theta self errors self gradientdescent logging info theta s self theta flatten def predictself xnone x self addinterceptx return x dotself theta def gradientdescentself theta self theta errors self costself x self y theta get derivative of the loss function costd gradself loss for i in range1 self maxiters 1 calculate gradient and update theta delta costdtheta theta self lr delta errors appendself costself x self y theta logging infoiteration s error s i errorsi errordiff np linalg normerrorsi 1 errorsi if errordiff self tolerance logging infoconvergence has reached break return theta errors class linearregressionbasicregression binary logistic regression with gradient descent optimizer def initcostself self costfunc binarycrossentropy def lossself w loss self costfuncself y self sigmoidnp dotself x w return self addpenaltyloss w staticmethod def sigmoidx return 0 5 np tanh0 5 x 1 def predictself xnone x self addinterceptx return self sigmoidx dotself theta coding utf 8 basic class for implementing continuous regression estimators which are trained with gradient descent optimization on their particular loss function parameters lr float default 0 001 learning rate penalty str l1 l2 none default none regularization function name c float default 0 01 the regularization coefficient tolerance float default 0 0001 if the gradient descent updates are smaller than tolerance then stop optimization process max_iters int default 10000 the maximum number of iterations apply regularization to the loss initialize weights bias term add an intercept column get derivative of the loss function calculate gradient and update theta linear regression with gradient descent optimizer binary logistic regression with gradient descent optimizer
import logging

import autograd.numpy as np
from autograd import grad

from mla.base import BaseEstimator
from mla.metrics.metrics import mean_squared_error, binary_crossentropy

np.random.seed(1000)


class BasicRegression(BaseEstimator):
    def __init__(self, lr=0.001, penalty="None", C=0.01, tolerance=0.0001, max_iters=1000):
        
        self.C = C
        self.penalty = penalty
        self.tolerance = tolerance
        self.lr = lr
        self.max_iters = max_iters
        self.errors = []
        self.theta = []
        self.n_samples, self.n_features = None, None
        self.cost_func = None

    def _loss(self, w):
        raise NotImplementedError()

    def init_cost(self):
        raise NotImplementedError()

    def _add_penalty(self, loss, w):
        
        if self.penalty == "l1":
            loss += self.C * np.abs(w[1:]).sum()
        elif self.penalty == "l2":
            loss += (0.5 * self.C) * (w[1:] ** 2).sum()
        return loss

    def _cost(self, X, y, theta):
        prediction = X.dot(theta)
        error = self.cost_func(y, prediction)
        return error

    def fit(self, X, y=None):
        self._setup_input(X, y)
        self.init_cost()
        self.n_samples, self.n_features = X.shape

        
        self.theta = np.random.normal(size=(self.n_features + 1), scale=0.5)

        
        self.X = self._add_intercept(self.X)

        self._train()

    @staticmethod
    def _add_intercept(X):
        b = np.ones([X.shape[0], 1])
        return np.concatenate([b, X], axis=1)

    def _train(self):
        self.theta, self.errors = self._gradient_descent()
        logging.info(" Theta: %s" % self.theta.flatten())

    def _predict(self, X=None):
        X = self._add_intercept(X)
        return X.dot(self.theta)

    def _gradient_descent(self):
        theta = self.theta
        errors = [self._cost(self.X, self.y, theta)]
        
        cost_d = grad(self._loss)
        for i in range(1, self.max_iters + 1):
            
            delta = cost_d(theta)
            theta -= self.lr * delta

            errors.append(self._cost(self.X, self.y, theta))
            logging.info("Iteration %s, error %s" % (i, errors[i]))

            error_diff = np.linalg.norm(errors[i - 1] - errors[i])
            if error_diff < self.tolerance:
                logging.info("Convergence has reached.")
                break
        return theta, errors


class LinearRegression(BasicRegression):
    

    def _loss(self, w):
        loss = self.cost_func(self.y, np.dot(self.X, w))
        return self._add_penalty(loss, w)

    def init_cost(self):
        self.cost_func = mean_squared_error


class LogisticRegression(BasicRegression):
    

    def init_cost(self):
        self.cost_func = binary_crossentropy

    def _loss(self, w):
        loss = self.cost_func(self.y, self.sigmoid(np.dot(self.X, w)))
        return self._add_penalty(loss, w)

    @staticmethod
    def sigmoid(x):
        return 0.5 * (np.tanh(0.5 * x) + 1)

    def _predict(self, X=None):
        X = self._add_intercept(X)
        return self.sigmoid(X.dot(self.theta))