<jupyter_start><jupyter_text>Logistic regressionOptimize decision boundary with logistic regression. 1. Feedforward\begin{equation}\mathbf{z} = XW\end{equation}\begin{equation}\mathbf{\hat{y}} = \sigma(\mathbf{z})\end{equation}\begin{equation}\sigma(z) = \frac{1}{1+e^{-z}}\end{equation} 2. Compute cost function\begin{equation}L(\mathbf{y}, \mathbf{\hat{y}}) = -\frac{1}{m}\big(\mathbf{y}^T\log(\mathbf{\hat{y}})+(1-\mathbf{y})^T\log(1-\mathbf{\hat{y}})\big)\end{equation} 3. Backpropagation\begin{equation}\frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial W} = \frac{\partial L(\mathbf{y}, \mathbf{\hat{y}})}{\partial \mathbf{\hat{y}}} \cdot \frac{\partial \mathbf{\hat{y}}}{\partial W} = \frac{1}{m}X^T(\mathbf{\hat{y}} - \mathbf{y})\end{equation} 4. Gradient descent\begin{equation}W = W - \alpha  \frac{\delta Loss(y, \hat{y})}{\delta W}\end{equation}<jupyter_code>class LogisticRegression:
    """ Simple logistic regression """
    def __init__(self):
        pass
    
    def _init_params(self, x, y, iterations, learning_rate, reg_factor):
        """ Initilize parameters. 
        
        ----------
        W : ndarray, shape (n_features+1,)
            Coefficient vector
        """
        self._X = np.hstack([np.ones((x.shape[0], 1)), x])
        self._y = y
        self._learning_rate = learning_rate
        self._reg_factor = reg_factor
        self.weights_ = np.random.rand(self._X.shape[1],1)
        self.costs_ = np.zeros(iterations)
        
    def _sigmoid(self, x):
        """ Computes sigmoid function. """
        return 1.0/(1 + np.exp(-x))
    
    def _feedforward(self):
        """ Computes np.dot(X, W). """
        self._y_hat = self._sigmoid(self._X.dot(self.weights_))
        
    def _backprop(self):
        """ Update weights. """
        m = len(self._y)
        # update weights with L2 regularization term
        _weights = self.weights_.copy()
        # ignore bias term
        _weights[0, 0] = 0 
        self.weights_ -= self._learning_rate * self._X.T.dot(self._y_hat - self._y)/m + self._reg_factor/m*_weights
    
    def _get_cost(self):
        """ Compute loss. """
        m = len(self._y)
        # cost function with L2 regularization term
        _weights = self.weights_.copy()
        # ignore bias term
        _weights[0, 0] = 0    
        return -1/m * (self._y.T.dot(np.log(self._y_hat)) + (1-self._y).T.dot(np.log(1-self._y_hat))) + self._reg_factor/(2*m)*_weights.T.dot(_weights)

    def fit(self, x, y, iterations=1000, learning_rate=0.2, reg_factor=0.5):
        """ Fit model.
        
        ----------
        x : ndarray, shape (n_samples, n_features)
            Training data
        y : ndarray, shape (n_samples,)
            Target data
        """
        self._init_params(x, y, iterations, learning_rate, reg_factor)
        
        # train model
        for i in range(iterations):
            self._feedforward()
            self._backprop()
            self.costs_[i] = self._get_cost()
lr = LogisticRegression()
lr.fit(x, y)

new_y_values = -(lr.weights_[0] + lr.weights_[1]* x_values)/lr.weights_[2]

draw_result(x, x_values, new_y_values, lr.costs_[-1])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1747176913425345, 'fraction_numerical': 0.013174404015056462, 'mean_word_length': 2.5631284916201116, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '291621', 'license': 'MIT', 'n_tokens_mistral': 1109, 'n_tokens_neox': 1070, 'n_words': 193, 'path': '006_logistic_regression.ipynb', 'repo_name': 'ValleyZw/hill', 'size': '3188'}