<jupyter_start><jupyter_text>Logistic regression from scratch<jupyter_code>import pandas as pd
import numpy as np
def get_accuracy(x: pd.DataFrame, y: pd.Series, y_hat: pd.Series):
    correct = y_hat == y
    acc = np.sum(correct) / len(y)
    cond = y == 1
    y1 = len(y[cond])
    y0 = len(y[~cond])

    print(f'Class 0: tested {y0}, correctly classified {correct[~cond].sum()}')
    print(f'Class 1: tested {y1}, correctly classified {correct[cond].sum()}')
    print(f'Overall: tested {len(y)}, correctly classified {correct.sum()}')
    print(f'Accuracy = {acc:.2f}')
class Classifier:
    def __init__(self, dataset: str = None, mle: bool=True):
        if dataset:
            x_train, y_train = reader(f'datasets/{dataset}-train.txt')
            x_test, y_test = reader(f'datasets/{dataset}-test.txt')
            self.train(x_train, y_train, mle)
            print('Training accuracy')
            print('=' * 10)
            self.accuracy(x_train, y_train)
            print('Test accuracy')
            print('=' * 10)
            self.accuracy(x_test, y_test)
            
    def accuracy(self, x: pd.DataFrame, y: pd.DataFrame) -> None:
        y_hat = self.predict(x)
        get_accuracy(x, y, y_hat)
class LR(Classifier):
    def __init__(self,
                 eta: float = 1e-4,
                 epochs: int = int(1e4),
                 dataset: str = None):
        self.ws = None
        self.eta = eta
        self.epochs = epochs
        
        super().__init__(dataset, mle=True)
        
    @staticmethod
    def _sigmoid(x: np.array):
        return 1 / (1 + np.exp(-x))
        
    @staticmethod
    def _prepend_intercept(x: pd.DataFrame) -> pd.DataFrame:
        return pd.concat([pd.DataFrame([1] * len(x),
                                       columns=['intercept']), x], axis=1)
    
    def train(self, x: pd.DataFrame, y: pd.Series, mle=None) -> None:
        xx = self._prepend_intercept(x)
        self.ws = np.zeros(xx.shape[1])
        for _ in range(self.epochs):
            gs = (y - self._sigmoid(xx @ self.ws)) @ xx
            self.ws += self.eta * gs.values
    
    def predict(self, x: pd.DataFrame) -> pd.Series:
        xx = self._prepend_intercept(x)
        return (xx @ self.ws) > 0.5
    
    def _ll(self, x: pd.DataFrame,
            y: pd.DataFrame, ws: np.array) -> float:
        s = self._sigmoid(x @ ws)
        return y @ np.log(s) + (1 - y) @ np.log(1 - s)
    
    def ll(self, x: pd.DataFrame, y: pd.Series) -> float:
        xx = self._prepend_intercept(x)
        return self._ll(xx, y, self.ws)
    
    def ll_all_zero(self, x: pd.DataFrame, y: pd.Series) -> float:
        xx = self._prepend_intercept(x)
        ws_zero = np.zeros(xx.shape[1])
        return self._ll(xx, y, ws_zero)
x = pd.DataFrame({'x1': [0, 0, 1, 1], 'x2': [0, 1, 0, 1]})
y = pd.Series([0, 0, 1, 1])
x
y
%%time
lr = LR(epochs=10, eta=1e-1)
lr.train(x, y)
lr.accuracy(x, y)
lr.ws<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15382014874915484, 'fraction_numerical': 0.014874915483434753, 'mean_word_length': 2.15795090715048, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '582538', 'license': 'MIT', 'n_tokens_mistral': 1113, 'n_tokens_neox': 1077, 'n_words': 245, 'path': 'machine-learning/logistic-regression-from-scratch.ipynb', 'repo_name': 'ParvathiRajan/learning', 'size': '2958'}