<jupyter_start><jupyter_text>Batch versus online<jupyter_code>%matplotlib inline
import matplotlib.pyplot as plt

plt.style.use('Solarize_Light2')
plt.rc('legend', fontsize=15)    
plt.rc('axes', titlesize=15, labelsize=15)
plt.rc('xtick', labelsize=13)   
plt.rc('ytick', labelsize=13) 
from sklearn import exceptions
import warnings

warnings.filterwarnings(action='ignore', category=exceptions.ConvergenceWarning)
warnings.filterwarnings(action='ignore', category=RuntimeWarning)<jupyter_output><empty_output><jupyter_text>Higgs We'll be using the Higgs dataset available [here](https://archive.ics.uci.edu/ml/datasets/HIGGS). The task is to perform binary classification in order to determine if a signal produces Higgs bosons or not. The dataset contains 11.000.000 rows and has 28 numeric features.<jupyter_code>import pandas as pd

X = pd.read_csv('HIGGS.csv.gz', header=None)
y = X.pop(0)

X = X.to_numpy()
y = y.to_numpy()
from sklearn import linear_model
from sklearn import pipeline
from sklearn import preprocessing

batch = pipeline.make_pipeline(
    preprocessing.StandardScaler(),
    linear_model.LogisticRegression(solver='sag')
)

online = pipeline.make_pipeline(
    preprocessing.StandardScaler(),
    linear_model.SGDClassifier(loss='log', max_iter=1, tol=1e-3)
)<jupyter_output><empty_output><jupyter_text>Split train from test observations.<jupyter_code>X_train, X_test = X[:-500_000], X[-500_000:]
y_train, y_test = y[:-500_000], y[-500_000:]<jupyter_output><empty_output><jupyter_text>Let's record the score of each model for different sample sizes.<jupyter_code>import time
import numpy as np
from sklearn import metrics

batch_times = {}
batch_scores = {}
online_times = {}
online_scores = {}

for train_size in np.logspace(0.5, 1, 100, base=len(X_train), dtype=int)[1:]:
    
    tic = time.time()
    batch.fit(X_train[:train_size], y_train[:train_size])
    batch_times[train_size] = time.time() - tic
    y_pred = batch.predict_proba(X_test)[:, 1]
    batch_scores[train_size] = metrics.roc_auc_score(y_test, y_pred)
    
    tic = time.time()
    online.fit(X_train[:train_size], y_train[:train_size])
    online_times[train_size] = time.time() - tic
    y_pred = online.predict_proba(X_test)[:, 1]
    online_scores[train_size] = metrics.roc_auc_score(y_test, y_pred)<jupyter_output><empty_output><jupyter_text>Plot the test set score for every training sample size.<jupyter_code>fig, ax = plt.subplots(figsize=(14, 8))
ax.plot(batch_scores.keys(), batch_scores.values(), alpha=0.7, marker='o', linestyle='dashed', label='LogisticRegression')
ax.plot(online_scores.keys(), online_scores.values(), alpha=0.7, marker='o', linestyle='dashed', label='SGDClassifier')
ax.set_title('LogisticRegression vs. SGDClassifier')
ax.set_xscale('log')
ax.set_ylim(0, 1)
ax.grid(True)
ax.set_xlabel('Number of train observations')
ax.set_ylabel('Test ROC AUC')
ax.legend();<jupyter_output><empty_output><jupyter_text>Plot the training time for every training sample size.<jupyter_code>fig, ax = plt.subplots(figsize=(14, 8))
ax.plot(batch_times.keys(), batch_times.values(), alpha=0.7, marker='o', linestyle='dashed', label='LogisticRegression')
ax.plot(online_times.keys(), online_times.values(), alpha=0.7, marker='o', linestyle='dashed', label='SGDClassifier')
ax.set_title('LogisticRegression vs. SGDClassifier')
ax.set_xscale('log')
ax.grid(True)
ax.set_xlabel('Number of train observations')
ax.set_ylabel('Number of seconds')
ax.legend();<jupyter_output><empty_output><jupyter_text>Show the score and the training time when the entire training set is used.<jupyter_code>print(f'Batch final ROC AUC is {list(batch_scores.values())[-1]:.5f} in {round(list(batch_times.values())[-1])} seconds')
print(f'Online final ROC AUC is {list(online_scores.values())[-1]:.5f} in {round(list(online_times.values())[-1])} seconds')<jupyter_output>Batch final ROC AUC is 0.68446 in 261 seconds
Online final ROC AUC is 0.68298 in 22 seconds<jupyter_text>Covertype<jupyter_code>from sklearn import datasets

X, y = datasets.fetch_covtype(return_X_y=True)
from sklearn import linear_model
from sklearn import pipeline
from sklearn import preprocessing


batch = pipeline.make_pipeline(
    preprocessing.StandardScaler(),
    linear_model.LogisticRegression(solver='sag', multi_class='auto')
)

online = pipeline.make_pipeline(
    preprocessing.StandardScaler(),
    linear_model.SGDClassifier(loss='log', max_iter=1, tol=1e-3)
)
X_train, X_test = X[:-100_000], X[-100_000:]
y_train, y_test = y[:-100_000], y[-100_000:]
import time
import numpy as np
from sklearn import metrics

batch_times = {}
batch_scores = {}
online_times = {}
online_scores = {}

for train_size in np.logspace(0.5, 1, 100, base=len(X_train), dtype=int)[1:]:
    
    tic = time.time()
    batch.fit(X_train[:train_size], y_train[:train_size])
    batch_times[train_size] = time.time() - tic
    y_pred = batch.predict(X_test)
    batch_scores[train_size] = metrics.accuracy_score(y_test, y_pred)
    
    tic = time.time()
    online.fit(X_train[:train_size], y_train[:train_size])
    online_times[train_size] = time.time() - tic
    y_pred = online.predict(X_test)
    online_scores[train_size] = metrics.accuracy_score(y_test, y_pred)
fig, ax = plt.subplots(figsize=(14, 8))
ax.plot(batch_scores.keys(), batch_scores.values(), alpha=0.7, marker='o', linestyle='dashed', label='LogisticRegression')
ax.plot(online_scores.keys(), online_scores.values(), alpha=0.7, marker='o', linestyle='dashed', label='SGDClassifier')
ax.set_title('LogisticRegression vs. SGDClassifier')
ax.set_xscale('log')
ax.set_ylim(0, 1)
ax.grid(True)
ax.set_xlabel('Number of train observations')
ax.set_ylabel('Test accuracy')
ax.legend();
fig, ax = plt.subplots(figsize=(14, 8))
ax.plot(batch_times.keys(), batch_times.values(), alpha=0.7, marker='o', linestyle='dashed', label='LogisticRegression')
ax.plot(online_times.keys(), online_times.values(), alpha=0.7, marker='o', linestyle='dashed', label='SGDClassifier')
ax.set_title('LogisticRegression vs. SGDClassifier')
ax.set_xscale('log')
ax.grid(True)
ax.set_xlabel('Number of train observations')
ax.set_ylabel('Number of seconds')
ax.legend();<jupyter_output><empty_output><jupyter_text>California housing<jupyter_code>from sklearn import datasets

X, y = datasets.fetch_california_housing(return_X_y=True)
from sklearn import linear_model
from sklearn import pipeline
from sklearn import preprocessing


batch = pipeline.make_pipeline(
    preprocessing.StandardScaler(),
    linear_model.LinearRegression()
)

online = pipeline.make_pipeline(
    preprocessing.StandardScaler(),
    linear_model.SGDRegressor(max_iter=1, tol=1e-3)
)
X_train, X_test = X[:-2_000], X[-2_000:]
y_train, y_test = y[:-2_000], y[-2_000:]
import time
import numpy as np
from sklearn import metrics

batch_times = {}
batch_scores = {}
online_times = {}
online_scores = {}

for train_size in np.logspace(0.5, 1, 100, base=len(X_train), dtype=int)[1:]:
    
    tic = time.time()
    batch.fit(X_train[:train_size], y_train[:train_size])
    batch_times[train_size] = time.time() - tic
    y_pred = batch.predict(X_test)
    batch_scores[train_size] = metrics.mean_absolute_error(y_test, y_pred)
    
    tic = time.time()
    online.fit(X_train[:train_size], y_train[:train_size])
    online_times[train_size] = time.time() - tic
    y_pred = online.predict(X_test)
    online_scores[train_size] = metrics.mean_absolute_error(y_test, y_pred)
fig, ax = plt.subplots(figsize=(14, 8))
ax.plot(batch_scores.keys(), batch_scores.values(), alpha=0.7, marker='o', linestyle='dashed', label='LinearRegression')
ax.plot(online_scores.keys(), online_scores.values(), alpha=0.7, marker='o', linestyle='dashed', label='SGDRegressor')
ax.set_title('LinearRegression vs. SGDRegressor')
ax.set_xscale('log')
ax.set_ylim(0)
ax.grid(True)
ax.set_xlabel('Number of train observations')
ax.set_ylabel('Test MAE')
ax.legend();
fig, ax = plt.subplots(figsize=(14, 8))
ax.plot(batch_times.keys(), batch_times.values(), alpha=0.7, marker='o', linestyle='dashed', label='LinearRegression')
ax.plot(online_times.keys(), online_times.values(), alpha=0.7, marker='o', linestyle='dashed', label='SGDRegressor')
ax.set_title('LinearRegression vs. SGDRegressor')
ax.set_xscale('log')
ax.grid(True)
ax.set_xlabel('Number of train observations')
ax.set_ylabel('Number of seconds')
ax.legend();<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1459253072425725, 'fraction_numerical': 0.022431690729029948, 'mean_word_length': 6.599274705349048, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '423575', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 3142, 'n_tokens_neox': 2960, 'n_words': 572, 'path': 'benchmarks/Batch versus online.ipynb', 'repo_name': 'Leo-VK/creme', 'size': '8381'}