<jupyter_start><jupyter_text>Linear models - scalability In this notebook, we will make a quick note to show the `partial_fit` functionality of some estimator that could be used to train a model.<jupyter_code>import pandas as pd

data = pd.read_csv("../datasets/adult-census-numeric-all.csv")
data.head()
target_name = "class"
X = data.drop(columns=target_name)
y = data[target_name]
from sklearn.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(
    X, y, random_state=0,
)
from sklearn.preprocessing import StandardScaler

scaler = StandardScaler()
batch_size = 100
start = 0
while start < y_train.size:
    stop = start + batch_size
    scaler.partial_fit(X_train[start:stop])
    start = stop
scaler.mean_, scaler.var_
scaler = StandardScaler().fit(X_train)
scaler.mean_, scaler.var_
from sklearn.linear_model import SGDClassifier

model = SGDClassifier(
    loss="hinge", alpha=0.01, max_iter=200
)
import numpy as np

batch_size = 4_000
start = 0
iteration = 1
while start < y_train.size:
    stop = start + batch_size
    X_scaled = scaler.transform(X_train[start:stop])
    if not start:
        params = {"classes": np.unique(y)}
    else:
        params = {}
    model.partial_fit(X_scaled, y_train[start:stop], **params)
    print(
        f"Iteration #{iteration}: Weights:\n"
        f"{model.coef_}"
    )
    iteration += 1
    start = stop<jupyter_output>Iteration #1: Weights:
[[0.09907963 0.26380117 1.52122195 0.35878544 0.01237512]]
Iteration #2: Weights:
[[0.01640582 0.07287119 1.16940378 0.27487187 0.09471367]]
Iteration #3: Weights:
[[0.15245868 0.25970974 1.31388277 0.36729879 0.08375345]]
Iteration #4: Weights:
[[0.15590466 0.2335446  1.22708766 0.3044234  0.21978365]]
Iteration #5: Weights:
[[0.13269746 0.19178599 1.11354771 0.27223249 0.08386686]]
Iteration #6: Weights:
[[0.1194259  0.17126778 1.13674605 0.32037163 0.07275619]]
Iteration #7: Weights:
[[0.06842697 0.08706685 1.18157248 0.40481137 0.06110402]]
Iteration #8: Weights:
[[0.06268194 0.09917399 1.20545059 0.23858605 0.09608195]]
Iteration #9: Weights:
[[0.13817491 0.21181688 1.22910588 0.25993019 0.18156781]]
Iteration #10: Weights:
[[0.13333257 0.18698275 1.23286096 0.16778199 0.12090864]]
{'fraction_non_alphanumeric': 0.12125279642058165, 'fraction_numerical': 0.21297539149888142, 'mean_word_length': 4.466992665036675, 'pattern_counts': {'":': 1, '<': 6, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '56616', 'license': 'CC0-1.0', 'n_tokens_mistral': 1162, 'n_tokens_neox': 881, 'n_words': 203, 'path': '02_linear_models/05_linear_model_scalability.ipynb', 'repo_name': 'jparrax/datascience_middle_course', 'size': '2235'}