<jupyter_start><jupyter_text>Data<jupyter_code>bc = datasets.load_breast_cancer()

X_bc = bc.data.astype(np.float32)
y_bc = bc.target.astype(np.float32)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(
    X_bc, y_bc, test_size = .3, random_state=42)

scaler = preprocessing.StandardScaler().fit(X_train)
X_train = scaler.transform(X_train)
X_test = scaler.transform(X_test)
class Data(torch.utils.data.Dataset):
    
    def __init__(self, x, y):
        self.x = torch.tensor(x)
        self.y = torch.tensor(np.expand_dims(y, 1))
        self.len = self.x.shape[0]
    
    def __getitem__(self, index):      
        return self.x[index], self.y[index]
    
    def __len__(self):
        return self.len
train_data = Data(X_train, y_train)
batch_size = len(train_data) // 10
trainloader = torch.utils.data.DataLoader(dataset=train_data, batch_size=batch_size)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class Net(torch.nn.Module):
    
    def __init__(self, n_inputs, n_hidden):
        super(Net, self).__init__()
        self.linear1 = torch.nn.Linear(n_inputs, n_hidden)
        self.linear2 = torch.nn.Linear(n_hidden, 1)
        
    def forward(self, x):
        h    = torch.sigmoid(self.linear1(x))
        yhat = torch.sigmoid(self.linear2(h))
        return yhat
model = Net(X_bc.shape[1], 10)<jupyter_output><empty_output><jupyter_text>Baseline training<jupyter_code>objective = torch.nn.BCELoss()
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
epochs = 100
errors_baseline = []

for epoch in range(epochs):
    total_error = 0
    for x, y in trainloader:
        yhat = model(x)
        loss = objective(yhat, y)
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        total_error += loss.data.item()
    yhat = model(torch.tensor(X_test))
    test_error = objective(yhat, torch.tensor(y_test).view(-1, 1)).data.item()
    errors_baseline.append(test_error)<jupyter_output>/usr/local/lib/python3.6/dist-packages/torch/autograd/__init__.py:132: UserWarning: CUDA initialization: Found no NVIDIA driver on your system. Please check that you have an NVIDIA GPU and installed a driver from http://www.nvidia.com/Download/index.aspx (Triggered internally at  /pytorch/c10/cuda/CUDAFunctions.cpp:100.)
  allow_unreachable=True)  # allow_unreachable flag<jupyter_text>Fast training Train a k-NN classifier<jupyter_code>from sklearn.neighbors import KNeighborsClassifier
knn = KNeighborsClassifier(n_neighbors=10)
knn.fit(X_train, y_train)<jupyter_output><empty_output><jupyter_text>We need two optimizers<jupyter_code>model = Net(X_bc.shape[1], 10)

optimizer_fast = torch.optim.Adam(model.parameters(), lr=0.01)
optimizer_slow = torch.optim.Adam(model.parameters(), lr=0.001)<jupyter_output><empty_output><jupyter_text>First, we fit the network to k-NN with higher LR<jupyter_code>epochs = 10
errors_fast = []

for epoch in range(epochs):
    total_error = 0
    for x, y in trainloader:
        y_hat = model(x)
        # This is how we fit a neural network to k-NN
        #########################################################################################
        y_knn = torch.tensor(knn.predict_proba(x.numpy())[:, 1], dtype=torch.float32).view(-1, 1)
        loss = objective(y_hat, y_knn)
        #########################################################################################
        optimizer_fast.zero_grad()
        loss.backward()
        optimizer_fast.step()
        total_error += loss.item()
    yhat = model(torch.tensor(X_test))
    test_error = objective(yhat, torch.tensor(y_test).view(-1, 1)).data.item()
    errors_fast.append(test_error)<jupyter_output><empty_output><jupyter_text>Then, we fit the network to data with usual LR<jupyter_code>epochs = 90

for epoch in range(epochs):
    total_error = 0
    for x, y in trainloader:
        y_hat = model(x)
        loss = objective(y_hat, y)
        optimizer_slow.zero_grad()
        loss.backward()
        optimizer_slow.step()
        total_error += loss.item()
    yhat = model(torch.tensor(X_test))
    test_error = objective(yhat, torch.tensor(y_test).view(-1, 1)).data.item()
    errors_fast.append(test_error)<jupyter_output><empty_output><jupyter_text>Plot learning curves<jupyter_code>errors = pd.DataFrame({'baseline': errors_baseline, 'fast': errors_fast})
errors.plot()

plt.title('Shallow Network on Breast Cancer Data')
plt.xlabel('epoch')
plt.ylabel('valid loss')

plt.savefig('knn-neural.pdf', format='pdf', dpi=300)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15986913849509268, 'fraction_numerical': 0.01504907306434024, 'mean_word_length': 4.217292377701934, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 32, 'https://': 0, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '337441', 'license': 'MIT', 'n_tokens_mistral': 1669, 'n_tokens_neox': 1604, 'n_words': 284, 'path': 'knn-shallow.ipynb', 'repo_name': 'zh3nis/ngram-nlm', 'size': '4585'}