<jupyter_start><jupyter_text>Clearly, the data is not linearly separable<jupyter_code>import torch
from tqdm import tqdm
CUDA = torch.cuda.is_available()
device = torch.device('cuda' if CUDA else 'cpu')
color_to_numeric = {'r': 0, 'g': 1, 'b': 2}

numerical_labels = [0] * len(data[0]) + [1] * len(data[1]) + [2] * len(data[2])
numerical_labels = np.asarray(numerical_labels)
data = np.concatenate(data, axis = 0)
data.shape
dset = torch.utils.data.TensorDataset(
    torch.tensor(data, dtype=torch.float32),
    torch.tensor(numerical_labels, dtype=torch.long)
)
dloader = torch.utils.data.DataLoader(
    dset, batch_size=16, shuffle=True, num_workers=2
)
net = torch.nn.Sequential(
    torch.nn.Linear(2, 100),
    torch.nn.Linear(100, 2),
    torch.nn.Linear(2, NUM_CLASSES)
)

net = net.to(device)

optimizer = torch.optim.Adam(net.parameters(), lr = 1e-3)

criterion = torch.nn.CrossEntropyLoss()
view_output = [None]

def hook_fn(module, input, output):
    view_output[0] = output.detach().cpu().numpy()

hook = net[-2].register_forward_hook(hook_fn)
# call hook.remove() to remove<jupyter_output><empty_output><jupyter_text>Notice that we havve iintentionally not put any non-linearities in this network.This is to see how such a network performms on non-linear data<jupyter_code>NUM_EPOCHS = 1000
PLOT_AFTER = 10
loop = tqdm(range(NUM_EPOCHS))

for epoch in loop:
    transformed_inputs = np.zeros((0, 2))
    labels = np.zeros((0,), dtype=np.int64)

    for x, y in dloader:
        x = x.to(device)
        y = y.to(device)

        y_pred = net(x)

        optimizer.zero_grad()

        loss = criterion(y_pred, y)
        loss.backward()
        
        # Get activation of the second last layer
        transformed_inputs = np.append(transformed_inputs, view_output[0], axis=0)
        labels = np.append(labels, y.detach().cpu().numpy(), axis=0)

        optimizer.step()

    if epoch % PLOT_AFTER == 0:
        clear_output(wait=True)
        print(f'Epoch: {epoch} \t Loss: {loss.item()}')

        space_transform_fig = plt.figure(figsize=(10, 10))
        colors = [['r', 'g', 'b'][i] for i in labels]
        plt.scatter(transformed_inputs[:, 0], transformed_inputs[:, 1], c=colors)
        plt.show()
net = torch.nn.Sequential(
    torch.nn.Linear(2, 100),
    torch.nn.ReLU(),
    torch.nn.Linear(100, 2),
    torch.nn.Linear(2, NUM_CLASSES)
)

net = net.to(device)

optimizer = torch.optim.Adam(net.parameters(), lr = 1e-3)

criterion = torch.nn.CrossEntropyLoss()
view_output = [None]

def hook_fn(module, input, output):
    view_output[0] = output.detach().cpu().numpy()

hook = net[-2].register_forward_hook(hook_fn)
# call hook.remove() to remove
loop = tqdm(range(NUM_EPOCHS))

for epoch in loop:
    transformed_inputs = np.zeros((0, 2))
    labels = np.zeros((0,), dtype=np.int64)

    for x, y in dloader:
        x = x.to(device)
        y = y.to(device)

        y_pred = net(x)

        optimizer.zero_grad()

        loss = criterion(y_pred, y)
        loss.backward()
        
        # Get activation of the second last layer
        transformed_inputs = np.append(transformed_inputs, view_output[0], axis=0)
        labels = np.append(labels, y.detach().cpu().numpy(), axis=0)

        optimizer.step()

    if epoch % PLOT_AFTER == 0:
        clear_output(wait=True)
        print(f'Epoch: {epoch} \t Loss: {loss.item()}')

        space_transform_fig = plt.figure(figsize=(10, 10))
        colors = [['r', 'g', 'b'][i] for i in labels]
        plt.scatter(transformed_inputs[:, 0], transformed_inputs[:, 1], c=colors)
        plt.show()<jupyter_output>Epoch: 990 	 Loss: 0.19341225922107697
{'fraction_non_alphanumeric': 0.14883081155433286, 'fraction_numerical': 0.026960110041265476, 'mean_word_length': 3.3649459783913565, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '521846', 'license': 'MIT', 'n_tokens_mistral': 1401, 'n_tokens_neox': 1327, 'n_words': 284, 'path': 'notebooks/nb_03_warping_vis.ipynb', 'repo_name': 'facesvision/nyu_deep_learning', 'size': '3635'}