<jupyter_start><jupyter_text>Baseline<jupyter_code>%reload_ext autoreload
%autoreload 2
%matplotlib inline
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

from sklearn import metrics
from sklearn import model_selection

import albumentations as A

import torch
import torch.nn as nn
import torch.nn.functional as F
import torchcontrib
DATA_PATH = "../input/"
PIXEL_COLS = [str(i) for i in range(784)]
DEVICE = "cuda"
EPOCHS = 200<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>df = pd.read_csv(DATA_PATH + "train.csv")
df.head()
digits = df.digit.values
letters = df.letter.values
images = df[PIXEL_COLS].values
MEAN = np.mean(images / 255.0)
STD = np.std(images / 255.0)
MEAN, STD
idx = np.random.randint(len(images))
image = images[idx].reshape((28, 28))
plt.imshow(image)
plt.title(f"{digits[idx]} {letters[idx]}")
plt.axis("off");
plt.figure(figsize=(15, 4.5))
for i in range(30):
    plt.subplot(3, 10, i+1)
    plt.imshow(images[i].reshape(28, 28))
    plt.title(f"{digits[i]} {letters[i]}")
    plt.axis("off")
plt.show()<jupyter_output><empty_output><jupyter_text>Dataset<jupyter_code>class TrainDataset(torch.utils.data.Dataset):
    def __init__(self, df):
        self.images = df[PIXEL_COLS].values
        self.images = self.images
        self.targets = df.digit.values
        self.augs = A.Compose([
            A.Normalize(mean=MEAN, std=STD, max_pixel_value=255.0, always_apply=True),
        ])
#         self.augs = A.Compose(
#             [
#                 A.ShiftScaleRotate(shift_limit=0.0625, scale_limit=0.1, rotate_limit=10, p=0.9),
#                 A.OneOf(
#                     [
#                         A.ElasticTransform(alpha=120, sigma=120 * 0.05, alpha_affine=120 * 0.03),
#                         A.GridDistortion(),
#                         A.OpticalDistortion(distort_limit=2, shift_limit=0.5),
#                     ],
#                     p = 0.5
#                 ),
#                 A.CoarseDropout(max_holes=4, max_height=4, max_width=4, p=0.25),
#                 A.Normalize(MEAN, STD, max_pixel_value=255.0, always_apply=True),
#             ]
#         )
        
    def __len__(self):
        return len(self.images)
    
    def __getitem__(self, item):
        image = self.images[item]
        image = image.astype(np.uint8)
        image = image.reshape(28, 28, 1)
        image = self.augs(image=image)["image"]
        image = torch.tensor(image, dtype=torch.float)
        image = image.permute(2, 0, 1)
        
        target = self.targets[item]
        target = torch.tensor(target, dtype=torch.long)
        
        return {"image": image, "target": target}
df_train, df_valid = model_selection.train_test_split(df, test_size=0.1, stratify=df.digit)
train_dataset = TrainDataset(df_train)
valid_dataset = TrainDataset(df_valid)
train_loader = torch.utils.data.DataLoader(train_dataset, batch_size=64, shuffle=True)
valid_loader = torch.utils.data.DataLoader(valid_dataset, batch_size=128)<jupyter_output><empty_output><jupyter_text>Model<jupyter_code>class Model(nn.Module):
    def __init__(self):
        super(Model, self).__init__()
        self.conv1 = nn.Conv2d(1, 32, kernel_size=3)
        self.bn1 = nn.BatchNorm2d(32)
        self.conv2 = nn.Conv2d(32, 32, kernel_size=5)
        self.bn2 = nn.BatchNorm2d(32)
        self.conv3 = nn.Conv2d(32, 32, kernel_size=5, stride=2, padding=13)
        self.bn3 = nn.BatchNorm2d(32)
        
        self.conv2_1 = nn.Conv2d(32, 64, kernel_size=3)
        self.bn2_1 = nn.BatchNorm2d(64)
        self.conv2_2 = nn.Conv2d(64, 64, 3)
        self.bn2_2 = nn.BatchNorm2d(64)
        self.conv2_3 = nn.Conv2d(64, 64, 5, stride=2, padding=11)
        self.bn2_3 = nn.BatchNorm2d(64)
        
        self.conv3_1 = nn.Conv2d(64, 128, 4)
        self.bn3_1 = nn.BatchNorm2d(128)
        
        self.out = nn.Linear(28800, 10)
         
    def forward(self, x):
        x = self.conv1(x)
        x = F.relu(x)
        x = self.bn1(x)
        
        x = self.conv2(x)
        x = F.relu(x)
        x = self.bn2(x)
        
        x = self.conv3(x)
        x = F.relu(x)
        x = self.bn3(x)
        
        x = F.dropout(x, p=0.4)
        
        x = self.conv2_1(x)
        x = F.relu(x)
        x = self.bn2_1(x)
        
        x = self.conv2_2(x)
        x = F.relu(x)
        x = self.bn2_2(x)
        
        x = self.conv2_3(x)
        x = F.relu(x)
        x = self.bn2_3(x)
        
        x = F.dropout(x, p=0.4)
        
        x = self.conv3_1(x)
        x = F.relu(x)
        x = self.bn3_1(x)
        
        x = x.view(-1, 28800)
        x = F.dropout(x, p=0.4)
        x = self.out(x)
        
#         x = F.softmax(x)
        
        return x<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>def train(data_loader, model, optimizer, device, scheduler=None):
    model.train()
    for data in data_loader:
        inputs = data["image"]
        targets = data["target"]
        inputs = inputs.to(device)
        targets = targets.to(device)
        
        optimizer.zero_grad()
        outputs = model(inputs)
        loss = nn.CrossEntropyLoss()(outputs, targets)
        loss.backward()
        
        # Gradient clipping
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        
        optimizer.step()
        
        if scheduler is not None:
            scheduler.step()

        
def evaluate(data_loader, model, device):
    model.eval()
    final_outputs = []
    final_targets = []
    with torch.no_grad():
        for data in data_loader:
            inputs = data["image"]
            inputs = inputs.to(device)
            outputs = model(inputs)
            outputs = outputs.detach().cpu().numpy().tolist()
            
            targets = data["target"]
            targets = targets.numpy().tolist()
            
            final_outputs.extend(outputs)
            final_targets.extend(targets)
            
    return final_outputs, final_targets
class EarlyStopping:
    def __init__(self, patience=10, mode="max", epsilon=0.0001):
        self.patience = patience
        self.mode = mode
        self.epsilon = epsilon

        # Scores
        self.best_score = None
        self.counter = 0
        self.early_stop = False
        if self.mode == "max":
            self.val_score = -np.inf
        else:
            self.val_score = np.inf

    def __call__(self, epoch_score, model, model_path):
        if self.mode == "max":
            score = epoch_score
        else:
            score = -1.0 * epoch_score

        if self.best_score is None:
            self.best_score = score
            self.save_checkpoint(epoch_score, model, model_path)
        elif score < self.best_score + self.epsilon:
            self.counter += 1
            print(f"EarlyStopping counter: {self.counter} out of {self.patience}")
            if self.counter >= self.patience:
                self.early_stop = True
        else:
            self.best_score = score
            self.counter = 0
            self.save_checkpoint(epoch_score, model, model_path)

    def save_checkpoint(self, epoch_score, model, model_path):
        if epoch_score not in (np.inf, -np.inf, np.nan, -np.nan):
            print(
                f"Validation score improved ({self.val_score} -> {epoch_score}). Saving model..."
            )
            torch.save(model.state_dict(), model_path)
        self.val_score = epoch_score
device = torch.device(DEVICE)
model_path = "./tmp.pt"
model = Model()
model.to(device)

optimizer = torch.optim.Adam(model.parameters(), lr=1e-3)
optimizer = torchcontrib.optim.SWA(optimizer, swa_start=10, swa_freq=5, swa_lr=1e-3)
scheduler = torch.optim.lr_scheduler.ReduceLROnPlateau(optimizer, mode='max', verbose=True, patience=10, factor=0.5)
es = EarlyStopping(patience=20, mode="max")

for epoch in range(EPOCHS):
    train(train_loader, model, optimizer, device)
    predictions, targets = evaluate(valid_loader, model, device)
    
    predictions = np.array(predictions)
    predictions = np.argmax(predictions, axis=1)
    accuracy = metrics.accuracy_score(targets, predictions)
    
    if epoch % 10 == 0:
        print(f"Epoch: {epoch}, Accuracy={accuracy}")
    
    scheduler.step(accuracy)
    
    es(accuracy, model, model_path)
    if es.early_stop:
        print(f"Early stopping. Best score={es.best_score}. Loading weights...")
        model.load_state_dict(torch.load(model_path))
        break

optimizer.swap_swa_sgd()<jupyter_output>Epoch: 0, Accuracy=0.16585365853658537
Validation score improved (-inf -> 0.16585365853658537). Saving model...
Validation score improved (0.16585365853658537 -> 0.21951219512195122). Saving model...
Validation score improved (0.21951219512195122 -> 0.22439024390243903). Saving model...
Validation score improved (0.22439024390243903 -> 0.2926829268292683). Saving model...
Validation score improved (0.2926829268292683 -> 0.3121951219512195). Saving model...
EarlyStopping counter: 1 out of 20
Validation score improved (0.3121951219512195 -> 0.375609756097561). Saving model...
Validation score improved (0.375609756097561 -> 0.44878048780487806). Saving model...
Validation score improved (0.44878048780487806 -> 0.45365853658536587). Saving model...
EarlyStopping counter: 1 out of 20
Epoch: 10, Accuracy=0.3853658536585366
EarlyStopping counter: 2 out of 20
EarlyStopping counter: 3 out of 20
Validation score improved (0.45365853658536587 -> 0.47317073170731705). Saving model...
Validation sc[...]<jupyter_text>Inference<jupyter_code>df_test = pd.read_csv(DATA_PATH + "test.csv")
df_test.head()
class EMNISTTestDataset(torch.utils.data.Dataset):
    def __init__(self, df):
        self.images = df[PIXEL_COLS].values
        self.augs = A.Compose([
            A.Normalize(mean=MEAN, std=STD, max_pixel_value=255.0, always_apply=True),
        ])
    
    def __len__(self):
        return len(self.images)
    
    def __getitem__(self, item):
        image = self.images[item]
        image = image.astype(np.uint8)
        image = image.reshape(28, 28, 1)
        image = self.augs(image=image)["image"]
        image = torch.tensor(image, dtype=torch.float)
        image = image.permute(2, 0, 1)
        
        return {"image": image}
def infer(data_loader, model, device):
    model.eval()
    final_predictions = []
    for data in data_loader:
        images = data["image"]
        images = images.to(device)
        predictions = model(images)
        predictions = predictions.detach().cpu().numpy().tolist()
        final_predictions.extend(predictions)
        
    return final_predictions
test_dataset = EMNISTTestDataset(df_test)
test_loader = torch.utils.data.DataLoader(test_dataset, batch_size=128)
predictions = infer(test_loader, model, device)
predictions = np.array(predictions)

digits = np.argmax(predictions, axis=1)

probs = np.exp(predictions)
probs = probs / np.sum(probs, axis=1)[:, None]
probs = np.max(probs, axis=1)
subm = pd.DataFrame({"id": df_test.id, "digit": digits, "prob": probs})
subm.head()
df_pl = df_test[subm.prob > 0.995]
df_pl = df_pl.merge(subm, how="left", on="id")
print(df_pl.shape)
df_pl.head()
df_pl.to_csv("../input/pl.csv", index=False)
train_pl = pd.concat([df, df_pl.drop("prob", axis=1)])
train_pl.shape
train_pl.to_csv("../input/train_pl.csv", index=False)
submission = pd.DataFrame({"id": df_test.id, "digit": predictions})
submission.to_csv("../output/baseline_swa.csv", index=False)
submission.head()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12881090940675757, 'fraction_numerical': 0.05472075045600625, 'mean_word_length': 2.5416794832359275, 'pattern_counts': {'":': 10, '<': 25, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '621223', 'license': 'MIT', 'n_tokens_mistral': 4357, 'n_tokens_neox': 3990, 'n_words': 753, 'path': 'notebooks/01-baseline.ipynb', 'repo_name': 'giwankim/emnist', 'size': '11513'}