<jupyter_start><jupyter_text>0. Parameters<jupyter_code>device = "cuda"
fine_tune = False
batch_size = 128
load_model_dir = "vrd/model2.pth"
save_dir = "vrd/model3.pth"
is_train = True
lr = 0.001<jupyter_output><empty_output><jupyter_text>1.Prepare<jupyter_code>df_train_vrd_is = pd.read_hdf('vrd/temp/df_train_vrd_is.h5', 'df_train_vrd_is')
df_valid_vrd_is = pd.read_hdf('vrd/temp/df_valid_vrd_is.h5', 'df_valid_vrd_is')
df_test_is = pd.read_hdf('vrd/temp/df_test_is.h5', 'df_test_is') 
# to remove as non existing
df_train_vrd_is = df_train_vrd_is[df_train_vrd_is["ImageID"]!="634483a6a8a74df4"]
list(df_train_vrd_is["LabelName2"].unique())
train_dfs = []
for cls in df_train_vrd_is["LabelName2"].unique():
    print(len(df_train_vrd_is[df_train_vrd_is["LabelName2"]==cls]))
    df_temp = df_train_vrd_is[df_train_vrd_is["LabelName2"]==cls]
    df_temp = df_temp.reset_index(drop=True)
    train_dfs.append(df_temp)
dict_clsid_to_idx = {
    '/m/083vt': 0,
    '/m/02gy9n': 1,
    '/m/05z87': 2,
    '/m/04lbp': 3,
    '/m/0dnr7': 4
}

dict_idx_to_clsid = {v:k for k,v in dict_clsid_to_idx.items()}
num_classes = len(dict_clsid_to_idx)<jupyter_output><empty_output><jupyter_text>2. Dataloader<jupyter_code>from torch.utils.data import Dataset
from torch.utils.data import DataLoader
from PIL import Image, ImageFile

import numpy as np
import json
from pathlib import Path
import torch
import pickle

ImageFile.LOAD_TRUNCATED_IMAGES = True

class DS(Dataset):
    def __init__(self, images_dir, df, dict_clsid_to_idx, transform):
        self.images_dir = Path(images_dir)
        self.transform = transform
        self.df = df
        self.dict_clsid_to_idx = dict_clsid_to_idx
        
        if isinstance(self.df, list):
            self.num_items = 25000
        else:
            self.num_items = len(self.df)
            
    def image_path(self, image_id):
        image_name = image_id+'.jpg'
        path = self.images_dir/image_name
        return path
    
    def load_image(self, img_path):
        image = Image.open(img_path).convert('RGB')
        return image
        
    def __len__(self):
        return self.num_items
    
    def crop_to_object(self, img, coordinates):
        return img.crop(coordinates)

    def __getitem__(self, idx):
        if isinstance(self.df, list):
            draw = np.random.choice(range(0,len(self.df)))
            idx = np.random.choice(range(0,len(self.df[draw])))
            row = self.df[draw].iloc[idx]
        else:
            row = self.df.iloc[idx]
        img_id = row["ImageID"]
        target_label = row["LabelName2"]
        coordinates = row[["XMin1","YMin1","XMax1","YMax1"]].values.astype(float)
        img_path = self.image_path(img_id)
        img = self.load_image(img_path)
        width, height = img.size
        coordinates[0] = width * coordinates[0]
        coordinates[1] = height * coordinates[1]
        coordinates[2] = width * coordinates[2]
        coordinates[3] = height * coordinates[3]
        
        cropped_img = self.crop_to_object(img, coordinates)

        cropped_img = self.transform(cropped_img)
        if target_label!="":
            target_label_id = self.dict_clsid_to_idx[target_label]
        else:
            target_label_id = -1
        return idx, img_id, cropped_img, target_label_id
    
    def collate_fn(self, samples):
        idxs, img_ids, imgs, tgts = zip(*samples)
        imgs = torch.stack(imgs, 0)
        return idxs, img_ids, imgs, torch.tensor(tgts)
    
    def init_workers_fn(self, worker_id):
        new_seed = int.from_bytes(os.urandom(4), byteorder='little')
        np.random.seed(new_seed)
    
def get_dl(images_dir, df, dict_clsid_to_idx, transform_fn, batch_size):
    dataset = DS(
        images_dir, 
        df, 
        dict_clsid_to_idx,
        transform_fn
    )
    dataloader = DataLoader(dataset, 
        batch_size= batch_size,
        shuffle= True, 
        num_workers= 4, 
        pin_memory= True, 
        collate_fn= dataset.collate_fn,
        worker_init_fn=dataset.init_workers_fn,
        drop_last = True
    )
    return dataloader

def get_test_dl(images_dir, df, dict_clsid_to_idx, transform_fn, batch_size):
    dataset = DS(
        images_dir, 
        df, 
        dict_clsid_to_idx,
        transform_fn
    )
    dataloader = DataLoader(dataset, 
        batch_size= batch_size,
        shuffle= False, 
        num_workers= 8, 
        pin_memory= True, 
        collate_fn= dataset.collate_fn,
        drop_last = False
    )
    return dataloader
data_transforms = {
    'train': transforms.Compose([
        transforms.Resize((128,128)),
        transforms.RandomHorizontalFlip(),
        transforms.RandomVerticalFlip(),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'val': transforms.Compose([
        transforms.Resize((128,128)),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
    'test': transforms.Compose([
        transforms.Resize((128,128)),
        transforms.ToTensor(),
        transforms.Normalize([0.485, 0.456, 0.406], [0.229, 0.224, 0.225])
    ]),
}
train_dl = get_dl("dataset/train", df_train_vrd_is, dict_clsid_to_idx, data_transforms['train'], batch_size)
val_dl = get_dl("dataset/validation", df_valid_vrd_is, dict_clsid_to_idx, data_transforms['val'], batch_size)
dataloaders = {
    "train": train_dl,
    "val" : val_dl
}
def train_model(model, criterion, optimizer, scheduler, num_epochs=25):
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_acc = 0.0

    for epoch in range(num_epochs):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0
            running_items = 0
            # Iterate over data.
            pbar = tqdm(dataloaders[phase])
            try:
                
                for _, _, inputs, labels in pbar:
                    try:
                        inputs = inputs.to(device)
                        labels = labels.to(device)

                        # zero the parameter gradients
                        optimizer.zero_grad()

                        # forward
                        # track history if only in train
                        with torch.set_grad_enabled(phase == 'train'):
                            outputs = model(inputs)
                            _, preds = torch.max(outputs, 1)
                            loss = criterion(outputs, labels)
                            pbar.set_description(str(round(loss.item(),5)))
                            # backward + optimize only if in training phase
                            if phase == 'train':
                                loss.backward()
                                optimizer.step()
                                scheduler.step()

                        # statistics
                        running_loss += loss.item() * inputs.size(0)
                        running_items += inputs.size(0)
                        running_corrects += torch.sum(preds == labels.data)
                    except Exception as b:
                        print("Skipping as error with batch", b)
            except Exception as e:
                print("Stopping due to no existing file", e)
                    
            epoch_loss = running_loss / running_items
            epoch_acc = running_corrects.double() / running_items

            print('{} Loss: {:.4f} Acc: {:.4f}'.format(phase, epoch_loss, epoch_acc))

            # deep copy the model
            if phase == 'val' and epoch_acc > best_acc:
                best_acc = epoch_acc
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val Acc: {:4f}'.format(best_acc))

    # load best model weights
    model.load_state_dict(best_model_wts)
    return model

def set_parameter_requires_grad(model):
    for name, param in model.named_parameters():
        if (name.split('.')[0]) not in ["fc"]:
            param.requires_grad = False
            
class Head(nn.Module):
    def __init__(self, num_ftrs, n_classes):
        super(Head, self).__init__()
        self.l1 = nn.Linear(num_ftrs, 256)
        self.dropout = nn.Dropout(0.2)
        self.l2 = nn.Linear(256,n_classes) 
        self.relu = nn.LeakyReLU()
        
    def forward(self, x):
        x = self.dropout(self.relu(self.l1(x)))
        x = self.l2(x)
        return x
model_ft = models.resnet34(pretrained=True)
num_ftrs = model_ft.fc.in_features
head = Head(num_ftrs, num_classes)
model_ft.fc = head # nn.Linear(num_ftrs, num_classes)
if fine_tune:
    set_parameter_requires_grad(model_ft)
criterion = nn.CrossEntropyLoss()
optimizer_ft = optim.Adam(model_ft.parameters(), lr=lr)
exp_lr_scheduler = lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)
if load_model_dir !=None:
    checkpoint = torch.load(load_model_dir)
    model_ft.load_state_dict(checkpoint['model'])
#     optimizer_ft.load_state_dict(checkpoint['optimizer'])
    exp_lr_scheduler.load_state_dict(checkpoint['scheduler'])
model_ft = model_ft.to(device)
if is_train:
    model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                           num_epochs=5)
torch.save({
    "model": model_ft.state_dict(),
    "optimizer": optimizer_ft.state_dict(),
    "scheduler": exp_lr_scheduler.state_dict(),
}, save_dir)<jupyter_output><empty_output><jupyter_text>3. Evaluate<jupyter_code>test_dl = get_test_dl("dataset/test/test", df_test_is, dict_clsid_to_idx, data_transforms['test'], batch_size)
model_ft.eval()
with torch.no_grad():
    for idxs, image_ids, inputs, labels in tqdm(test_dl):
        inputs = inputs.to(device)
        labels = labels.to(device)
        
        outputs = model_ft(inputs)
        _, preds = torch.max(outputs, 1)
        for i, idx in enumerate(idxs):
            df_test_is.at[idx,'LabelName2'] = dict_idx_to_clsid[preds[i].item()]
model_ft.train()
df_test_is.to_csv("submission_is.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11433447098976109, 'fraction_numerical': 0.019908987485779295, 'mean_word_length': 2.3908711025393763, 'pattern_counts': {'":': 5, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '454892', 'license': 'Apache-2.0', 'n_tokens_mistral': 3697, 'n_tokens_neox': 3516, 'n_words': 642, 'path': '03_VRD_Competition_Is.ipynb', 'repo_name': 'ryanwongsa/open-images-2019-challenge', 'size': '10548'}