<jupyter_start><jupyter_text>os.environ["TORCH_MODEL_ZOO"] = "./models/pretrained_VGG16/"vgg16 = models.vgg16(pretrained=True)if use_gpu:    vgg16.cuda() .cuda() will move everything to the GPU sid<jupyter_code>vgg16 = models.vgg16()

if use_gpu:
    vgg16.cuda()

vgg16.load_state_dict(torch.load('./models/VGG16_imagenet_130_overfit.pt'))<jupyter_output><empty_output><jupyter_text>G<jupyter_code>image_datasets, dataloaders, dataset_sizes, class_names= load_imagenet(data_dir,
                        32, #batch_size
                        shuffle=True,
                        num_workers=4,
                        pin_memory=False)
vgg16.eval()

with torch.no_grad():
    train_loader, test_loader = generate_dataloader_g(vgg16, 
                                                      dataloaders[TRAIN], 
                                                      dataloaders[TEST],
                                                      [vgg16], ['end'], cpu=True)
                                                      #[vgg16.features[28]], ['conv'], dim=256, axis=1, cpu=True)

    dataset_sizes_g = {TRAIN: len(train_loader), TEST: len(test_loader)}
def vgg_duh(layer_dim):
    dim = layer_dim[1]
    pool = 1
    finalVectorSize = layer_dim[2]//pool * layer_dim[3]//pool * dim
    model = nn.Sequential(
        nn.Conv2d(dim, dim, kernel_size=5, padding=2),
        nn.ReLU(),
        nn.BatchNorm2d(dim),
        
        View([-1]),
        nn.Linear(int(finalVectorSize), 1000),
        nn.ReLU(),
        nn.Linear(1000, 2),
    )
    return model

def denseG():
    model = nn.Sequential(
        nn.ReLU(),
        View([1, -1]),
        nn.AvgPool1d(16, stride=16),
        View([-1]),
        
        nn.Linear(3136, 8),
        nn.RReLU(),
        nn.Dropout(0.5),
        nn.BatchNorm1d(8),
        nn.Linear(8, 8),
        nn.RReLU(),
        nn.BatchNorm1d(8),
        nn.Linear(8, 2)
    )
    return model
def VGG16ResnetBlock(layer_dim):
    dim0 = layer_dim[1]
    dim = 16
    pad = (1, 1)
    pool = 1
    finalVectorSize = dim//pool * layer_dim[2]//pool * layer_dim[3]//pool
    print(finalVectorSize)
    model = nn.Sequential(
        nn.Conv2d(dim0, dim, kernel_size=3, padding = pad),
        nn.ReLU(),
        nn.BatchNorm2d(dim),
        
        nn.Conv2d(dim, dim, kernel_size=3, padding = pad),
        nn.ReLU(),
        nn.BatchNorm2d(dim),
        
        nn.Conv2d(dim, dim, kernel_size=3, padding = pad),
        nn.ReLU(),
        nn.BatchNorm2d(dim),
        
        nn.Conv2d(dim, dim, kernel_size=3, padding = pad),
        nn.ReLU(),
        nn.BatchNorm2d(dim),
        nn.MaxPool3d(pool),

        View([-1]),
        nn.Linear(finalVectorSize, 128),
        nn.ReLU(),
        nn.Linear(128, 128),
        nn.ReLU(),
        nn.Linear(128, 2)
    )
    return model

def VGG16ResnetG(layer_dim):
    input_dim = layer_dim[1]
    dim = 8
    pool = 1
    finalVectorSize = dim * (layer_dim[2]//pool//pool//pool) * (layer_dim[3]//pool//pool//pool)
    print(finalVectorSize)
    model = nn.Sequential(
        nn.Conv2d(input_dim, dim, kernel_size=3, padding = 1),
        nn.RReLU(),
        nn.BatchNorm2d(dim),
        nn.MaxPool2d(pool),
        
        nn.Conv2d(dim, dim, kernel_size=3, padding = 1),
        nn.RReLU(),
        nn.BatchNorm2d(dim),
        nn.MaxPool2d(pool),
        
        nn.Conv2d(dim, dim, kernel_size=3, padding = 1),
        nn.RReLU(),
        nn.BatchNorm2d(dim),
        nn.MaxPool2d(pool),
        
        #nn.AvgPool2d(layer_dim[2], stride=layer_dim[2]),
        View([-1]),
        nn.Linear(finalVectorSize, 8),
        nn.RReLU(),
        nn.Linear(8, 8),
        nn.RReLU(),
        nn.Linear(8, 2)
    )
    return model

def denseG2(vector_size):
    model = nn.Sequential(
        View([-1]),
        nn.Linear(vector_size, 32),
        nn.ReLU(),
        nn.BatchNorm1d(32),
        nn.Dropout(0.5),

        nn.Linear(32, 32),
        nn.ReLU(),
        nn.BatchNorm1d(32),
        nn.Dropout(0.5),
        
        nn.Linear(32, 16),
        nn.ReLU(),
        nn.BatchNorm1d(16),
        nn.Dropout(0.5),
        
        nn.Linear(16, 2),
        nn.Softmax(1)
    )
    return model
layer_dim
debug_layers_dims(denseG2(1000), layer_dim)
model = denseG2(1000)
sum(x.numel() for x in model.parameters())
train_dataloader = DataLoader(train_loader, shuffle=True, pin_memory=False, batch_size=64)
test_dataloader = DataLoader(test_loader, shuffle=True, pin_memory=False, batch_size=64)
layer_dim = next(iter(train_dataloader))[0].shape
print("sample batch dimension :", layer_dim)
model = denseG2(1000)

if use_gpu:
    model.cuda() #.cuda() will move everything to the GPU side

resume_training = False

if resume_training:
    print("Loading model..")
    model.load_state_dict(torch.load('./models/G_vgg_duh.pt'))
    print("Loaded!")

criterion = nn.CrossEntropyLoss()

#optimizer_ft = optim.RMSprop(model.parameters(), lr=1e-3, momentum=0.2)
optimizer_ft = optim.SGD(model.parameters(), lr=5e-2)
#optimizer_ft = optim.Adamax(model.parameters(), lr=1e-4)
exp_lr_scheduler = None#lr_scheduler.StepLR(optimizer_ft, step_size=7, gamma=0.1)

with open('G_training_output.txt', 'w') as f:
    model, hist = train_model(model, {TRAIN: train_dataloader, TEST: test_dataloader}, \
                  criterion, optimizer_ft, exp_lr_scheduler, dataset_sizes_g, num_epochs=120, log_file=f)
    
torch.save(model.state_dict(), './models/G_vgg_duh.pt')
hist.plot("G Learning curves", 1)
plt.savefig("G_training_curves", dpi=300, bbox_inches='tight')
model.eval()
diagnostic_plots(model, train_dataloader, test_dataloader, bw=0.3, savefig=True)<jupyter_output>false negative percentage : 41.92
false positive percentage : 37.984<jupyter_text>For report:training on layer 7 with VGG16ResnetG    train acc : 0.9719    val acc : 0.5181    false negative percentage : 44.192    false positive percentage : 52.192    ROC score train : 0.9983631908301265    ROC score test : 0.5299173376training on layer 19 with VGG16ResnetG    train acc : 0.9017    val acc : 0.5336    false negative percentage : 49.6    false positive percentage : 47.968    ROC score train : 0.9698352704310425    ROC score test : 0.5193706496training on layer 28 with VGG16ResnetG    train acc : 0.9216    val acc : 0.5197    false negative percentage : 54.528    false positive percentage : 41.536    ROC score train : 0.9840197735459162    ROC score test : 0.5232690176training on layer 28 with dense_duh()    train acc : 0.7344    val acc : 0.5382    false negative percentage : 54.496    false positive percentage : 37.856    ROC score train : 0.9072115413949005    ROC score test : 0.549835776training on layer final with denseG2    train acc : 0.6591    val acc : 0.6005    false negative percentage : 41.92    false positive percentage : 37.984    ROC score train : 0.7576299968223704    ROC score test : 0.6423002624<jupyter_code>#Free memory without restarting kernel
del train_loader, test_loader, dataloaders 
del train_dataloader, test_dataloader
del model, vgg16
del criterion, optimizer_ft, exp_lr_scheduler

gc.collect()
torch.cuda.empty_cache()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12484254723582926, 'fraction_numerical': 0.06731980405878236, 'mean_word_length': 2.515002459419577, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '516317', 'license': 'MIT', 'n_tokens_mistral': 2810, 'n_tokens_neox': 2520, 'n_words': 491, 'path': 'VGG_imagenet_cleaned.ipynb', 'repo_name': 'palexlee/DeepWololo', 'size': '7145'}