<jupyter_start><jupyter_text><jupyter_code>!pip install adabelief-pytorch
!pip install timm
!pip install gpustat
import torchvision
from torchvision import transforms, datasets

import torch.utils.data.dataloader
import matplotlib.pyplot as plt


import torch.optim as optim
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import Dataset,ConcatDataset
import adabelief_pytorch
import numpy  as np

from sklearn.metrics import roc_auc_score, auc

from timm.data import IMAGENET_DEFAULT_MEAN, IMAGENET_DEFAULT_STD, IMAGENET_INCEPTION_MEAN, IMAGENET_INCEPTION_STD
from timm.models.efficientnet_blocks import round_channels, resolve_bn_args, resolve_act_layer, BN_EPS_TF_DEFAULT
from timm.models.efficientnet_builder import EfficientNetBuilder, decode_arch_def, efficientnet_init_weights
from timm.models.features import FeatureInfo, FeatureHooks
from timm.models.helpers import build_model_with_cfg, default_cfg_for_features
from timm.models.layers import create_conv2d, create_classifier, DropPath, to_2tuple, trunc_normal_
from timm.models.registry import register_model
import timm
import timm.models.efficientnet
device = 'cuda' if torch.cuda.is_available() else 'cpu'

torch.manual_seed(456)
if device =='cuda':
    torch.cuda.manual_seed_all(456)
class PatchEmbed(nn.Module):
    """ Image to Patch Embedding
    """
    def __init__(self, img_size=224, patch_size=16, in_chans=3, embed_dim=768):
        super().__init__()
        img_size = to_2tuple(img_size)
        patch_size = to_2tuple(patch_size)
        num_patches = (img_size[1] // patch_size[1]) * (img_size[0] // patch_size[0])
        self.img_size = img_size
        self.patch_size = patch_size
        self.num_patches = num_patches

        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=patch_size, stride=patch_size)

    def forward(self, x):
        B, C, H, W = x.shape  
        # FIXME look at relaxing size constraints
        assert H == self.img_size[0] and W == self.img_size[1], \
            f"Input image size ({H}*{W}) doesn't match model ({self.img_size[0]}*{self.img_size[1]})."
        x = self.proj(x).flatten(2).transpose(1, 2)
        x=x.reshape(len(x),self.num_patches*3,self.patch_size[0],self.patch_size[1])
        #print(x.shape)
        return x

patch1=PatchEmbed(1500,250,3,250*250*3).to(device)
sum(p.numel() for p in patch1.parameters())
class PatchEmbedV2(nn.Module):
    """ Image to Patch Embedding
    """
    def __init__(self, img_size=1500, patch_size=250, in_chans=3, embed_dim=36*3):
        super().__init__()
        
        self.proj = nn.Conv2d(in_chans, embed_dim, kernel_size=img_size//patch_size, stride=img_size//patch_size)

    def forward(self, x):
        
        x = self.proj(x)
        
        #print(x.shape)
        return x

patch2=PatchEmbedV2().to(device)
sum(p.numel() for p in patch2.parameters())
x=torch.rand((1,3,1500,1500)).to(device)
patch2(x).shape
class skipblock(nn.Module):
  def __init__(self, in_channel,out_channel,outsize):
    super(skipblock, self).__init__()
    
    
    self.conv0=nn.Conv2d(in_channel,out_channel,1)
    self.silu=nn.SiLU()
    
    self.block=nn.Sequential(
                    
            nn.Conv2d(out_channel, 64*out_channel, 1),
            nn.BatchNorm2d(out_channel*64),         
            nn.SiLU(),
            nn.Dropout2d(),
            

            nn.Conv2d(out_channel*64, out_channel*64, 3, 1, 1),
            nn.BatchNorm2d(out_channel*64),          
            nn.SiLU(),
            nn.Dropout2d(),

            nn.Conv2d(out_channel*64,out_channel,1),
            nn.BatchNorm2d(out_channel),
            nn.SiLU(),
            nn.Dropout2d(),
            
        )
    
    self.bn=nn.BatchNorm2d(out_channel)
    self.pool=nn.AdaptiveAvgPool2d(outsize)

  def forward(self, x):

    out=self.conv0(x)
    out=self.silu(out)
    nn.Dropout2d()
    skip=out
    #print(out.shape)
    out=self.block(out)
    #print(out.shape)
    out=out+skip
    #print(out.shape)
    out=self.bn(out)
    out=self.pool(out)

    return out
#gelu=nn.GELU()
class testblock2(nn.Module):
    def __init__(self,ni,num_class):
        super(testblock2, self).__init__()

        self.embeding=PatchEmbedV2(img_size=1000, patch_size=200, in_chans=3, embed_dim=25*3)   # Filter size 5*5

        
        self.downsample=skipblock(25*3,64,100)

  
        self.block0=skipblock(64,32,80)
        
        self.block1=skipblock(32,3,50)
        
        self.block2=skipblock(3,1,10)
        
     
        self.classifier = nn.Sequential(
            nn.Linear(1*10*10,num_class),
            
        )

    def forward(self,x):
        out=self.embeding(x)
        out=self.downsample(out)
        out=self.block0(out)
        out=self.block1(out)
        out=self.block2(out)


        out = out.view(out.size(0),-1)
        out=self.classifier(out)
        
        return out

model=testblock2(3,39)
#model=torch.nn.DataParallel(model)
model=model.to(device)
x=torch.rand((1,3,1000,1000)).to(device)
model(x)

sum(p.numel() for p in model.parameters())
from google.colab import drive
drive.mount("/content/gdrive")
PATH='/content/gdrive/MyDrive/Colab Notebooks/fundusdataset/1000images/'
transform = transforms.Compose(
    [
     #transforms.RandomRotation(10),
     transforms.RandomHorizontalFlip(),
     transforms.RandomCrop((1000,1000)),
     transforms.ToTensor(),
     
     
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))])

fundus_data = torchvision.datasets.ImageFolder(root=PATH,transform=transform,)


transform = torch.nn.Sequential(
    
     #transforms.RandomRotation(10),
     transforms.RandomHorizontalFlip(),
     transforms.RandomCrop((1000,1000)),
     #transforms.ToTensor(),
     
     
     transforms.Normalize((0.5, 0.5, 0.5), (0.5, 0.5, 0.5))
     
     )

transform1 = transforms.Compose(
    [
     transforms.ToTensor()
     ]
     )

fundus_data = torchvision.datasets.ImageFolder(root=PATH,transform=transform1)



class CustomDataset(Dataset): 
  def __init__(self,fundus_data):
     #self.transform=transform
    self.data= fundus_data

    

 
  def __len__(self): 
    return len(self.data)

  
  def __getitem__(self, idx): 
    img, label = self.data[idx]
    
    if img.shape[1] < 1000:
      temp=torch.rand(3,1000,1000)
      temp[:,:img.shape[1],:img.shape[2]]=temp[:,:img.shape[1],:img.shape[2]]+img
      img=temp


    img=transform(img)
    x_0 = torch.FloatTensor(img)
    
    
    y = label
    return x_0, y

fundus_data=CustomDataset(fundus_data=fundus_data)
train_size=int(0.8*len(fundus_data))
test_size=len(fundus_data)-train_size
train_set, test_set=torch.utils.data.random_split(fundus_data,[train_size,test_size])
torch.save(train_set,PATH+'trainset.csv')
torch.save(test_set,PATH+'testset.csv')
train_set=torch.load(PATH+'trainset.csv')
test_set=torch.load(PATH+'testset.csv')
train_loader = torch.utils.data.DataLoader(train_set,drop_last=True,
                                          batch_size=1,
                                          shuffle=True,
                                          num_workers=2,
                                          )
test_loader = torch.utils.data.DataLoader(test_set,
                                          batch_size=1,
                                          shuffle=False,
                                          num_workers=4,
                                          )
criterion = nn.CrossEntropyLoss().to(device)
#optimizer = torch.optim.Adam(model.parameters(), lr = 0.001)
optimizer=adabelief_pytorch.AdaBelief(model.parameters(),lr=0.001)
#lr_sche = optim.lr_scheduler.StepLR(optimizer, step_size=10, gamma=0.5)

print(len(train_loader))
trainedepochs=0
print(len(train_loader))

epochs = 500


for epoch in range(epochs):
  model.train()
  
  running_loss = 0.0

  for i, data in enumerate(train_loader, 0):
    # get the inputs
    inputs, labels = data
    inputs=inputs.to(device)
    labels = labels.to(device)

    # zero the parameter gradients
    optimizer.zero_grad()

    # forward + backward + optimize
    outputs = model(inputs)

    
    
       # (numarray.max()/numarray[labels[0][0].to(torch.int).tolist()-1])
    #loss = (1/numnum[labels[0][0].to(torch.int).tolist()-1])*criterion(outputs, labels.to(torch.float)/10)
    loss = criterion(outputs, labels)
    
    #print( (numnum.max()/numnum[labels[0][0].to(torch.int).tolist()-1]))
    loss.backward()
    optimizer.step()

        # print statistics
    running_loss += loss.item()
    if i % 100 == 99:    
      print('[%d, %5d] loss: %.3f' %
            (epoch + 1, i + 1, running_loss / 20))
      running_loss = 0.0
    #lr_sche.step()
  
  if epoch % 50==49:
    correct = 0
    total = 0
    model.eval()
    with torch.no_grad():
      for data in test_loader:
        
        images, labels = data
        images = images.to(device)
        labels = labels.to(device)
        outputs = model(images)
        
        _, predicted = torch.max(outputs.data, 1)
        
        total += labels.size(0)
        #print(predicted ,labels)
        correct += (predicted == labels).sum().item()

      print('Accuracy of the network on the {} test images: {} %%'.format(len(test_set),
                100 * correct / total))

        


  #if ( epoch==100 or epoch==150 or epoch==200 or epoch==250 or epoch==300 or epoch==350 or epoch==400 or epoch==450 or epoch==500 or epoch==550):
      
  if (epoch==100 or epoch==200 or epoch==300 or  epoch==400 or epoch==500 ):
    torch.save(optimizer.state_dict, PATH+'optimizer{}.pt'.format(trainedepochs+epoch))
    torch.save(model.state_dict,PATH+'model{}.pt'.format(trainedepochs+epoch))

        
        
      
      # loop over the dataset multiple times

    #Check Accuracy
    #acc = acc_check(resnet50, testloader, epoch, save=1)
    
    
torch.save(model.state_dict,PATH+'model{}.pt'.format(trainedepochs+epochs))
torch.save(optimizer.state_dict, PATH+'optimizer{}.pt'.format(trainedepochs+epochs))
print('Finished Training')
torch.cuda.empty_cache() 
correct = 0
total = 0
model.eval() 
with torch.no_grad():
    for data in test_loader:
        
        images, labels = data
        images = images.to(device)
        labels = labels.to(device)
        outputs = model(images)
         
        _, predicted = torch.max(outputs.data, 1)
        
        total += labels.size(0)
        #print(predicted ,labels)
        correct += (predicted == labels).sum().item()

print('Accuracy of the network on the {} test images: {} %%'.format(len(test_set),
    100 * correct / total))<jupyter_output>Accuracy of the network on the 200 test images: 34.5 %%
{'fraction_non_alphanumeric': 0.12891396332863186, 'fraction_numerical': 0.03469675599435825, 'mean_word_length': 2.754324038122132, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '34837', 'license': 'Apache-2.0', 'n_tokens_mistral': 3864, 'n_tokens_neox': 3649, 'n_words': 619, 'path': 'testblock.ipynb', 'repo_name': 'hihi5456/pytorch', 'size': '10635'}