<jupyter_start><jupyter_text>CUSTOM COVID-19 X-RAYs SCANS DATASET BUILDER<jupyter_code>import torch
import torchvision
import torchvision.transforms as transforms
from torchsummary import summary
import os
from tqdm import notebook
from sklearn.metrics import classification_report, confusion_matrix
from tabulate import tabulate
import gc
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image
!git clone https://github.com/vj2050/Transfer-Learning-COVID-19.git
!git clone https://github.com/SimoneRosset/AUGMENTATION_GAN.git
for func in [lambda: os.mkdir(os.path.join('.','AUGMENTATION_GAN/custom_covid_dataset/')),
            lambda: os.mkdir(os.path.join('.', 'AUGMENTATION_GAN/custom_covid_dataset/test/')),
            lambda: os.mkdir(os.path.join('.', 'AUGMENTATION_GAN/custom_covid_dataset/test/covid')),
            lambda: os.mkdir(os.path.join('.', 'AUGMENTATION_GAN/custom_covid_dataset/test/normal')),
            lambda: os.mkdir(os.path.join('.', 'AUGMENTATION_GAN/custom_covid_dataset/test/pneumonia_bac')),
            lambda: os.mkdir(os.path.join('.', 'AUGMENTATION_GAN/custom_covid_dataset/test/pneumonia_vir'))]:
  try:
    func()
  except Exception as error:
    print(error)
    continue

sets=[500,1000,2000,3000]

paths=[]

for i in range(len(sets)):
  paths.append('AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/')
  paths.append('AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/covid')
  paths.append('AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/normal')
  paths.append('AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/pneumonia_bac')
  paths.append('AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/pneumonia_vir')

for path in paths:
  try:
    os.mkdir(os.path.join('.', path))
  except Exception as error:
    print(error)
    continue
img_dim=64
N_rot=2
N_flip=2
N_tran=4
N_scal=4
N_color=2
transform_list=[]

transform_list.append(transforms.Resize((img_dim*2,img_dim*2),interpolation=2))

for i in range(N_color):
  transform_list.append(transforms.RandomApply([transforms.ColorJitter(brightness=0.5, contrast=0.5, saturation=0.5, hue=0.5)],p=0.5))
for i in range(N_rot):
  transform_list.append(transforms.RandomApply([transforms.RandomRotation(degrees=(-5,5))], p=0.5))
for i in range(N_flip):
  transform_list.append(transforms.RandomHorizontalFlip(p=0.5))
for i in range(N_scal):
  transform_list.append(transforms.RandomApply([transforms.RandomAffine(degrees=0,scale=(0.9,1.1))], p=0.5))
for i in range(N_tran):
  transform_list.append(transforms.RandomApply([transforms.RandomAffine(degrees=0,translate=(0.1,0.1))], p=0.5))

transform_list.append(transforms.Resize((img_dim,img_dim),interpolation=2))

transform_list.append(transforms.ToTensor())

transform = transforms.Compose(transform_list)

test_transform = transforms.Compose([transforms.Resize((img_dim,img_dim),interpolation=2),
                                    transforms.ToTensor(),
                                    ])
class MyImageFolder(torchvision.datasets.ImageFolder):
  def __getitem__(self, index):
    return super(MyImageFolder, self).__getitem__(index)[0], super(MyImageFolder, self).__getitem__(index)[1], self.imgs[index][0]#return image path
batch_size=1
data_dir='Transfer-Learning-COVID-19/dataset_4_classes/'

trainset = MyImageFolder(os.path.join(
          data_dir, "train/"), transform=transform)
trainloader = torch.utils.data.DataLoader(
          trainset, batch_size=batch_size, shuffle=True)
testset = MyImageFolder(os.path.join( 
          data_dir, "test/"), transform=test_transform)
testloader = torch.utils.data.DataLoader(
          testset, batch_size=batch_size, shuffle=False)
normalset = MyImageFolder(os.path.join(
          data_dir, "train/"), transform=test_transform)
normalloader = torch.utils.data.DataLoader(
          normalset, batch_size=batch_size, shuffle=False)

classes = ("covid", "normal", "pneumonia_bac","pneumonia_vir")
# functions to show an image

def imshow(img):
    #img = img / 2 + 0.5     # unnormalize
    npimg = img.numpy()
    plt.imshow(np.transpose(npimg, (1, 2, 0)))
    plt.show()

# get some random training images
dataiter = iter(trainloader)
images, labels, path = dataiter.next()

# show images
imshow(torchvision.utils.make_grid(images))
# print labels
print(' '.join('%5s' % classes[labels[j]] for j in range(batch_size)))

name, ext= (path[0].split('/')[-1]).rsplit('.', 1)
#MAKE FOLDER TEST
dataiter = iter(testloader)

for i in range(len(testloader.dataset)):
    images, labels, path = dataiter.next()
   
    npimg = np.transpose(images.squeeze(0).numpy(), (1, 2, 0))

    img = Image.fromarray((npimg * 255).astype(np.uint8))

    name, ext= (path[0].split("/")[-1]).rsplit('.', 1)

    fold=classes[labels[0]]

    abs_path='AUGMENTATION_GAN/custom_covid_dataset/test/'+fold+'/'+name+'.'+ext

    if ext=='png':
      img.save(abs_path, 'PNG')
    else:
      img.save(abs_path, 'JPEG')
#MAKE FOLDER 0
dataiter = iter(normalloader)

for i in range(len(normalloader.dataset)):
    images, labels, path = dataiter.next()
   
    npimg = np.transpose(images.squeeze(0).numpy(), (1, 2, 0))

    img = Image.fromarray((npimg * 255).astype(np.uint8))

    name, ext= (path[0].split("/")[-1]).rsplit('.', 1)

    fold=classes[labels[0]]

    abs_path='AUGMENTATION_GAN/custom_covid_dataset/train/'+fold+'/'+name+'.'+ext

    if ext=='png':
      img.save(abs_path, 'PNG')
    else:
      img.save(abs_path, 'JPEG')
dataiter = iter(normalloader)

images, labels, path = dataiter.next()
   
npimg = np.transpose(images.squeeze(0).numpy(), (1, 2, 0))

im = Image.fromarray((npimg * 255).astype(np.uint8))

display(im)
dataiter = iter(trainloader) # MAKE OTHER FOLDERS

for i in range(1,len(sets),1):
  covid=sets[i-1] #sequential
  normal=sets[i-1]
  pneu=sets[i-1]
  vir=sets[i-1]
  # covid=0 
  # normal=0
  # pneu=0
  none=1
  print('folder start: ',sets[i])
  dataiter = iter(trainloader)
 
  while covid<sets[i] or normal<sets[i] or pneu<sets[i] or vir<sets[i]:

    images, labels, path = dataiter.next()
   
    npimg = np.transpose(images.squeeze(0).numpy(), (1, 2, 0))

    img = Image.fromarray((npimg * 255).astype(np.uint8))

    name, ext= (path[0].split('/')[-1]).rsplit('.', 1)

    fold=classes[labels[0]]


    if fold == 'covid' and covid<sets[i]:
      abs_path='AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/'+fold+'/'+str(sets[i])+'.'+str(covid)+'.'+name+'.'+ext
      covid+=1
      if ext=='png':
        img.save(abs_path, 'PNG')
      else:
        img.save(abs_path, 'JPEG')
    else:
      if fold == 'normal' and normal<sets[i]:
        abs_path='AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/'+fold+'/'+str(sets[i])+'.'+str(normal)+'.'+name+'.'+ext
        normal+=1
        if ext=='png':
          img.save(abs_path, 'PNG')
        else:
          img.save(abs_path, 'JPEG')
      else:
        if fold == 'pneumonia_bac' and pneu<sets[i]:
          abs_path='AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/'+fold+'/'+str(sets[i])+'.'+str(pneu)+'.'+name+'.'+ext
          pneu+=1
          if ext=='png':
            img.save(abs_path, 'PNG')
          else:
            img.save(abs_path, 'JPEG')
        else:
          if fold == 'pneumonia_vir' and vir<sets[i]:
            abs_path='AUGMENTATION_GAN/custom_covid_dataset/train_classic/'+str(sets[i])+'/'+fold+'/'+str(sets[i])+'.'+str(vir)+'.'+name+'.'+ext
            vir+=1
            if ext=='png':
              img.save(abs_path, 'PNG')
            else:
              img.save(abs_path, 'JPEG')
          else:
            none+=1

    if (covid+normal+pneu+vir+none)%200==0:
      dataiter = iter(trainloader)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1677551543091305, 'fraction_numerical': 0.018696375976437443, 'mean_word_length': 4.084635416666667, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '592929', 'license': 'MIT', 'n_tokens_mistral': 2981, 'n_tokens_neox': 2878, 'n_words': 441, 'path': 'DATASET_BUILDER.ipynb', 'repo_name': 'rossettisimone/AUGMENTATION_GAN', 'size': '7809'}