<jupyter_start><jupyter_text>!!!!!!!!!!!!!! Train!!!!!!!!!!!!!!!!!!!!<jupyter_code># train_csv.shape
# train_csv = train_csv[0:1500]
#Create ImageDataset class
class ImageDataset_train(Dataset):
    def __init__(self, csv, train, test):
        self.csv = csv
        self.train = train
        self.test = test
        self.all_image_names = self.csv[:]['StudyInstanceUID']
        self.all_labels = np.array(self.csv.drop(['StudyInstanceUID', 'PatientID'], axis=1))
        self.train_ratio = int(0.85 * len(self.csv))
        self.valid_ratio = len(self.csv) - self.train_ratio
        # set the training data images and labels
        if self.train == True:
            print(f"Number of training images: {self.train_ratio}")
            self.image_names = list(self.all_image_names[:self.train_ratio])
            self.labels = list(self.all_labels[:self.train_ratio])
            # define the training transforms
            self.transform = transforms.Compose([
#                 transforms.ToPILImage(),
                transforms.CenterCrop(2800),
                transforms.Resize((args['shape'], args['shape'])),
#                 transforms.RandomAffine(degrees=10),
#                 transforms.RandomHorizontalFlip(p=0.3),
#                 transforms.RandomCrop(size=32),
#                 transforms.RandomErasing(p=0.2, scale=(0.02, 0.33), ratio=(0.3, 3.3), value=0, inplace=False),
#                 transforms.RandomRotation(degrees=),
                transforms.ToTensor(),
                transforms.Normalize(*stats)
            ])
            
            # set the validation data images and labels
        elif self.train == False and self.test == False:
            print(f"Number of validation images: {self.valid_ratio}")
#             self.image_names = list(self.all_image_names[-self.valid_ratio:-10])
            self.image_names = list(self.all_image_names[-self.valid_ratio:])
            self.labels = list(self.all_labels[-self.valid_ratio:])
            # define the validation transforms
            self.transform = transforms.Compose([
#                 transforms.ToPILImage(),
                transforms.CenterCrop(2800),
                transforms.Resize((args['shape'], args['shape'])),
                transforms.ToTensor(),
                transforms.Normalize(*stats)
                ])
            
            
    
    def __len__(self):
        return len(self.image_names)
    
    def __getitem__(self, index):
        image = Image.open(f"../input/ranzcr-clip-catheter-line-classification/train/{self.image_names[index]}.jpg")  
        image = self.transform(image)
        targets = self.labels[index]
        
        return {
            'image': torch.tensor(image, dtype=torch.float32),
            'label': torch.tensor(targets, dtype=torch.float32)
        }
            
for i in range(2):
    image = Image.open(f"../input/ranzcr-clip-catheter-line-classification/train/"+list_train[i]) 
    print(np.array(image).shape)
    print(image)

# Create Test dataset
class ImageDataset_test(Dataset):

    def __init__(self, root, image_dir, transform=None):
        self.root = root
        self.image_dir = image_dir
        self.image_files = os.listdir(image_dir)
        self.transform = transform

    def __len__(self):
        return len(self.image_files)

    def __getitem__(self, index):
        image_name = os.path.join(self.image_dir, self.image_files[index])  
        image = Image.open(image_name)
        #label = self.data[index]
        if self.transform:
            image = self.transform(image)
        return image

stats = (0.4454318881034851, 0.25035765767097473)<jupyter_output><empty_output><jupyter_text>Test dataset<jupyter_code>root = data_dir
image_dir = root+'test/'
transform_img = transforms.Compose([ 
#     transforms.ToPILImage(),
                                    transforms.CenterCrop(2800),
                                    transforms.Resize((args['shape'], args['shape'])),
                                    transforms.ToTensor(),
                                    transforms.Normalize(*stats)
])

test_ds = ImageDataset_test(root,image_dir, transform = transform_img)
test_loader = DataLoader(test_ds,1,shuffle=False,num_workers = args['workers'])
for i in test_loader:
    
    print(i)
    break<jupyter_output>tensor([[[[-1.7792, -1.7792, -1.7792,  ..., -1.7792, -1.7792, -1.7792],
          [-1.7792, -1.7792, -1.7792,  ..., -1.7792, -1.7792, -1.7792],
          [-1.7792, -1.7792, -1.7792,  ..., -1.7792, -1.7792, -1.7792],
          ...,
          [-1.7792, -1.7792, -1.7792,  ..., -1.7792, -1.7792, -1.7792],
          [-1.7792, -1.7792, -1.7792,  ..., -1.7792, -1.7792, -1.7792],
          [-1.7792, -1.7792, -1.7792,  ..., -1.7792, -1.7792, -1.7792]]]])<jupyter_text>train dataset<jupyter_code>#train_ds
train_ds = ImageDataset_train(train_csv, train=True, test=False)<jupyter_output>Number of training images: 25570<jupyter_text>validation dataset<jupyter_code>#val_ds
val_ds = ImageDataset_train(train_csv, train=False, test=False)<jupyter_output>Number of validation images: 4513<jupyter_text>Dataloader<jupyter_code>batch_size = args['batch']
# train data loader
# train_loader = DataLoader(train_ds, batch_size=batch_size,shuffle=True)
# validation data loader
# valid_loader = DataLoader(val_ds,batch_size=batch_size,shuffle=False)
# test data loader

train_ds[0]['image'].shape
lab_name = sample_submission.columns.drop(['StudyInstanceUID'])

#plot an example

def show_example(dataset):
    print("labels: {}".format(dataset['label']))
    print(lab_name[dataset['label']==1])
    plt.imshow(dataset['image'].squeeze())
    plt.show
      
show_example(train_ds[15])
# means = []
# stds = []
# for img in tqdm(train_ds):
#     means.append(torch.mean(img['image']))
#     stds.append(torch.std(img['image']))
    
    
# mean = torch.mean(torch.tensor(means))
# std = torch.mean(torch.tensor(stds))
# stats = mean.item(), std.item()
# stats
#Create dictinary with train and val
image_datasets = {'train':train_ds, 'val':val_ds}
#batch_size
batch = batch_size
#Create dictionary with train dataloader and val dataloader
dataloaders = {x: torch.utils.data.DataLoader(image_datasets[x], batch_size = batch, shuffle=True,num_workers = 4) 
               for x in ['train', 'val']}
#Dataset sizes
dataset_sizes = {x: len(image_datasets[x]) for x in ['train', 'val']}
#label names
class_names = lab_name

len(dataloaders['train'])
for i in dataloaders['train']:
    print(i['image'].shape)
    
#     print(label['label'])
    break
from torchvision.utils import make_grid

def show_batch(dl):
    for images in tqdm(dl, desc = "Progress"):
        fig, ax = plt.subplots(figsize = (22,18))
        ax.set_xticks([]); ax.set_yticks([])
        ax.imshow(make_grid(images['image'],10).permute(1,2,0))
        break


show_batch(dataloaders['train'])

import time
import copy
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
device
for i in dataloaders['train']:
    print(i['image'])
    print(i['label'])
    break


def train_model(model, criterion, optimizer, scheduler=None, num_epochs=25, grad_clip=None):

   
    since = time.time()

    best_model_wts = copy.deepcopy(model.state_dict())
    best_loss = 10.0

    for epoch in tqdm(range(num_epochs)):
        print('Epoch {}/{}'.format(epoch, num_epochs - 1))
        print('-' * 10)
        

        # Each epoch has a training and validation phase
        for phase in ['train', 'val']:
            
            if phase == 'train':
                model.train()  # Set model to training mode
            else:
                model.eval()   # Set model to evaluate mode

            running_loss = 0.0
            running_corrects = 0

            # Iterate over data.
            for i in dataloaders[phase]:
                inputs = i["image"].to(device)
                labels = i["label"].to(device)

                # zero the parameter gradients
                optimizer.zero_grad()
                # forward
                # track history if only in train
                with torch.set_grad_enabled(phase == 'train'):
                    outputs = model(inputs)
                    # apply sigmoid activation to get all the outputs between 0 and 1
                    outputs = torch.sigmoid(outputs)
                    loss = criterion(outputs, labels)
                    
                    
                    # backward + optimize only if in training phase
                    if phase == 'train':
                        loss.backward()
                        #Gradiend clipping
                        if grad_clip:
                            nn.utils.clip_grad_value_(model.parameters(), grad_clip)
                            
                        optimizer.step()

                # statistics
                running_loss += loss.item() * inputs.size(0)
#                 running_corrects += torch.sum(outputs == labels.data)
            if phase == 'train':
                
                if scheduler:
                    scheduler.step()

            epoch_loss = running_loss / dataset_sizes[phase]
            
            
#             epoch_acc = running_corrects.double() / dataset_sizes[phase]
            print('{} Loss: {:.4f} '.format(phase, epoch_loss))
                
#             writer.add_scalar('epoch_loss',epoch_loss, epoch)  
#             logger.report_scalar(title = phase, series='epoch_loss',iteration = epoch, value = epoch_loss)
#             logger.report_scalar(title =phase, series='epoch_acc',iteration = epoch, value = epoch_acc)

            # deep copy the model
            if phase == 'val' and epoch_loss < best_loss:
                print(phase)
                best_loss = epoch_loss
                
#                 print("best_acc_val:" , best_loss)
                best_model_wts = copy.deepcopy(model.state_dict())

        print()

    time_elapsed = time.time() - since
    print('Training complete in {:.0f}m {:.0f}s'.format(
        time_elapsed // 60, time_elapsed % 60))
    print('Best val loss: {:4f}'.format(best_loss))

    # load best model weights
    model.load_state_dict(best_model_wts)
    
    return model
# Conv block function
def conv_block(in_channels, out_channels, pool = False):
    layers = [nn.Conv2d(in_channels, out_channels, kernel_size = 3, padding=1),
             nn.BatchNorm2d(out_channels),
             nn.ReLU(inplace = True)]
    
    if pool: layers.append(nn.MaxPool2d(2))
   
   
    return nn.Sequential(*layers) #asterki gia na parei olo to dictionary


# Resnet 
class ResNet(nn.Module):
    def __init__(self, in_channels, num_classes):
        super().__init__()  # 1*128*128 = 16384
        self.conv1 = conv_block(in_channels, 64)  #128
        self.conv2 = conv_block(64,128,pool=True) # 64
        self.res1 = nn.Sequential(conv_block(128,128),conv_block(128,128))
        self.conv3 = conv_block(128,128,pool=True) #32
        self.conv4 = conv_block(128,256,pool=True)# 16
        self.res2 = nn.Sequential(conv_block(256,256),conv_block(256,256))
        self.conv5 = conv_block(256,512,pool=True) # 8
        self.conv6 = conv_block(512,512,pool=True) # 4
        self.res3 = nn.Sequential(conv_block(512,512),conv_block(512,512))
#         self.conv7 = conv_block(256,512,pool=True) # 8
#         self.conv8 = conv_block(512,512,pool=True) # 4
#         self.res4 = nn.Sequential(conv_block(512,512),conv_block(512,512))
#         self.conv9 = conv_block(512,512,pool=True) # 2
#         self.conv10 = conv_block(512,512,pool=True) # 1
        self.classifier = nn.Sequential(nn.MaxPool2d(4),
                
                                        nn.Flatten(),
                                        nn.Dropout(args['Dropout']),
                                        nn.Linear(512,num_classes)
        )
                                            
                                       
        
    def forward(self,xb):
        out = self.conv1(xb)
        out = self.conv2(out)
        out = self.res1(out) + out
        out = self.conv3(out)
        out = self.conv4(out)
        out = self.res2(out) + out
        out = self.conv5(out) 
        out = self.conv6(out)
        out = self.res3(out) + out
#         out = self.conv7(out)
#         out = self.conv8(out)
#         out = self.res4(out) + out
#         out = self.conv9(out)
#         out = self.conv10(out)
        out = self.classifier(out)
        
        return out
    
model_ft = ResNet(1,11)
model_ft
## Pretrained
# model_ft = models.resnet18(pretrained=True)
# num_ftrs = model_ft.fc.in_features
# model_ft.conv1=nn.Conv2d(1, 64, kernel_size=(7, 7), stride=(2, 2), padding=(3, 3), bias=False)
# model_ft.fc = nn.Linear(num_ftrs, 11)


model_ft = model_ft.to(device)

criterion = nn.BCELoss() #binary cross entropy loss
# Observe that all parameters are being optimized
optimizer_ft = torch.optim.Adam(model_ft.parameters(), 
                                lr=args['initial_lr'], 
                                weight_decay = args['weight_decay'])

# Decay LR by a factor of 0.1 every 7 epochs
exp_lr_scheduler = lr_scheduler.OneCycleLR(optimizer_ft,
                                           max_lr = args['max_lr'],
                                           epochs = args['epochs'],
                                           steps_per_epoch = len(dataloaders['train']),
                                           verbose =True)

model_ft = train_model(model_ft, criterion, optimizer_ft, exp_lr_scheduler,
                       num_epochs=args['epochs'], grad_clip = args['grad_clip'])

# torch.save(model_ft, "model_ft_manualResnet11.pth")
# # Model class must be defined somewhere
# model_ft = torch.load("../input/ranzcr-clip-catheter-line-classification/model_ft.pth")
# # model1.eval()<jupyter_output><empty_output><jupyter_text>Test DataSet<jupyter_code>model_ft.to(device)
model_ft.eval()

StudyInstanceUID = pd.DataFrame({"StudyInstanceUID":os.listdir(root+'test')})
# print(StudyInstanceUID)

submission = pd.DataFrame()
for img in tqdm(test_loader):

    output = model_ft(img.to(device))

    output = torch.sigmoid(output)
    output = np.array(output.detach().cpu())
#     print(output)
    kouvas = pd.DataFrame(data = output)
    submission = pd.concat([submission, kouvas],axis = 0)
#     break

submission.head()    

#Restet index in submission
submission = submission.reset_index(drop=True)
# submission.index
submission = pd.concat([StudyInstanceUID,submission],axis=1)
submission['StudyInstanceUID']=submission['StudyInstanceUID'].str.replace(".jpg","")

submission.columns = sample_submission.columns
submission.to_csv("submission.csv",index=False)
submission<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1296472184531886, 'fraction_numerical': 0.03426051560379918, 'mean_word_length': 2.271415889924545, 'pattern_counts': {'":': 1, '<': 23, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 17, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '559739', 'license': 'MIT', 'n_tokens_mistral': 4944, 'n_tokens_neox': 4595, 'n_words': 835, 'path': 'ranzcr-catheter.ipynb', 'repo_name': 'GiorgosAnastasakos/PythonCatheterLineML', 'size': '14740'}