<jupyter_start><jupyter_text>1. 데이터 로드<jupyter_code>train_dir = '../../input/data/train'
test_dir = '../../input/data/eval'
save_dir = '../saved/models/'<jupyter_output><empty_output><jupyter_text>하이퍼파라미터<jupyter_code>#model_name = 'efficientnet_b1'
model_name = 'vit_base_patch16_384'
learning_rate = 2e-5
batch_size = 16
step_size = 5
epochs = 15
earlystop = 5

A_transform = {
    'train':
        A.Compose([
            A.Resize(512, 512),
            A.RandomCrop(384, 384),
            # A.Resize(224, 224),
            A.HorizontalFlip(p=0.5),
            A.Cutout(num_holes=8, max_h_size=32,max_w_size=32),
            A.ElasticTransform(),
            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
            ToTensorV2()
        ]),
    'valid':
        A.Compose([
            A.Resize(384, 384),
            # A.Resize(224, 224),
            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
            ToTensorV2()
        ]),
    'test':
        A.Compose([
            A.Resize(384, 384),
            # A.Resize(224, 224),
            A.Normalize(mean=[0.485, 0.456, 0.406], std=[0.229, 0.224, 0.225]),
            ToTensorV2()
        ])
}
class LoadCSV():
    def __init__(self, dir):
        self.dir = dir
        self.img_dir =train_dir + '/new_images/'
        self.origin_csv_path = train_dir + '/train.csv'
        self.trans_csv_path = train_dir + '/trans_train.csv'
        
        if not os.path.exists(self.trans_csv_path):
            self._makeCSV()
        self.df = pd.read_csv(self.trans_csv_path)
        #self.df = self.df[:200]
    def _makeCSV(self):        
        with open(self.trans_csv_path, 'w', newline='') as f:
            writer = csv.writer(f)
            writer.writerow(["path", "label"])

            df = pd.read_csv(self.origin_csv_path)
            for idx in range(len(df)):
                data = df.iloc[idx]
                img_path_base = os.path.join(os.path.join(self.img_dir, data['path']), '*')
                for img_path in glob.glob(img_path_base):
                    label = 0
                    if "incorrect" in img_path:
                        label+=6
                    elif 'normal' in img_path:
                        label+=12
                    elif data['gender']=='female':
                        label+=3
                    elif data['age'] >= 30 and data['age'] < 60:
                        label+=1
                    elif data['age'] >= 60:
                        label+=2
                    writer.writerow([img_path, label])
        f.close()

class MaskDataset(Dataset):
    def __init__(self, dataframe, transform=None):
        super().__init__()
        self.df = dataframe
        self.transform = transform

    def __len__(self):
        return len(self.df)

    def __getitem__(self, idx):
        class_id = torch.tensor(self.df['label'].iloc[idx])
        img = PIL.Image.open(self.df['path'].iloc[idx])
        img = np.array(img.convert("RGB"))
        if self.transform:
            img = self.transform(image=img)['image']
        return img, class_id<jupyter_output><empty_output><jupyter_text>2. 모델 설계<jupyter_code>class MyModel(nn.Module):
    def __init__(self, model_name, num_classes):
        super(MyModel, self).__init__()
        self.num_classes = num_classes
        self.model = timm.create_model(model_name, pretrained=True)

        # n_features = self.model.classifier.in_features
        # self.model.classifier = torch.nn.Linear(in_features=n_features, out_features=num_classes, bias=True)
        # torch.nn.init.xavier_uniform_(self.model.classifier.weight)
        # stdv = 1/np.sqrt(self.num_classes)
        # self.model.classifier.bias.data.uniform_(-stdv, stdv)

        n_features = self.model.head.in_features
        self.model.head = torch.nn.Linear(in_features=n_features, out_features=self.num_classes, bias=True)
        torch.nn.init.xavier_uniform_(self.model.head.weight)
        stdv = 1/np.sqrt(self.num_classes)
        self.model.head.bias.data.uniform_(-stdv, stdv)
        
    def forward(self, x):
        return self.model(x)
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

criterion = nn.CrossEntropyLoss()
optimizer = torch.optim.Adam(model.parameters(), learning_rate)
lr_scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer, T_max=50, eta_min=0)<jupyter_output><empty_output><jupyter_text>3. 학습<jupyter_code>today = time.strftime('%Y%m%d_%H%M%S', time.localtime(time.time()))
if not os.path.exists(save_dir + today):
    os.makedirs(save_dir + today)
from sklearn.model_selection import StratifiedKFold
mask_csv = LoadCSV(train_dir)
kfold = StratifiedKFold(n_splits=5, shuffle=False)

for fold, (train_idx, valid_idx) in enumerate(kfold.split(mask_csv.df['path'], mask_csv.df['label'])):
    print(f'FOLD {fold}')
    mask_train = MaskDataset(mask_csv.df,  transform=A_transform['train'])
    train_subsampler = torch.utils.data.SubsetRandomSampler(train_idx)
    valid_subsampler = torch.utils.data.SubsetRandomSampler(valid_idx)

    train_loader = DataLoader(mask_train, batch_size=batch_size, sampler=train_subsampler, drop_last=False, num_workers=8, pin_memory=True)
    valid_loader = DataLoader(mask_train, batch_size=batch_size, sampler=valid_subsampler, drop_last=False, num_workers=8, pin_memory=True)
    dataloaders = {'train': train_loader, 'valid':valid_loader}

    model = MyModel(model_name, 18).to(device)
    earlystop_value = 0
    best_acc = 0
    best_loss = 999999999
    best_model = copy.deepcopy(model.state_dict())
    for epoch in range(epochs):
        if earlystop_value >= earlystop:
            break
        train_loss, valid_loss, train_acc_list, valid_acc_list = 0, 0, [],[]

        for phase in ['train', 'valid']:
            if phase == 'train':
                model.train()
            else:
                model.eval()

            running_loss = 0.0
            running_corrects = 0
            with tqdm(dataloaders[phase], total=dataloaders[phase].__len__(), unit="batch") as train_bar:
                for inputs, labels in train_bar:
                    train_bar.set_description(f"{phase} Epoch {epoch} ")
                    inputs, labels = inputs.to(device), labels.to(device)

                    optimizer.zero_grad()
                    outputs = model(inputs)
                    loss = criterion(outputs, labels)
                    
                    if phase == 'train':
                        loss.backward()
                        optimizer.step()

                    outputs = outputs.cpu().detach().numpy()
                    labels = labels.cpu().detach().numpy()

                    running_loss += loss.item() * inputs.size(0)
                    running_corrects += (np.argmax(outputs, axis=1)== labels).mean()
                    epoch_loss = running_loss / len(dataloaders[phase].dataset)
                    epoch_acc = running_corrects / len(dataloaders[phase].dataset)
                    train_bar.set_postfix(loss=epoch_loss, acc=epoch_acc)

            lr_scheduler.step()
            if phase=='valid':
                if epoch_loss < best_loss:
                    best_loss = epoch_loss
                    best_model_wts = copy.deepcopy(model.state_dict())
                    torch.save(best_model_wts, f'{save_dir}{today}/baseline_{model_name}_lr{learning_rate}_stepLR{step_size}_batch{batch_size}_kfold{fold}_epoch{epoch}_valid_loss_{epoch_loss:.5f}.pt')
                    earlystop_value = 0
                else:
                    earlystop_value += 1<jupyter_output>0%|          | 0/945 [00:00<?, ?batch/s]<jupyter_text>4. 추론<jupyter_code>class TestDataset(Dataset):
    def __init__(self, img_paths, transform):
        self.img_paths = img_paths
        self.transform = transform

    def __getitem__(self, index):
        image = PIL.Image.open(self.img_paths[index])
        image = np.array(image.convert("RGB"))
        if self.transform:
            image = self.transform(image=image)
            image = image['image']
        return image

    def __len__(self):
        return len(self.img_paths)
submission = pd.read_csv(os.path.join(test_dir, 'info.csv'))
image_dir = os.path.join(test_dir, 'new_images')

image_paths = [os.path.join(image_dir, img_id) for img_id in submission.ImageID]
dataset = TestDataset(image_paths, A_transform['test'])
test_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False)

model.eval()
all_predictions = []
with tqdm(test_loader, total=test_loader.__len__(), unit="batch") as test_bar:
    for images in test_bar:
        with torch.no_grad():
            images = images.to(device)
            pred = model(images)
            pred = pred.argmax(dim=-1)
            all_predictions.extend(pred.cpu().numpy())
    
submission['ans'] = all_predictions
submission.to_csv(os.path.join(test_dir, 'submission.csv'), index=False)
print('test inference is done!')
import numpy as np
import torch.nn.functional as F
model_num1=MyModel('efficientnet_b3', 18).to(device)
model_num1.load_state_dict(torch.load('/opt/ml/image-classification-level1-04/saved/models/PretrainModelTimm_NoOversampling_Cutout_Elastic_CLAHE_Cutmix_Transform_input_224/0831_204359/model_best.pth')['state_dict'])
model_num2 = MyModel(model_name, 18).to(device)
model_num2.load_state_dict(torch.load('saved/models/20210901_022127/baseline_efficientnet_b1_lr1e-05_stepLR5_kfoldStratifiedKFold(n_splits=5, random_state=None, shuffle=False)_batch16_epoch1_valid_loss_0.46484.pt'))

submission = pd.read_csv(os.path.join(test_dir, 'info.csv'))
image_dir = os.path.join(test_dir, 'images')

image_paths = [os.path.join(image_dir, img_id) for img_id in submission.ImageID]
dataset = TestDataset(image_paths[:200], A_transform['test'])
test_loader = DataLoader(dataset, batch_size=batch_size, shuffle=False)

best_models=[model_num1,model_num2] 

prediction_array=np.zeros((12600,18))
ratio=[0.6,0.4]

for i,model in enumerate(best_models):
    idx=0
    with tqdm(test_loader, total=test_loader.__len__(), unit="batch") as test_bar:
        for images in test_bar:            
            with torch.no_grad():
                predictions_list = []
                images = images.to(device)
                pred = model(images)
                pred=F.softmax(pred,dim=-1)
                pred=pred*ratio[i]
                #pred = pred.argmax(dim=-1)
                pred = pred.tolist()
                batch_idx = batch_size * idx
                #print('pred',pred)
                # images.shape[0] 는 64. 즉, 배치 만큼만 채워 넣는다.
                if (idx+1) == len(test_loader):
                    prediction_array[batch_idx:batch_idx + 8,:] = pred
                    predictions_list.append(prediction_array[...,np.newaxis])
                else :
                    prediction_array[batch_idx: batch_idx + 16, :] = pred
                    predictions_list.append(prediction_array[..., np.newaxis])
                idx+=1
            
print(predictions_list[-1][-16:])    
# for i in range(200):
    
#     print(predictions_list[-1][i])
    #  axis = 2를 기준으로 평균
#predictions_array = np.concatenate(predictions_list, axis = 1)
#predictions_mean = predictions_array.sum(axis = 1)

# 평균 값이 0.5보다 클 경우 1 작으면 0

#print(predictions_mean)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12774045261669023, 'fraction_numerical': 0.02599009900990099, 'mean_word_length': 2.485212569316081, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '62188', 'license': 'MIT', 'n_tokens_mistral': 3993, 'n_tokens_neox': 3774, 'n_words': 594, 'path': 'jupyter/baseline kfold.ipynb', 'repo_name': 'gkswns3708/boostcampaitech2-image-classification-level1-04', 'size': '11312'}