<jupyter_start><jupyter_text>Preparing Dataframe  Configuration<jupyter_code>class config:
    BASE_LR = 1e-4
    NUM_CLASSES = 20
    BIN_SIZE = int(200/NUM_CLASSES)
    NUM_EPOCHS = 8
    MODEL_NAMES = ["mixnet_xl", "dpn68b", "mixnet_l", "efficientnet_es", "mobilenetv3_large_100", "seresnext26t_32x4d"]
    MODEL_NAME = None
    OPTIMIZER_NAME = "Ranger"
    FILE_PREFIX = "stack2-cyclr-oldaug"
    STACK = 2
    SEED = 43
    IMG_SIZE = 320
    MEAN = [0.485, 0.456, 0.406]
    STD = [0.229, 0.224, 0.225]
    BATCH_SIZES = [64]*len(MODEL_NAMES)
    BATCH_SIZE = 32
    WORKERS = 8
    FOLD = [0, 1, 2, 3, 4]
    DEBUG = False
    MODE = 6 #{0: Train; 1: Val Logits; 2: Test Logits; 3: ALL; 4: Embeddings; 5: Train Logits}
def seed_everything(seed):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed(seed)
    torch.backends.cudnn.deterministic = True
seed_everything(config.SEED)<jupyter_output><empty_output><jupyter_text>Dataloader<jupyter_code>from albumentations import (
    HorizontalFlip, VerticalFlip, ShiftScaleRotate, Transpose, HueSaturationValue, MotionBlur, 
    RandomResizedCrop, RandomBrightnessContrast, OneOf, Compose, Normalize, Cutout, CoarseDropout,
    CenterCrop, Resize, RandomCrop, CenterCrop
)

from albumentations.pytorch import ToTensorV2

def get_train_transforms():
    return Compose([
            RandomCrop(config.IMG_SIZE, config.IMG_SIZE),
            Transpose(p=0.5),
            HorizontalFlip(p=0.5),
            VerticalFlip(p=0.5),
            ShiftScaleRotate(p=0.5),
            Normalize(mean=config.MEAN, std=config.STD, max_pixel_value=255.0, p=1.0),
            ToTensorV2(p=1.0),
        ], p=1.)
  
        
def get_valid_transforms(tta=False):
    aug = []
    if not tta: aug.extend([CenterCrop(config.IMG_SIZE, config.IMG_SIZE)])
    aug.extend([
            Normalize(mean=config.MEAN, std=config.STD, max_pixel_value=255.0, p=1.0),
            ToTensorV2(p=1.0),
        ])
    return Compose(aug)

train_transform = get_train_transforms()
valid_transform = get_valid_transforms()
tta_test_transform = get_valid_transforms(tta=True)

tta_transforms = tta.Compose(
    [
#         tta.HorizontalFlip(),
#         tta.VerticalFlip(),
        tta.FiveCrops(config.IMG_SIZE, config.IMG_SIZE)  
    ]
)
class DatasetWIND(Dataset):
    """Reads in an image, transforms pixel values, and serves
    a dictionary containing the image id, image tensors, and label.
    """

    def __init__(self, df, df_train, transforms = None, phase='train'):
        self.data = df.reset_index(drop=True)
        self.data_train = df_train.reset_index(drop=True)
        self.label = df.wind_speed
        self.transform = transforms
        
    def __len__(self):
        return len(self.data)

    def __getitem__(self, index):
        image = cv2.imread(self.data.iloc[index]["file_name"])
        temp = image
        index_train = self.data_train[self.data_train.image_id==self.data.image_id.iloc[index]].index.values[0]
        if index_train-config.STACK<0:
            image = image
        else:
            image_num = int(self.data_train.image_id.iloc[index_train].split('_')[-1])
            image_nump = int(self.data_train.image_id.iloc[index_train-1].split('_')[-1])
            image_numpp = int(self.data_train.image_id.iloc[index_train-2].split('_')[-1])
            if image_numpp-image_num==-2:
                image[...,1] = cv2.imread(self.data_train.iloc[index_train-1]["file_name"])[...,0]
                image[...,0] = cv2.imread(self.data_train.iloc[index_train-2]["file_name"])[...,0]
            else:
                image = image
                
        if self.transform is not None:
            image = self.transform(image=image)["image"]
        if self.label is not None:
            wind_speed = self.label.iloc[index].astype(float)
            label = np.zeros(config.NUM_CLASSES)
            label[:int(wind_speed//config.BIN_SIZE)]=1.
            label[int(wind_speed//config.BIN_SIZE)]=(wind_speed%config.BIN_SIZE)/config.BIN_SIZE
            return image, label.astype(np.float32)
        else:
            return image<jupyter_output><empty_output><jupyter_text>Training<jupyter_code>def run_epoch(model, loss_fn, optimizer, phase, scheduler=None):
    running_loss = AverageMeter()
    tk1 = tqdm_notebook(dataloaders[phase], total=len(dataloaders[phase]))
    if phase == "train":
        model.train()
        for x_var, y_var in tk1:
            bs = x_var.shape[0]
            x_var = x_var.to(device=device)
            y_var = y_var.to(device=device)
            
            with torch.cuda.amp.autocast():
                scores = model(x_var)
                loss = loss_fn(scores, y_var)            
            running_loss.update(loss.item(), n=bs)
            tk1.set_postfix(loss=running_loss.avg)
            optimizer.zero_grad()
            scaler.scale(loss).backward()
            scaler.step(optimizer)
            if scheduler is not None:
                scheduler.step()
            scaler.update()

            if config.DEBUG: break
        return running_loss.avg
    else:
        model.eval()
        y_true = np.array([])
        y_pred = np.array([])
        running_mse = AverageMeter()
        with torch.no_grad():
            for x_var, y_var in tk1:
                bs = x_var.shape[0]
                x_var = x_var.to(device=device)
                y_var = y_var.to(device=device)
                with torch.cuda.amp.autocast():
                    scores = model(x_var)
                    loss = loss_fn(scores, y_var)
                
                scores = torch.sum(F.sigmoid(scores), 1)*config.BIN_SIZE
                y_var = torch.sum(y_var, 1)*config.BIN_SIZE
                    
                
                running_loss.update(loss.item(), n=bs)
                
                y_var = y_var.cpu().detach().numpy()
                scores = scores.cpu().detach().numpy()
                
                mse = np.sum((scores-y_var)**2)/bs
                running_mse.update(mse, n=bs)
                tk1.set_postfix(loss=running_loss.avg, rmse=math.sqrt(running_mse.avg))
                if config.DEBUG: break
        rmse = math.sqrt(running_mse.avg)
        return running_loss.avg, rmse
if config.MODE in [0,3]:
    for config.MODEL_NAME, config.BATCH_SIZE in zip(config.MODEL_NAMES, config.BATCH_SIZES):
        for fold in config.FOLD:
            scaler = torch.cuda.amp.GradScaler()
            X = pd.read_csv('new_split.csv')
            train = X[X.fold!=fold]
            val = X[X.fold==fold]
            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

            train_losses = []
            valid_losses = []
            valid_rmse = []

            trainset = DatasetWIND(train, train, transforms = train_transform, phase = 'train')
            validset = DatasetWIND(val, X, transforms = valid_transform, phase = 'val')

            train_loader = DataLoader(trainset, config.BATCH_SIZE, num_workers=config.WORKERS, shuffle=True, pin_memory=True)
            valid_loader = DataLoader(validset, config.BATCH_SIZE, num_workers=config.WORKERS, shuffle=False, pin_memory=True)

            dataloaders = {
                "train" : train_loader,
                "valid" : valid_loader,
            }

            model = globals()[config.MODEL_NAME](pretrained=True, num_classes=config.NUM_CLASSES, in_chans=config.STACK+1)
        #     load_pretrained(model)
            model.to(device)
            criterion = nn.BCEWithLogitsLoss()
            optimizer = globals()[config.OPTIMIZER_NAME](model.parameters(), lr=config.BASE_LR)
            scheduler = torch.optim.lr_scheduler.CyclicLR(optimizer, config.BASE_LR, config.BASE_LR*10, step_size_up=len(train_loader)//2, cycle_momentum=False)
            best_rmse = 10000
            for epoch in range(config.NUM_EPOCHS):
                print('Starting epoch [%d / %d]' % (epoch + 1, config.NUM_EPOCHS))
                train_loss = run_epoch(model, criterion, optimizer, "train", scheduler)
                valid_loss, rmse = run_epoch(model, criterion, optimizer, "valid")


                if rmse<best_rmse:
                    print("**Saving model**")
                    best_rmse=rmse
                    torch.save({
                        "epoch": epoch + 1,
                        "state_dict" : model.state_dict(),
                        "rmse" : best_rmse,
                        "optim_dict" : optimizer.state_dict(),
                        "config_class" : config
                    }, f"models/{config.MODEL_NAME}_{config.FILE_PREFIX}_FOLD{fold}.pth")

                train_losses.append(train_loss)
                valid_losses.append(valid_loss)
                valid_rmse.append(rmse)
                df_data=np.array([train_losses, valid_losses, valid_rmse]).T
                df = pd.DataFrame(df_data, columns = ['train_losses','valid_losses','valid_rmse'])
                df.to_csv(f'logs/{config.MODEL_NAME}_{config.FILE_PREFIX}_FOLD{fold}.csv')
                if config.DEBUG: break<jupyter_output><empty_output><jupyter_text>Validation logits<jupyter_code>mkdir val_npys
if config.MODE in [1, 3]:
    for config.MODEL_NAME, config.BATCH_SIZE in zip(config.MODEL_NAMES, config.BATCH_SIZES):
        for fold in config.FOLD:
            X = pd.read_csv('new_split.csv')
            train = X[X.fold!=fold]
            val = X[X.fold==fold]

#             x_val = val.drop("wind_speed", axis=1)
#             y_val = val.wind_speed
            validset = DatasetWIND(val, X, transforms = valid_transform, phase = 'val')
            valid_loader = DataLoader(validset, config.BATCH_SIZE, num_workers=config.WORKERS, shuffle=False, pin_memory=True)
            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
            model = globals()[config.MODEL_NAME](pretrained=True, num_classes=config.NUM_CLASSES)
            model.load_state_dict(torch.load(f"models/{config.MODEL_NAME}_{config.FILE_PREFIX}_FOLD{fold}.pth")["state_dict"])
            model.to(device)
            model.eval()
            y_logits = []
            tk1 = tqdm_notebook(valid_loader, total=len(valid_loader))
            with torch.no_grad():
                for x_var, y_var in tk1:
                    bs = x_var.shape[0]
                    scores = model(x_var.to(device))
                    y_logits.append((F.sigmoid(scores)))

            y_logits = torch.cat(y_logits).cpu().numpy()
            np.save(f'val_npys/{config.MODEL_NAME}_{config.FILE_PREFIX}_val_preds_fold_{fold}.npy', y_logits)<jupyter_output><empty_output><jupyter_text>Train Logits<jupyter_code>mkdir train_npys
if config.MODE in [5, 3]:
    for config.MODEL_NAME, config.BATCH_SIZE in zip(config.MODEL_NAMES, config.BATCH_SIZES):
        for fold in config.FOLD:
            X = pd.read_csv('new_split.csv')
            train = X[X.fold!=fold]
            val = X[X.fold==fold]

#             x_train = train.drop("wind_speed", axis=1)
#             y_train = train.wind_speed
            trainset = DatasetWIND(train, X, transforms = valid_transform, phase = 'val')
            train_loader = DataLoader(trainset, config.BATCH_SIZE, num_workers=config.WORKERS, shuffle=False, pin_memory=True)
            device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
            model = globals()[config.MODEL_NAME](pretrained=False, num_classes=config.NUM_CLASSES)
            model.load_state_dict(torch.load(f"models/{config.MODEL_NAME}_{config.FILE_PREFIX}_FOLD{fold}.pth")["state_dict"])
            model.to(device)
            model.eval()
            y_logits = []
            tk1 = tqdm_notebook(train_loader, total=len(train_loader))
            with torch.no_grad():
                for x_var, y_var in tk1:
                    bs = x_var.shape[0]
                    scores = model(x_var.to(device))
                    y_logits.append((F.sigmoid(scores)))

            y_logits = torch.cat(y_logits).cpu().numpy()
            np.save(f'train_npys/{config.MODEL_NAME}_{config.FILE_PREFIX}_train_preds_fold_{fold}.npy', y_logits)<jupyter_output><empty_output><jupyter_text>Test Logits<jupyter_code>mkdir test_npys
x_test = pd.read_csv('data/test_set_features.csv')
x_test["file_name"] = "data/test/test/"+x_test.image_id+".jpg"
# testset = DatasetWIND(x_train = x_test, y_train = None, transforms = tta_test_transform, phase = 'test')
# test_loader = DataLoader(testset, config.BATCH_SIZE*4, num_workers=config.WORKERS, shuffle=False, pin_memory=True)
if config.MODE in [2, 3]:
    for config.MODEL_NAME, config.BATCH_SIZE in zip(config.MODEL_NAMES, config.BATCH_SIZES):
        PREDS = np.zeros(len(testset))
        device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        for fold in config.FOLD:
            model = globals()[config.MODEL_NAME](pretrained=True, num_classes=config.NUM_CLASSES)
            model.load_state_dict(torch.load(f"models/{config.MODEL_NAME}_{config.FILE_PREFIX}_FOLD{fold}.pth")["state_dict"])
            model.to(device)
            model = tta.ClassificationTTAWrapper(model, tta_transforms)
            model.eval()
            y_logits = []
            tk1 = tqdm_notebook(test_loader, total=len(test_loader))
            with torch.no_grad():
                for x_test in tk1:
                    bs = x_test.shape[0]
                    scores = model(x_test.to(device))
                    y_logits.append((F.sigmoid(scores)))
            y_logits = torch.cat(y_logits).cpu().numpy()
            np.save(f'test_npys/{config.MODEL_NAME}_{config.FILE_PREFIX}_preds_fold_test_{fold}.npy', y_logits)<jupyter_output><empty_output><jupyter_text>Embeddings<jupyter_code>mkdir class_embeddings
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
X = pd.read_csv('new_split.csv')
x_train = X
y_train = X.wind_speed
trainset = DatasetWIND(X, X, transforms = valid_transform, phase = 'val')
train_loader = DataLoader(trainset, config.BATCH_SIZE, num_workers=config.WORKERS, shuffle=False, pin_memory=True)
X.head()
from joblib import Parallel, delayed
def temp(i,j):
    np.save(f'class_embeddings/{config.MODEL_NAME}_FOLD{fold}_{j}.npy', i)
if config.MODE in [4]:
    for config.MODEL_NAME, config.BATCH_SIZE in zip(config.MODEL_NAMES, config.BATCH_SIZES):
        for fold in config.FOLD:
            counter = 0 
            model = globals()[config.MODEL_NAME](pretrained=False, num_classes=config.NUM_CLASSES)
            model.load_state_dict(torch.load(f"models/{config.MODEL_NAME}_{config.FILE_PREFIX}_FOLD{fold}.pth")["state_dict"])
            model.classifier = nn.Identity()
            model.to(device)
            model.eval()
            tk1 = tqdm_notebook(train_loader, total=len(train_loader))
            embeddings = []
            with torch.no_grad():
                for x_var, y_var in tk1:
                    embeddings.extend(model(x_var.to(device)).detach().cpu().numpy())   
            a = Parallel(n_jobs=128,prefer='threads')(delayed(temp)(i, j) for i,j in tqdm_notebook(zip(embeddings, X.image_id.values),total=len(embeddings)))
x_test = pd.read_csv('data/test_set_features.csv')
x_test["file_name"] = "data/test/test/"+x_test.image_id+".jpg"
x_test["wind_speed"] = 0
testset = DatasetWIND(x_test, x_test, transforms = valid_transform, phase = 'test')
test_loader = DataLoader(testset, config.BATCH_SIZE, num_workers=config.WORKERS, shuffle=False, pin_memory=True)
if config.MODE in [6]:
    for config.MODEL_NAME, config.BATCH_SIZE in zip(config.MODEL_NAMES, config.BATCH_SIZES):
        for fold in config.FOLD:
            counter = 0 
            model = globals()[config.MODEL_NAME](pretrained=False, num_classes=config.NUM_CLASSES)
            model.load_state_dict(torch.load(f"models/{config.MODEL_NAME}_{config.FILE_PREFIX}_FOLD{fold}.pth")["state_dict"])
            model.classifier = nn.Identity()
            model.to(device)
            model.eval()
            tk1 = tqdm_notebook(test_loader, total=len(test_loader))
            embeddings = []
            with torch.no_grad():
                for x_var, y_var in tk1:
                    embeddings.extend(model(x_var.to(device)).detach().cpu().numpy())   
            Parallel(n_jobs=32,prefer='threads')(delayed(temp)(i, j) for i,j in tqdm_notebook(zip(embeddings, x_test.image_id.values),total=len(embeddings)))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12358799951415037, 'fraction_numerical': 0.010324304627717721, 'mean_word_length': 2.5261241970021415, 'pattern_counts': {'":': 2, '<': 31, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '192203', 'license': 'MIT', 'n_tokens_mistral': 5722, 'n_tokens_neox': 5471, 'n_words': 841, 'path': 'Phase:01-CNN.ipynb', 'repo_name': 'Cyber-Labs/Wind-Speeds-Prediction-of-Tropical-Storms', 'size': '16466'}