<jupyter_start><jupyter_text><jupyter_code>!pip install fastai -q --upgrade
!pip install --upgrade albumentations
!pip install gdown
!apt-get install zip unzip
!unzip ./owl.zip
!rm ./owl.zip
gdown.download('https://drive.google.com/u/1/uc?id=1PsX8ULvWg-peKtUduyTqkyxfxXoom13S', './owl.zip', quiet=False)<jupyter_output><empty_output><jupyter_text>Specs<jupyter_code>!nvidia-smi
import warnings
warnings.filterwarnings('ignore')<jupyter_output><empty_output><jupyter_text>Import dependencies<jupyter_code>from fastai.vision.all import *
import gdown
import os
import pandas as pd<jupyter_output><empty_output><jupyter_text>Preprocessing<jupyter_code>seed = 1000

def seed_everything(seed = seed):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    torch.manual_seed(seed)
    torch.cuda.manual_seed_all(seed)
    torch.backends.cudnn.deterministic = True
    torch.backends.cudnn.benchmark = False

seed_everything()
path = Path('./Dataset_OWL_COD10K_v3/')
train_images = path/'Train/Image'
train_labels = path/'Train/GT'
test_name = "Test"
test_images = path/test_name/'Image'
test_labels = path/test_name/'GT'
def get_y_fn(x): 
  return Path(str(x).replace("Image","GT").replace(".jpg",".png"))
def ParentSplitter(x):
    return Path(x).parent.name==test_name
codes = ['background','owl']
name2id = {v:k for k,v in enumerate(codes)}
print(name2id)
void_code = name2id['owl']
def foreground_acc(input, target):
    target = cast(target.squeeze(1), TensorBase)
    mask = target != void_code
    return (input.argmax(dim=1)[mask]==target[mask]).float().mean()
class SegmentationAlbumentationsTransform(ItemTransform):
    split_idx = 0
    def __init__(self, aug): 
        self.aug = aug
    def encodes(self, x):
        img,mask = x
        aug = self.aug(image=np.array(img), mask=np.array(mask))
        return PILImage.create(aug["image"]), PILMask.create(aug["mask"])
class TargetMaskConvertTransform(ItemTransform):
    def __init__(self): 
        pass
    def encodes(self, x):
        img,mask = x
        mask = np.array(mask)
        mask[mask==255]=1
        mask = PILMask.create(mask)
        return img, mask<jupyter_output><empty_output><jupyter_text>Compute Method  Constant<jupyter_code>size = 224
bs = 6
splitter = RandomSplitter(valid_pct=0.2, seed=seed)
item_tfms = [Resize((size,size)), TargetMaskConvertTransform()]
batch_tfms = []
arch = resnet34
pretrained=True
metrics = [foreground_acc, Dice(), JaccardCoeff()]
act_cls = nn.ReLU
self_attention = False
lr = 1e-3
wd = None
epochs = 200
monitor_training = "valid_loss"
comp_training = np.less

monitor_evaluating = "dice"
comp_evaluating = np.greater

patience = 50
# Import csv or make new dataframe
if os.path.isfile("result.csv"):
  result_df = pd.read_csv("result.csv", header=[0,1], index_col=[0,1])
else:
  col_names = [('method_type',''), ('method_name',''), ('Training','valid_loss'),("Training",'foreground_acc'),("Training",'dice'),('Training','jaccard_coeff'),('Testing','valid_loss'),("Testing",'foreground_acc'),("Testing",'dice'),('Testing','jaccard_coeff')]
  col_list = pd.MultiIndex.from_tuples(col_names)
  result_df = pd.DataFrame(columns = col_list).set_index([('method_type'), ('method_name')])

result_df = result_df.sort_index(0)<jupyter_output><empty_output><jupyter_text>Compute Function<jupyter_code>def compute(method_name, method_type, size = size, splitter=splitter, item_tfms = item_tfms, batch_tfms = batch_tfms, bs = bs, arch = arch, pretrained = pretrained, metrics = metrics,
            act_cls = act_cls, self_attention = self_attention, lr = lr, wd = wd, epochs = epochs, patience = patience):
  
  if(len(batch_tfms)):
    transformPipeline = Compose(batch_tfms,p=1)
    transformPipeline = SegmentationAlbumentationsTransform(transformPipeline)
    item_tfms.append(transformPipeline)

  callbacksFitBeforeUnfreeze = [
    ShowGraphCallback(),
    EarlyStoppingCallback(monitor = monitor_training, comp = comp_training, patience = patience),
    SaveModelCallback(monitor = monitor_training, comp = comp_training, every_epoch = False, fname = method_name)  
  ]
  
  dblock = DataBlock(blocks = (ImageBlock, MaskBlock(codes)),
                  get_items = get_image_files,
                  splitter = splitter,
                  get_y = get_y_fn,
                  item_tfms = item_tfms,
                  batch_tfms = [Normalize.from_stats(*imagenet_stats)]
                  )
  
  dls = dblock.dataloaders(train_images, bs=bs, num_workers=0)

  seed_everything()

  learn = unet_learner(dls, arch, pretrained = pretrained, metrics = metrics, wd=wd,
                       act_cls = act_cls, self_attention = self_attention)

  if(lr):
    learn.fit_one_cycle(epochs, lr, cbs=callbacksFitBeforeUnfreeze)
  else:
    lr = learn.lr_find()
    print("~~~~~~~~~~~~~~~~~~~~~~~~~Learning Rate~~~~~~~~~~~~~~~~~~~~~~~~~")
    print(lr[0].item())
    learn.fit_one_cycle(epochs, lr[0].item(), cbs=callbacksFitBeforeUnfreeze)

  # learn.load(method_name)
  # learn.unfreeze()
  # lr = learn.lr_find()
  # learn.fit_one_cycle(25, slice(lr[0].item()),cbs=callbacksFitBeforeUnfreeze)

  print("~~~~~~~~~~~~~~~~~~~~~~~~~Training Result~~~~~~~~~~~~~~~~~~~~~~~~~")
  train_validate = learn.validate()
  print(train_validate)

  test_img = get_image_files(test_images)
  dl = learn.dls.test_dl(test_img, with_labels=True)
  print("~~~~~~~~~~~~~~~~~~~~~~~~~Testing Result~~~~~~~~~~~~~~~~~~~~~~~~~~")
  test_validate = learn.validate(dl=dl)
  print(test_validate)
  
  result_df.loc[(method_type, method_name), :] = train_validate + test_validate
  #result_df.to_csv('result.csv')
  result_df.to_csv(f'result.csv')

  # Delete pth file
  !rm -rf ./models

  return learn<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15839860748476936, 'fraction_numerical': 0.008529155787641428, 'mean_word_length': 4.942088934850052, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '116610', 'license': 'Apache-2.0', 'n_tokens_mistral': 2062, 'n_tokens_neox': 1985, 'n_words': 327, 'path': 'PaperSpace_Template.ipynb', 'repo_name': 'Manu1ND/AR_android', 'size': '5745'}