<jupyter_start><jupyter_text>Mount and Fetch Data<jupyter_code>from google.colab import drive
drive.mount('/content/gdrive',force_remount=True)
%reload_ext autoreload
%autoreload 2<jupyter_output>Go to this URL in a browser: https://accounts.google.com/o/oauth2/auth?client_id=947318989803-6bn6qk8qdgf4n4g3pfee6491hc0brc4i.apps.googleusercontent.com&redirect_uri=urn%3aietf%3awg%3aoauth%3a2.0%3aoob&response_type=code&scope=email%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdocs.test%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive%20https%3a%2f%2fwww.googleapis.com%2fauth%2fdrive.photos.readonly%20https%3a%2f%2fwww.googleapis.com%2fauth%2fpeopleapi.readonly

Enter your authorization code:
··········
Mounted at /content/gdrive<jupyter_text>Data Statistics Utils to display file structure, directory size and image dimensions<jupyter_code>import os
import cv2
import subprocess

def list_files(startpath, cap_dir=3, cap_file=3):
    dcount = 0
    for root, dirs, files in os.walk(startpath):
        dcount += 1
        level = root.replace(startpath, '').count(os.sep)
        indent = ' ' * 4 * (level)
        print('{}{}/'.format(indent, os.path.basename(root)))
        subindent = ' ' * 4 * (level + 1)
        count = 0
        for f in files:
            print('{}{}'.format(subindent, f))
            count += 1
            if count == cap_file:
                print('{}{}'.format(subindent, "..."))
                break
        if dcount == cap_dir:
             print("...")
             break

def du(path):
    return subprocess.check_output(['du','-sh', path]).split()[0].decode('utf-8')

def imsize(path):
    im = cv2.imread(path, cv2.IMREAD_UNCHANGED)
    return im.shape<jupyter_output><empty_output><jupyter_text>Utils to compute mean and std of data<jupyter_code>import torch
import torchvision
import torchvision.transforms as transforms
from tqdm import tqdm

def compute_mean_std(path, bs=128, is_gray=False):
  transforms_list = []
  if is_gray:
    transforms_list.append(transforms.Grayscale())  
  transforms_list.append(transforms.ToTensor())

  dataset = torchvision.datasets.ImageFolder(root=path,
                                             transform=transforms.Compose(transforms_list))
  loader = torch.utils.data.DataLoader(
    dataset,
    batch_size=bs,
    num_workers=4,
    shuffle=False,
    pin_memory= True
  )

  mean = 0.0
  for data, _ in tqdm(loader, desc="Computing mean"):
    batch_samples = data.size(0)
    # Flatten each channel
    data = data.view(batch_samples, data.size(1), -1)
    # Since the denominator (224*224) is constant, instead of doing sum first and
    # then taking mean, we can directly take mean and sum it.
    mean += data.mean(2).sum(0)

  mean /= len(loader.dataset)

  std = 0.0
  for data, _ in tqdm(loader, desc="Computing std"):
    batch_samples = data.size(0)
    # Flatten each channel
    data = data.view(batch_samples, data.size(1), -1)
    # Sum (x-mean)^2 per channel across all batches pixels
    std += ((data - mean.unsqueeze(1))**2).sum([0,2])

  std = torch.sqrt(std / (len(loader.dataset)*224*224))
  print("\nMean: ", mean)
  print("Std: ", std)<jupyter_output><empty_output><jupyter_text>Utils to display images in a grid<jupyter_code>import matplotlib.pyplot as plt
from matplotlib.image import imread
import numpy as np
import glob

def show_images(images, cols = 1, titles = None, figsize=(5,5)):
    assert((titles is None)or (len(images) == len(titles)))
    n_images = len(images)
    if titles is None: titles = ['Image (%d)' % i for i in range(1,n_images + 1)]
    fig = plt.figure(figsize=figsize)
    for n, (image, title) in enumerate(zip(images, titles)):
        a = fig.add_subplot(cols, np.ceil(n_images/float(cols)), n + 1)
        if image.ndim == 2:
            plt.gray()
        plt.axis("off")
        plt.imshow(image)
        plt.tight_layout()
        # a.set_title(title)
    # fig.set_size_inches(0.1*np.array(fig.get_size_inches()) * n_images)
    fig.tight_layout()
    plt.show()

def display_grid(base_path, n, ext=".jpg", sub_dirs=False, figsize=(5,5), off=0):
  input_files = []
  cols = int(np.sqrt(n))
  if sub_dirs:
    for bg_idx in range(off, n+off):
      path = f"{base_path}/bg_{bg_idx:02d}/*{ext}"
      input_files.append(glob.glob(path)[0])
  else:
    input_files = glob.glob(f"{base_path}/*{ext}")[:n]
  
  image_list = list(map(lambda im: imread(im), input_files))
  show_images(image_list, cols=cols, figsize=figsize)<jupyter_output><empty_output><jupyter_text>File Structure<jupyter_code>import zipfile
import os

def extract_files(zip_file_path, zip_file_list, output_folder):
    for fp in zip_file_list:
        path_to_zip_file = os.path.join(zip_file_path, fp)
        print("Extracting zip file: %s" % path_to_zip_file)
        with zipfile.ZipFile(path_to_zip_file, 'r') as zip_ref:
            zip_ref.extractall(output_folder)
        print("Extracted at: %s" % output_folder)
# Copy Background Images to colab local directory

!rm -rf /content/bgfgdataset/bg/
!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/bg/" "/content/bgfgdataset/"
du("/content/bgfgdataset/bg")

extract_files("/content/bgfgdataset/bg", ["bg.zip"], "/content/bgfgdataset/bg")

!rm /content/bgfgdataset/bg/bg.zip
# Copy Foreground Images to colab local directory

!rm -rf /content/bgfgdataset/fg/
!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/fg/" "/content/bgfgdataset/"
du("/content/bgfgdataset/fg")

extract_files("/content/bgfgdataset/fg", ["fg.zip"], "/content/bgfgdataset/fg")

!rm /content/bgfgdataset/fg/fg.zip
# Copy overlay Images to colab local directory

!rm -rf /content/bgfgdataset/bgfg_overlay/

!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/bgfg_overlay/" "/content/bgfgdataset/bgfg_overlay/"
du("/content/bgfgdataset/bgfg_overlay")

extract_files("/content/bgfgdataset/bgfg_overlay", ["maskrcnndataset_overlay.zip"], "/content/bgfgdataset/bgfg_overlay")

!rm /content/bgfgdataset/bgfg_overlay/maskrcnndataset_overlay.zip
# Copy overlay mask Images to colab local directory

!rm -rf /content/bgfgdataset/bgfg_mask/

!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/bgfg_mask/" "/content/bgfgdataset/"
du("/content/bgfgdataset/bgfg_mask")

extract_files("/content/bgfgdataset/bgfg_mask", ["maskrcnndataset_mask.zip"], "/content/bgfgdataset/bgfg_mask")

!rm /content/bgfgdataset/bgfg_mask/maskrcnndataset_mask.zip
# Copy foreground mask Images to colab local directory
!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/fg_mask/" "/content/bgfgdataset/fg_mask/"
du("/content/bgfgdataset/fg_mask")
# Copy depth Images to colab local directory

!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/bgfg_depth/" "/content/bgfgdataset/bgfg_depth/"
du("/content/bgfgdataset/bgfg_depth")
# Copy depth Images to colab local directory

!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/bgfg_depth3/" "/content/bgfgdataset/bgfg_depth/"
print("Done copying Depth 3")
!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/bgfg_depth4/" "/content/bgfgdataset/bgfg_depth/"
print("Done copying Depth 4")
!cp -r "/content/gdrive/My Drive/mask_rcnn_dataset/bgfg_depth_zip/" "/content/bgfgdataset/bgfg_depth/"
print("Done copying Depth 5 zip")

extract_files("/content/bgfgdataset/bgfg_depth", ["bgfg_depth5"], "/content/bgfgdataset/bgfg_depth")
print("Done Extracting Depth 5")

!rm /content/bgfgdataset/bgfg_depth/bgfg_depth5
path = "/content/bgfgdataset"
dirs = sorted(os.listdir(path))
print("File structure:")
print("dataset/")
for dir_name in dirs:
  print(f"{' ' * 4}{dir_name}/")
print("Sub-dirs count:", len(dirs))
print("Dir size:", du(path),"\n")<jupyter_output>File structure:
dataset/
    .ipynb_checkpoints/
    bg/
    bgfg_depth/
    bgfg_mask/
    bgfg_overlay/
    fg/
    fg_mask/
Sub-dirs count: 7
Dir size: 5.5G<jupyter_text>Background<jupyter_code>path = "/content/bgfgdataset/bg"
print("File structure:")
list_files(path)
files = os.listdir(path)
print("File count:", len(files))
print("Image dimensions:", imsize(path+"/"+files[0]))
print("Dir size:", du(path),"\n")
!mkdir /content/bg_wrap
!cp -r /content/bgfgdataset/bg /content/bg_wrap/
compute_mean_std("/content/bg_wrap", bs=128, is_gray=False)
!rm -rf /content/bg_wrap/
print()
display_grid(path, 9, ext=".jpg", sub_dirs=False, figsize=(5,5))<jupyter_output>File structure:
bg/
    bg_040.jpg
    bg_017.jpg
    bg_062.jpg
    ...
File count: 99
Image dimensions: (224, 224, 3)
Dir size: 1.5M<jupyter_text>Foreground<jupyter_code>path = "/content/bgfgdataset/fg"
print("File structure:")
list_files(path)
files = os.listdir(path)
print("File count:", len(files))
print("Image dimensions:", imsize(path+"/"+files[0]))
print("Dir size:", du(path))
print()
display_grid(path, 9, ext=".png", sub_dirs=False, figsize=(4,4))<jupyter_output>File structure:
fg/
    fg_092.png
    fg_017.png
    fg_068.png
    ...
File count: 99
Image dimensions: (140, 96, 4)
Dir size: 1.8M<jupyter_text>Foreground Mask<jupyter_code>path = "/content/bgfgdataset/fg_mask"
print("File structure:")
list_files(path)
files = os.listdir(path)
print("File count:", len(files))
print("Image dimensions:", imsize(path+"/"+files[0]))
print("Dir size:", du(path))
print()
display_grid(path, 9, ext=".png", sub_dirs=False, figsize=(4,4))<jupyter_output>File structure:
fg_mask/
    mask_fg_020.png
    mask_fg_081.png
    mask_fg_046.png
    ...
File count: 99
Image dimensions: (140, 65, 4)
Dir size: 848K<jupyter_text>Foreground Overlayed on Background<jupyter_code>path = "/content/bgfgdataset/bgfg_overlay/content/fg_bg_dataset"
print("File structure:")
list_files(path)
dirs = os.listdir(path)
print("Sub-dirs count:", len(dirs))
files = os.listdir(path+"/"+dirs[0])
print("Files per sub-dir count:", len(files))
print("Image dimensions:", imsize(path+"/"+dirs[0]+"/"+files[0]))
print("Dir size:", du(path),"\n")
compute_mean_std(path, bs=1024, is_gray=False)
print()
display_grid(path, 9, ext=".png", sub_dirs=True, figsize=(5,5), off=65)<jupyter_output>File structure:
fg_bg_dataset/
    bgfg_overlay/
        ol_bg025fg0622f_fg_062.png
        ol_bg014fg0619_fg_061.png
        ol_bg021fg05313f_fg_053.png
        ...
Sub-dirs count: 1
Files per sub-dir count: 392041
Image dimensions: (224, 224, 3)
Dir size: 5.3G<jupyter_text>Foreground Overlayed on Background Mask<jupyter_code>path = "/content/bgfgdataset/bgfg_mask/content/fg_bg_dataset"
print("File structure:")
list_files(path)
dirs = os.listdir(path)
print("Sub-dirs count:", len(dirs))
files = os.listdir(path+"/"+dirs[0])
print("Files per sub-dir count:", len(files))
print("Image dimensions:", imsize(path+"/"+dirs[0]+"/"+files[0]))
print("Dir size:", du(path),"\n")
compute_mean_std(path, bs=1024, is_gray=True)
print()
display_grid(path, 9, ext=".png", sub_dirs=True, figsize=(5,5), off=65)<jupyter_output>File structure:
fg_bg_dataset/
    bgfg_mask/
        mask_ol_bg082fg06116_fg_061.png
        mask_ol_bg082fg0934f_fg_093.png
        mask_ol_bg042fg08810f_fg_088.png
        ...
Sub-dirs count: 1
Files per sub-dir count: 392041
Image dimensions: (224, 224)
Dir size: 1.6G<jupyter_text>Foreground Overlayed on Background Depth Map<jupyter_code>path = "/content/bgfgdataset/fg_bg_depth"
print("File structure:")
list_files(path)
dirs = os.listdir(path)
print("Sub-dirs count:", len(dirs))
files = os.listdir(path+"/"+dirs[0])
print("Files per sub-dir count:", len(files))
print("Image dimensions:", imsize(path+"/"+dirs[0]+"/"+files[0]))
print("Dir size:", du(path),"\n")
compute_mean_std(path, bs=1024, is_gray=True)
print()
display_grid(path, 9, ext=".jpg", sub_dirs=True, figsize=(5,5), off=65)
import shutil
output_fname = "/content/dataset"
dir_name = "/content/dataset"
shutil.make_archive(output_fname, 'zip', dir_name)
!cp '/content/dataset.zip' '/content/gdrive/My Drive/mask_rcnn_dataset/'<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15170016111252438, 'fraction_numerical': 0.0287458661918087, 'mean_word_length': 4.703094777562863, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 38, 'https://': 1, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '141572', 'license': 'MIT', 'n_tokens_mistral': 4565, 'n_tokens_neox': 4328, 'n_words': 829, 'path': 'generate_data_stats.ipynb', 'repo_name': 'anuragal/DepthModel', 'size': '11793'}