<jupyter_start><jupyter_text>Image loading and generation notebook  Notebook setup<jupyter_code># noqa
import os
COLAB = 'DATALAB_DEBUG' in os.environ

if COLAB:
    !apt-get update
    !apt-get install git
    !git clone https://gist.github.com/oskopek/e27ca34cb2b813cae614520e8374e741 bstrap
    import bstrap.bootstrap as bootstrap
else:
    wd = %%pwd
    if wd.endswith('notebooks'):
        print('Current directory:', wd)
        %cd ..
        %pwd
    import resources.our_colab_utils.bootstrap as bootstrap

bootstrap.bootstrap(branch='master', packages='imgaug dotmap==1.2.20 keras==2.1.4 pydicom==1.0.2 Pillow==5.0.0')

if COLAB:
    !rm -rf bstrap
else:
    !pip install imgaug opencv-python<jupyter_output><empty_output><jupyter_text>Actual notebook<jupyter_code># noqa
import csv
import os
from dotmap import DotMap

import matplotlib
import matplotlib.pyplot as plt
import numpy as np
from PIL import Image
import pydicom
import skimage.transform
import tensorflow as tf

import resources.data.loader as loader
import resources.data.transformer as transformer
import resources.image_utils as imutils
import resources.synthetic_data as synth_data

%load_ext autoreload
%autoreload 2
%matplotlib inline
matplotlib.rcParams['figure.figsize'] = (10, 10)
plt.rcParams['image.cmap'] = 'gray' # 'viridis', 'gray'<jupyter_output><empty_output><jupyter_text>Data setup<jupyter_code>breast_prefix = os.path.abspath('/scratch_net/biwidl100/oskopek')
loader.init(breast_prefix)
size = (64, 50)
transformed_folder = os.path.join(breast_prefix, "transformed")
transformed_out = os.path.join(transformed_folder, "small_all_{}x{}".format(size[0], size[1]))
os.makedirs(transformed_out, exist_ok=True)
transformed_final = os.path.join(transformed_folder, "small_all_{}x{}_final".format(size[0], size[1]))
os.makedirs(transformed_final, exist_ok=True)
out_healthy = os.path.join(transformed_final, 'healthy.tfrecord')
out_cancer = os.path.join(transformed_final, 'cancer.tfrecord')<jupyter_output><empty_output><jupyter_text>Convert datasets to trainable CycleGan Images * Add labels to BCDR images* Method to filter out CC view and resize and split according to label bcdr* Add labels to inBreast images* Method to filter out CC view and resize and split according to label inBreast* Merge them and copy the images to 2 folders based on label<jupyter_code>healthy, cancer = loader.load_all_datasets()
filter_n = 2
if filter_n is not None:
    healthy, cancer = healthy[:filter_n], cancer[:filter_n]

transformer.transform(healthy, cancer, transformed_out, threads=1, size=size, augment_epochs=2)<jupyter_output><empty_output><jupyter_text>Merge shards<jupyter_code>transformer.merge_shards_in_folder(transformed_out, transformed_final)<jupyter_output><empty_output><jupyter_text>Show images from tfrecords<jupyter_code>outf = '/scratch_net/biwidl100/oskopek/transformed/small_all_64x50/label_1.augment_False.r_0.t_0.tfrecord'
import notebooks.inspect2 as inspect2
inspect2.show_images(outf)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10135806558463067, 'fraction_numerical': 0.022192779065915866, 'mean_word_length': 6.607052896725441, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '620917', 'license': 'Apache-2.0', 'n_tokens_mistral': 1102, 'n_tokens_neox': 1036, 'n_words': 232, 'path': 'notebooks/images-conversion.ipynb', 'repo_name': 'BreastGAN/augmentation', 'size': '3019'}