<jupyter_start><jupyter_text>Util code snippetsThis notebook contains only isolated code snippets to do certain data manipulations and experiments that do not necessarily contribute to the main idea. This is only for reference on how secondary aspects might be achieved. Going from RGB to Black or White contour images<jupyter_code>import matplotlib.pyplot as plt
import numpy as np
from PIL import Image
from matplotlib import image as mpl_image

width, height = 200, 200
# width, height = 64, 64
original = Image.open('images/homer.png')
original.thumbnail((width, height))
original<jupyter_output><empty_output><jupyter_text>Grayscale<jupyter_code>gray = original.convert('L')
gray<jupyter_output><empty_output><jupyter_text>Black & White - PIL point style<jupyter_code>pil_bw = gray.point(lambda x: 0 if x<120 else 255, '1')
pil_bw<jupyter_output><empty_output><jupyter_text>Black & White - Pylab contour style<jupyter_code>from pylab import contour

pylab_bw = contour(gray, colors='black', origin='image')
pylab_bw<jupyter_output><empty_output><jupyter_text>Black & White - OpenCV style<jupyter_code>import cv2

original_cv2 = cv2.imread('images/homer.png')
# Grayscale
gray = cv2.cvtColor(original_cv2, cv2.COLOR_BGR2GRAY)
  
_, binary = cv2.threshold(gray, 120, 255, cv2.THRESH_BINARY)
plt.imshow(binary, cmap='gray')
plt.show()
pil_bw.save('images/homer_bw_pil.png')<jupyter_output><empty_output><jupyter_text>Create a corrupted image with gaussian noise<jupyter_code>bw = np.array(pil_bw)
sigma = 0.6
width, height = bw.shape
bw_corrupted = bw + sigma * np.random.randn(width, height)
bw_corrupted[bw_corrupted > 0] = 255
bw_corrupted[bw_corrupted < 0] = 0
out = Image.fromarray(bw_corrupted).convert('L')
out.save('images/homer_bw_noise.png')
out<jupyter_output><empty_output><jupyter_text>Create a corrupted image with a mask<jupyter_code>bw = np.array(pil_bw)
sigma = 0.6
width, height = bw.shape
bw_corrupted = bw.copy()
bw_corrupted[np.random.randint(0, height / 2):np.random.randint(width/2, height),
    np.random.randint(0, height/2):np.random.randint(width/2, height)] = 0
# bw_corrupted[width//2 + 10:width, 0: height] = 0
out = Image.fromarray(bw_corrupted).convert('L')
out.save('images/homer_bw_rand_mask.png')
out
bw_corrupted = bw.copy()
bw_corrupted[width//2 + 10:width, 0: height] = 0
out = Image.fromarray(bw_corrupted).convert('L')
out.save('images/homer_bw_fixed_mask.png')
out
gray_corrupted = gray.copy()
gray_corrupted = np.array(gray_corrupted)
gray_corrupted[width//2 + 10:width, 0: height] = 0
out = Image.fromarray(gray_corrupted).convert('L')
out.save('images/homer_gray_fixed_mask.png')
out<jupyter_output><empty_output><jupyter_text>Generate bw dataset<jupyter_code>!mkdir -p subset_bw
from PIL import Image
import os

originals = os.listdir('subset/')
originals = [(img, os.path.join('subset', img)) for img in originals]
for file, path in originals:
    bw = Image.open(path).convert('L').point(lambda x: 0 if x<100 else 255, '1')
    bw.save(f'subset_bw/{file}')<jupyter_output><empty_output><jupyter_text>Generate bw noisy dataset<jupyter_code>!mkdir -p subset_bw_noise
from PIL import Image
import numpy as np
import os

originals = os.listdir('subset/')
originals = [(img, os.path.join('subset', img)) for img in originals]
for file, path in originals:
    bw = Image.open(path).convert('L').point(lambda x: 0 if x<100 else 255, '1')
    bw = np.array(bw)
    sigma = 0.6
    width, height = bw.shape
    bw_corrupted = bw + sigma * np.random.randn(width, height)
    bw_corrupted[bw_corrupted > 0] = 255
    bw_corrupted[bw_corrupted < 0] = 0
    out = Image.fromarray(bw_corrupted).convert('L')
    out.save(f'subset_bw_noise/{file}')<jupyter_output><empty_output><jupyter_text>Generate bw masked dataset<jupyter_code>!mkdir -p subset_bw_masked
from PIL import Image
import numpy as np
import os

originals = os.listdir('subset/')
originals = [(img, os.path.join('subset', img)) for img in originals]
for file, path in originals:
    bw = Image.open(path).convert('L').point(lambda x: 0 if x<100 else 255, '1')
    bw = np.array(bw)
    sigma = 0.6
    width, height = bw.shape
    bw_corrupted = bw.copy()
    bw_corrupted[np.random.randint(0, height / 2):np.random.randint(width/2, height),
        np.random.randint(0, height/2):np.random.randint(width/2, height)] = 0
    out = Image.fromarray(bw_corrupted).convert('L')
    out.save(f'subset_bw_masked/{file}')
    out<jupyter_output><empty_output><jupyter_text>Generate gray scale dataset<jupyter_code>!mkdir -p subset_gray
from PIL import Image
import numpy as np
import os

originals = os.listdir('subset/')
originals = [(img, os.path.join('subset', img)) for img in originals]
for file, path in originals:
    gray = Image.open(path).convert('L')
    gray.save(f'subset_gray/{file}')<jupyter_output><empty_output><jupyter_text>Generate gray scale masked<jupyter_code>!mkdir -p subset_gray_masked
from PIL import Image
import numpy as np
import os

originals = os.listdir('subset/')
originals = [(img, os.path.join('subset', img)) for img in originals]
for file, path in originals:
    gray = Image.open(path).convert('L')
    gray = np.array(gray)
    sigma = 0.6
    width, height = gray.shape
    gray_corrupted = gray.copy()
    gray_corrupted[np.random.randint(0, height / 2):np.random.randint(width/2, height),
        np.random.randint(0, height/2):np.random.randint(width/2, height)] = 0
    out = Image.fromarray(gray_corrupted).convert('L')
    out.save(f'subset_gray_masked/{file}')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13275456682944475, 'fraction_numerical': 0.02079942123349611, 'mean_word_length': 5.52122641509434, 'pattern_counts': {'":': 0, '<': 55, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '363108', 'license': 'Apache-2.0', 'n_tokens_mistral': 2128, 'n_tokens_neox': 2043, 'n_words': 444, 'path': 'hopfield/0_utils.ipynb', 'repo_name': 'Ohtar10/dele-zoo', 'size': '5529'}