<jupyter_start><jupyter_text>Cleaning the data

Each file in the folder contains varying degree of dimensions. We need to clean the data, resize the images in a manner that they end up having the same size or sort<jupyter_code># total photos
print(len(photos_list))
def glitchify(image, min_size = 500, fill_color = (255, 255, 255)):
  width, height = image.size
  width_ratio = min_size / width
  height_ratio = min_size / height
  min_ratio = min(width_ratio, height_ratio)
  width *= min_ratio
  height *= min_ratio
  width, height = round(width), round(height)
  resized_image = image.resize((width, height))
  new_image = Image.new('RGB', (min_size, min_size), fill_color)
  new_image.paste(resized_image, (int((min_size - width)/2), int((min_size - height)/2)))
  resized_image = new_image
  glitched_image = pixelsort(new_image)
  return resized_image, glitched_image

def generate_glitch():
  for file_name in photos_list:
    print(f'Glitching {file_name}.....')
    original = Image.open(os.path.join(photos_root, file_name))
    original, glitched = glitchify(original)
    print('Done!')
    yield (original, glitched)

# handle with care
def generate_glitches():
  return ([result in generate_glitch()])<jupyter_output><empty_output><jupyter_text>Let's have a look at one original image and it's glitched image<jupyter_code>generator = generate_glitch()
original, glitched = next(generator)
original, glitched = next(generator)
original
glitched<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10488126649076517, 'fraction_numerical': 0.009234828496042216, 'mean_word_length': 4.92578125, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '52110', 'license': 'MIT', 'n_tokens_mistral': 515, 'n_tokens_neox': 485, 'n_words': 133, 'path': 'Pixel_Sort_Model.ipynb', 'repo_name': 'skywalker212/glitch-bot', 'size': '1516'}