<jupyter_start><jupyter_text>1. Import Libraries<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import cv2


import matplotlib.image as mpimg
from pathlib import Path
from PIL import Image<jupyter_output><empty_output><jupyter_text>2. Define Path to Images and Labels<jupyter_code>p = Path('../data/dataset')
file_list = [x for x in p.glob('**/*') if x.is_file()]
print(len(file_list))
path_img = p/'train_x'
path_lbl = p/'train_y'<jupyter_output><empty_output><jupyter_text>Extract All Files<jupyter_code>x_file = [x for x in path_img.glob('**/*') if x.is_file()]
y_file = [y for y in path_lbl.glob('**/*') if y.is_file()]
len(x_file)<jupyter_output><empty_output><jupyter_text>3. Visually Compare Image to Label and Determine Label Quality<jupyter_code>filenum = 380
x_name = str(x_file[filenum]).split('\\')[-1]
y_name = str(y_file[filenum]).split('\\')[-1]

x = mpimg.imread(x_file[filenum])
y = mpimg.imread(y_file[filenum])
fig, ax = plt.subplots(1, 3, figsize=(18, 16))

ax[0].imshow(x)
ax[0].set_title('Image')
ax[0].axis('off')

ax[1].imshow(y)
ax[1].set_title('Label')
ax[1].axis('off')

ax[2].imshow(x)
ax[2].imshow(y, alpha=0.5)
ax[2].set_title('Overlay Image with Label')
ax[2].axis('off')<jupyter_output><empty_output><jupyter_text>Assign Which Folder to Put Image Label Pair Depending on Assessment<jupyter_code>new_folder = 'good'

# The good folder is for image/label pairs that don't need any more fixing
if new_folder == 'good':
    cv2.imwrite(f'../data/dataset/good/train_x/{x_name}', x)
    cv2.imwrite(f'../data/dataset/good/train_y/{y_name}', y)

# The maybe folder is for image/label pairs that only need a bit of touching up to be acceptable
elif new_folder == 'maybe':
    cv2.imwrite(f'../data/dataset/maybe/train_x/{x_name}', x)
    cv2.imwrite(f'../data/dataset/maybe/train_y/{y_name}', y)

# The bad folder is for image/label pairs that are unsalvageable
elif new_folder == 'bad':
    cv2.imwrite(f'../data/dataset/bad/train_x/{x_name}', x)
    cv2.imwrite(f'../data/dataset/bad/train_y/{y_name}', y)<jupyter_output><empty_output><jupyter_text>Keep track of which file was last assessed<jupyter_code>print(filenum)<jupyter_output>380
{'fraction_non_alphanumeric': 0.16368638239339753, 'fraction_numerical': 0.016506189821182942, 'mean_word_length': 5.713846153846154, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '14299', 'license': 'Apache-2.0', 'n_tokens_mistral': 872, 'n_tokens_neox': 865, 'n_words': 180, 'path': 'experiments/Training_Data_Quality.ipynb', 'repo_name': 'lwang94/sem_size_analysis', 'size': '2181'}