<jupyter_start><jupyter_text>Import<jupyter_code>import cv2
import h5py
import numpy as np
import glob
import csv
from datetime import datetime
from matplotlib import pyplot as plt
from collections import defaultdict
def cv_img_size(img):
    return tuple(img.shape[1::-1])

def plotCVColorImage(image, size = (800, 800)):
    converted_size = (size[0] / 100, size[1] / 100)
    plt.figure(figsize=converted_size, dpi = 100)
    plt.axis("off")
    plt.imshow(cv2.cvtColor(image, cv2.COLOR_BGR2RGB))
    plt.show()
    
def plotRGBColorImage(image, size = (800, 800)):
    converted_size = (size[0] / 100, size[1] / 100)
    plt.figure(figsize=converted_size, dpi = 100)
    plt.axis("off")
    plt.imshow(image)
    plt.show()<jupyter_output><empty_output><jupyter_text>Mask Overlay<jupyter_code>resize_files = ['1'] + [str(i) for i in range(26, 120)]
image_list = [str(i) for i in range(1, 120)]
image_prefix = '../data/images/raw/<NAME> 20170903/'
mask_prefix = '../data/images/adjusted/mask/'
image_suffix = '.JPG'
image_output_path = '../data/images/adjusted/overlaid/'
alpha = 0.6
color = (0, 255, 0)
for image_name in image_list:
    
    img_full = cv2.imread(image_prefix + image_name + image_suffix)
    img_full_overlay = img_full.copy()
    img_mask = np.load(mask_prefix + image_name + '_mask.npy')

    max_mask_val = np.max(img_mask)
    
    image_full_width, image_full_height = cv_img_size(img_full)

    for mask_val in range(1, max_mask_val + 1):
        
        select_mask = img_mask == mask_val
        binary_mask = select_mask.astype('uint8')
        points = cv2.findNonZero(binary_mask)

        rect = cv2.minAreaRect(points)
        box = cv2.boxPoints(rect)
        box = np.int0(box)
        if image_name in resize_files:
            box = box * 2
            
        cv2.drawContours(img_full_overlay,[box],0,color,2)
    
    cv2.addWeighted(img_full_overlay, alpha, img_full, 1 - alpha, 0, img_full)
    cv2.imwrite(image_output_path + image_name + image_suffix, img_full)
#     plotCVColorImage(img_full, (2560, 1920))<jupyter_output><empty_output><jupyter_text>Image Resolution Adjustment<jupyter_code>images_expr = '../data/images/adjusted/labelled/*.JPG'
standard_resolution = (1280, 960)

# i = 0
image_paths = glob.glob(images_path)
for image_path in image_paths:
    img_full = cv2.imread(image_path)
#     if (img_full.shape[1], img_full.shape[0]) != standard_resolution:
    img_crop = cv2.resize(img_full, standard_resolution, interpolation = cv2.INTER_LINEAR)
    print (image_path)
    cv2.imwrite(image_path, img_crop)
    plt.imshow(img_crop)
    plt.show()<jupyter_output><empty_output><jupyter_text>Data Managment  Auxiliary Functions *<jupyter_code>def write_dataset_to_file(output_file_path, files_info_dictionary, label_name_dictionary, labels_dictionary):
    
    with open(output_file_path, 'w', newline='') as csvfile:
        db_writer = csv.writer(csvfile, delimiter=DELIMITER, quotechar=QUOTE, quoting=csv.QUOTE_ALL)
        csvfile.write(DB_VERSION + '\n')

        csvfile.write(FILES_SEPARATER + '\n')

        for file_id in sorted(files_info_dictionary):
            file_info = files_info_dictionary[file_id]
            db_writer.writerow([file_id] + list(file_info))

        csvfile.write(LABEL_SEPARATER + '\n')
        for label_id in sorted(label_name_dictionary):
            label_info = label_name_dictionary[label_id]
            db_writer.writerow([label_id, label_info])

        csvfile.write(LABELS_SEPARTER + '\n')
        for file_id in sorted(labels_dictionary):
            file_points = labels_dictionary[file_id]
            for point in file_points:
                db_writer.writerow([file_id] + list(point))<jupyter_output><empty_output><jupyter_text>Retrieve Labels * Run this to retrieve all the labels for next stage<jupyter_code>image_label_file = '../data/labels/cleaned/MAC.csv'

resize_files = ['1.JPG'] + [str(i) + '.JPG' for i in range(26, 120)]
ignore_label_id = [102, 103]

DELIMITER = '|'
QUOTE = "'"

DB_VERSION = "DBVERSION:1.1"
FILES_SEPARATER = "/////FILES/////"
LABEL_SEPARATER = "/////LABEL/////"
LABELS_SEPARTER = "/////LABELS/////"
files_dictionary = {} # store file names only
files_info_dictionary = {} # store other info
files_inverse_dictionary = {}
label_name_dictionary = {}
labels_dictionary = defaultdict(list)

with open(image_label_file) as csvfile:
    label_reader = csv.reader(csvfile, delimiter = DELIMITER, quotechar = QUOTE)
    current_section = 'BEGIN'

    for row in label_reader:
        if row == [FILES_SEPARATER]:
            current_section = 'FILES'
        elif row == [LABEL_SEPARATER]:
            current_section = 'LABEL'
        elif row == [LABELS_SEPARTER]:
            current_section = 'LABELS'

        if len(row) > 1:
            if current_section == 'FILES':
                files_dictionary[int(row[0])] = row[1]
                files_info_dictionary[int(row[0])] = (row[1], row[2], row[3])
                files_inverse_dictionary[row[1]] = int(row[0])
            elif current_section == 'LABEL':
                if not int(row[0]) in ignore_label_id:
                    label_name_dictionary[int(row[0])] = row[1]
            elif current_section == 'LABELS':
                if not int(row[3]) in ignore_label_id:
                    if files_dictionary[int(row[0])] in resize_files:
    #                     print('resized:' + files_dictionary[int(row[0])])
                        labels_dictionary[int(row[0])].append( ( int(row[1]) // 2 , int(row[2]) // 2, int(row[3])) )
                    else:
                        labels_dictionary[int(row[0])].append( (int(row[1]) , int(row[2]), int(row[3])) )
    #print(labels_dictionary)
        
print(files_dictionary)
print(files_info_dictionary)
print(label_name_dictionary)
print(files_inverse_dictionary)
print(labels_dictionary)
# Don't run if not wanted to output
image_label_output_file = '../data/labels/cleaned/Adj_MAC.csv'
write_dataset_to_file(image_label_output_file, files_info_dictionary, label_name_dictionary, labels_dictionary)<jupyter_output><empty_output><jupyter_text>Data Removal<jupyter_code># 60: file: 1, 2, 37, 38, 39, 40, 41, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53
# 90: 1, 2
# 119: 1, 2

image_label_file_input = '../data/labels/cleaned/60.dat'
image_label_file_output = '../data/labels/cleaned/60C.dat'

file_id_remove = [1, 2, 37, 38, 39, 40, 41, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53]
label_id_remove = []
print (file_id_remove)

DELIMITER = '|'
QUOTE = "'"

DB_VERSION = 'DBVERSION:1.1'
FILES_SEPARATER = "/////FILES/////"
LABEL_SEPARATER = "/////LABEL/////"
LABELS_SEPARTER = "/////LABELS/////"
files_dictionary = {}
label_name_dictionary = {}
labels_dictionary = defaultdict(list)

with open(image_label_file_input) as csvfile:
    label_reader = csv.reader(csvfile, delimiter = DELIMITER, quotechar = QUOTE)
    current_section = 'BEGIN'

    for row in label_reader:
        
        if row == [FILES_SEPARATER]:
            current_section = 'FILES'
        elif row == [LABEL_SEPARATER]:
            current_section = 'LABEL'
        elif row == [LABELS_SEPARTER]:
            current_section = 'LABELS'

        if len(row) > 1:
            if current_section == 'FILES':
                files_dictionary[int(row[0])] = (row[1], row[2], row[3])
            elif current_section == 'LABEL':
                label_name_dictionary[int(row[0])] = row[1]
            elif current_section == 'LABELS':
                labels_dictionary[int(row[0])].append( ( int(row[1]) , int(row[2]), int(row[3]) ) )
# remove data accordingly
for file_id in file_id_remove:
    if file_id in files_dictionary:
        del files_dictionary[file_id]
    if file_id in labels_dictionary:
        del labels_dictionary[file_id]

for label_id in label_id_remove:
    if label_id in label_name_dictionary:
        del label_name_dictionary[label_id]
    
    new_labels_dictionary = defaultdict(list) 
    for file_id, label_list in labels_dictionary.items():
        label_list = [label_point for label_point in label_list if (label_point[2] != label_id)]
        new_labels_dictionary[file_id] = label_list
    labels_dictionary = new_labels_dictionary
    
write_dataset_to_file(image_label_file_output, files_dictionary, label_name_dictionary, labels_dictionary)<jupyter_output><empty_output><jupyter_text>Label Merge *<jupyter_code>label_map_dict = {0: 0, 1: 1, 2: 2, 3: 3, 4: 4, 5: 4, 6: 4, 7: 4, 8: 4}
label_name_dict = {0: "Null", 1: "Lactobacillus", 2: "Gardnerella", 3: "Curved Rods", 4: "Noise"}
noise_label = 4
print("Previous Label:")
print(label_name_dictionary)
label_name_dictionary = {}
for label_id, label_name in label_name_dict.items():
    label_name_dictionary[label_id] = label_name
print("After merging")
print(label_name_dictionary)

print("Labels Dictionary:")
print(labels_dictionary)
labels_dictionary_new = {}
for file_id, label_list in labels_dictionary.items():
    label_list_new = [(x, y, label_map_dict[label]) for x, y, label in label_list]
    labels_dictionary_new[file_id] = label_list_new
labels_dictionary = labels_dictionary_new
print("After Merging")
print(labels_dictionary)<jupyter_output>Previous Label:
{1: 'Lactobacillus    ', 2: 'Gardnerella      ', 3: 'Curved Rod       ', 4: 'Unknown A        ', 5: 'Coccus           ', 6: 'Yeast            ', 7: 'Noise - Edge     ', 8: 'Noise - Decompose'}
After merging
{0: 'Null', 1: 'Lactobacillus', 2: 'Gardnerella', 3: 'Curved Rods', 4: 'Noise'}
Labels Dictionary:
defaultdict(<class 'list'>, {1: [(17, 686, 2), (64, 389, 2), (70, 768, 2), (87, 78, 7), (98, 627, 2), (106, 808, 2), (157, 462, 2), (162, 955, 2), (169, 770, 2), (197, 99, 2), (210, 711, 2), (222, 79, 2), (228, 929, 2), (288, 731, 2), (302, 608, 2), (306, 863, 2), (313, 259, 2), (324, 764, 2), (325, 777, 2), (330, 934, 2), (355, 519, 2), (383, 255, 2), (392, 249, 2), (412, 175, 2), (443, 524, 2), (463, 487, 2), (478, 874, 2), (514, 587, 2), (535, 889, 2), (552, 775, 2), (566, 665, 2), (573, 946, 2), (581, 573, 2), (583, 830, 2), (589, 489, 2), (597, 851, 2), (605, 532, 2), (606, 76, 2), (621, 435, 2), (656, 294, 2), (689, 51, 2), (700, 271, 2), (705, 632, 2), (706, 81, [...]<jupyter_text>Raw Data File Merge<jupyter_code>image_label_directory =  '../data/labels/cleaned/'
image_label_files_list = ['10C.dat', '30C.dat', '60C.dat', '90C.dat', '119C.dat']
image_label_file_output = 'merge.dat'

DELIMITER = '|'
QUOTE = "'"

DB_VERSION = 'DBVERSION:1.1'
FILES_SEPARATER = "/////FILES/////"
LABEL_SEPARATER = "/////LABEL/////"
LABELS_SEPARTER = "/////LABELS/////"
files_dictionary = {}
files_translation_dictionary = {}
label_name_dictionary = {}
labels_dictionary = defaultdict(list)


next_file_index = 1
for image_label_file in image_label_files_list:
    with open(image_label_directory + image_label_file) as csvfile:
        label_reader = csv.reader(csvfile, delimiter = DELIMITER, quotechar = QUOTE)
        current_section = 'BEGIN'

        for row in label_reader:

            if row == [FILES_SEPARATER]:
                current_section = 'FILES'
            elif row == [LABEL_SEPARATER]:
                current_section = 'LABEL'
            elif row == [LABELS_SEPARTER]:
                current_section = 'LABELS'

            if len(row) > 1:
                if current_section == 'FILES':
                    files_translation_dictionary[int(row[0])] = next_file_index
                    files_dictionary[next_file_index] = (row[1], row[2], row[3])
                    next_file_index += 1
                elif current_section == 'LABEL':
                    label_name_dictionary[int(row[0])] = row[1]
                elif current_section == 'LABELS':
                    labels_dictionary[files_translation_dictionary[int(row[0])]].append( (int(row[1]) , int(row[2]), int(row[3])) )

write_dataset_to_file(image_label_directory + image_label_file_output, files_dictionary, label_name_dictionary, labels_dictionary)<jupyter_output><empty_output><jupyter_text>Merge Images and Image Masks as Dataset<jupyter_code>data_file_directory = "../data/dataset/"
data_file_name = "mytestfileNewFullMergeA.h5"<jupyter_output><empty_output><jupyter_text>SetupOnly run this once to create the file<jupyter_code>f = h5py.File(data_file_directory + data_file_name, "a")

f.attrs['file_format_version'] = "1.0"

t = datetime.now()
time_str = t.strftime("%Y%m%d_%H%M%S")
f.attrs['creation_time'] = time_str

variable_length_nparray_type = h5py.special_dtype(vlen = np.dtype('uint8'))

index_name_tuple_type = np.dtype([
    ('label_id', np.uint32),
    ('label_description', 'S256')
])

name_count_type = np.dtype([
    ('file_name', 'S256'),
    ('region_count', np.uint32)
])


shape_location_name_tuple_type = np.dtype([
    ('shape', [('axis1', np.uint8), ('axis2', np.uint8), ('axis3', np.uint8)]),
    ('location', [('y', np.uint16), ('x', np.uint16)]),
    ('image_name', 'S20')
])


f.create_dataset("label_description_set", (0,), maxshape=(None,), dtype = index_name_tuple_type, compression="gzip", compression_opts = 4, chunks=True)


f.create_dataset("training_image_set", (0,), maxshape=(None,), dtype = variable_length_nparray_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("training_mask_set", (0,), maxshape=(None,), dtype = variable_length_nparray_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("training_label_set", (0,), maxshape=(None,), dtype = np.dtype('uint8'), compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("training_metadata_set", (0,), maxshape=(None,), dtype = shape_location_name_tuple_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("training_file_info_set", (0,), maxshape=(None,), dtype = name_count_type, compression="gzip", compression_opts = 4, chunks=True)

f.create_dataset("dev_image_set", (0,), maxshape=(None,), dtype = variable_length_nparray_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("dev_mask_set", (0,), maxshape=(None,), dtype = variable_length_nparray_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("dev_label_set", (0,), maxshape=(None,), dtype = np.dtype('uint8'), compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("dev_metadata_set", (0,), maxshape=(None,), dtype = shape_location_name_tuple_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("dev_file_info_set", (0,), maxshape=(None,), dtype = name_count_type, compression="gzip", compression_opts = 4, chunks=True)


f.create_dataset("testing_image_set", (0,), maxshape=(None,), dtype = variable_length_nparray_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("testing_mask_set", (0,), maxshape=(None,), dtype = variable_length_nparray_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("testing_label_set", (0,), maxshape=(None,), dtype = np.dtype('uint8'), compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("testing_metadata_set", (0,), maxshape=(None,), dtype = shape_location_name_tuple_type, compression="gzip", compression_opts = 4, chunks=True)
f.create_dataset("testing_file_info_set", (0,), maxshape=(None,), dtype = name_count_type, compression="gzip", compression_opts = 4, chunks=True)


f.close()<jupyter_output><empty_output><jupyter_text>Full Data Merge<jupyter_code>f = h5py.File(data_file_directory + data_file_name, "a")
# 0: Full, all regions of interests, 1: Partial, only regions with labels
dataset_mode = 0

dev_list = [7, 12, 19, 24, 39, 47, 68, 76, 86, 100, 115]
testing_list = [8, 20, 49, 67, 77, 85, 91, 117, 119]
training_list = [i for i in range(1, 120) if i not in dev_list and i not in testing_list]
print(training_list)
print(dev_list)
print(testing_list)

image_prefix = '../data/images/adjusted/labelled/'
mask_prefix = '../data/images/adjusted/mask/'
image_suffix = '.JPG'
verbose = 1

noise_label = 4
noise_label_suppression = True

label_extension_steps = [0, 5, 10]
bounding_box_adjust_enabled = True
bounding_box_adjust_factor = 1.0
bounding_box_adjust_constant = 15

bounding_box_dimension_cutoff_enabled = True
bounding_box_dimension_max = 150
bounding_box_dimension_min = 10

labels_dictionary = labels_dictionary
files_inverse_dictionary = files_inverse_dictionary
label_name_dictionary = label_name_dictionary
f.attrs['dataset_mode'] = dataset_mode
f.attrs['label_merged'] = True

label_description_list = []
for key, value in label_name_dictionary.items():
    label_description_list.append((key, value))

label_description_set = f["label_description_set"]
label_description_set.resize(label_description_set.shape[0] + len(label_description_list), axis=0)
label_description_set[-len(label_description_list):] = label_description_list

print(label_description_set)
label_count_dictionary = {}
for dataset_stage in range(3):
    
    if dataset_stage == 0:
        data_purpose = 'training'
        image_list = training_list
    elif dataset_stage == 1:
        data_purpose = 'dev'
        image_list = dev_list
    elif dataset_stage == 2:
        data_purpose = 'testing'
        image_list = testing_list
        
    print("Dataset: " + data_purpose)
    
    image_set = f[data_purpose + "_image_set"]
    mask_set = f[data_purpose + "_mask_set"]
    label_set = f[data_purpose + "_label_set"]
    metadata_set = f[data_purpose + "_metadata_set"]
    file_set = f[data_purpose + "_file_info_set"]

    for image_index, image_name in enumerate(image_list):
        image_name = str(image_name)
        img_full = plt.imread(image_prefix + image_name + image_suffix)
        img_mask = np.load(mask_prefix + image_name + '_mask.npy')
        label_point_array = np.zeros(img_full.shape, dtype = np.dtype('uint8'))
        max_mask_val = np.max(img_mask)

        image_full_width, image_full_height = cv_img_size(img_full)

        for x_coor, y_coor, label in labels_dictionary[files_inverse_dictionary[image_name + image_suffix]]:
            if x_coor >= 0 and x_coor < image_full_width and y_coor >= 0 and y_coor < image_full_height:
                label_point_array[y_coor][x_coor] = label

        crop_images = []
        crop_masks = []
        labels = []
        meta_data_list = []

        count = 0
        label_count_dictionary[image_name] = defaultdict(int)

        for mask_val in range(1, max_mask_val + 1):
            select_mask = img_mask == mask_val
            binary_mask = select_mask.astype('uint8')
            points = cv2.findNonZero(binary_mask)

            x, y, w, h = cv2.boundingRect(points)

            if bounding_box_dimension_cutoff_enabled:
                if w > bounding_box_dimension_max or h > bounding_box_dimension_max:
                    continue
                if w < bounding_box_dimension_min or h < bounding_box_dimension_min:
                    continue

            if bounding_box_adjust_enabled:
                x_ad = x - int( (w / 2) * (bounding_box_adjust_factor - 1) + bounding_box_adjust_constant )
                y_ad = y - int( (h / 2) * (bounding_box_adjust_factor - 1) + bounding_box_adjust_constant )
                w_ad = int(w * bounding_box_adjust_factor + bounding_box_adjust_constant * 2)
                h_ad = int(h * bounding_box_adjust_factor + bounding_box_adjust_constant * 2)
            else:
                x_ad = x
                y_ad = y
                w_ad = w
                h_ad = h

            crop_image = img_full[max(y_ad, 0) : min(y_ad + h_ad, image_full_height), max(x_ad, 0) : min(x_ad + w_ad, image_full_width)]
            crop_mask = binary_mask[max(y_ad, 0) : min(y_ad + h_ad, image_full_height), max(x_ad, 0) : min(x_ad + w_ad, image_full_width)]
            meta_data = (crop_image.shape, (max(y_ad, 0), max(x_ad, 0)), image_name)

            this_label = 0
            for step in label_extension_steps:
                array_slice = label_point_array[max(y - step, 0) : min(y + h + step, image_full_height), max(x - step, 0) : min(x + w + step, image_full_width)]
                if noise_label_suppression:
                    noise_suprpessed_slice = np.where(array_slice == noise_label, np.full(array_slice.shape, -1), array_slice)
                    label_max = np.max(noise_suprpessed_slice)
                    if label_max != 0:
                        this_label = label_max
                        break
                    label_min = np.min(noise_suprpessed_slice)
                    if label_min == -1:
                        this_label = noise_label
                        break
                else:
                    this_label = np.max(array_slice)
                    if this_label != 0:
                        break

            if this_label != 0 or dataset_mode == 0:
                crop_images.append( crop_image.flatten() )
                crop_masks.append( np.packbits(crop_mask.flatten()) )
                labels.append( this_label )
                meta_data_list.append( meta_data )
                
                label_count_dictionary[image_name][this_label] += 1
                count += 1

        image_set.resize(image_set.shape[0] + count, axis=0)
        mask_set.resize(mask_set.shape[0] + count, axis=0)
        label_set.resize(label_set.shape[0] + count, axis=0)
        metadata_set.resize(metadata_set.shape[0] + count, axis=0)
        file_set.resize(file_set.shape[0] + 1, axis=0)
        
        image_set[-count:] = crop_images
        mask_set[-count:] = crop_masks
        label_set[-count:] = labels
        metadata_set[-count:] = meta_data_list
        file_set[-1] = (image_name, count)
            
        if verbose >= 1:
            print('image:' + image_name, count)
            if verbose == 1:
                print(label_set[-count:])
            print(image_set, mask_set, label_set, metadata_set, file_set)
    print("-------------------------------------------------")

print("All data loading completed")

for dataset_stage in range(3):
    
    if dataset_stage == 0:
        data_purpose = 'training'
    elif dataset_stage == 1:
        data_purpose = 'dev'
    elif dataset_stage == 2:
        data_purpose = 'testing'
        
    print("Dataset: " + data_purpose)
    
    image_set = f[data_purpose + "_image_set"]
    mask_set = f[data_purpose + "_mask_set"]
    label_set = f[data_purpose + "_label_set"]
    metadata_set = f[data_purpose + "_metadata_set"]
    file_set = f[data_purpose + "_file_info_set"]
    print(image_set, mask_set, label_set, metadata_set, file_set)

f.close()<jupyter_output><empty_output><jupyter_text>Dataset Inspection<jupyter_code># label statistics

for dataset_stage in range(3):
    temp_count = defaultdict(int)
    
    if dataset_stage == 0:
        data_purpose = 'training'
        image_list = training_list
    elif dataset_stage == 1:
        data_purpose = 'dev'
        image_list = dev_list
    elif dataset_stage == 2:
        data_purpose = 'testing'
        image_list = testing_list


    for image_name in image_list:
        for k,v in label_count_dictionary[str(image_name)].items():
            temp_count[k] += v

    print(data_purpose)
    print(sorted(temp_count.items()))
    print('')
data_purpose = "dev"

file_set = f[data_purpose + "_file_info_set"]
image_set = f[data_purpose + "_image_set"]
mask_set = f[data_purpose + "_mask_set"]
label_set = f[data_purpose + "_label_set"]
metadata_set = f[data_purpose + "_metadata_set"]
file_set = f[data_purpose + "_file_info_set"]
file_set[1]
i = 100
s = metadata_set[i][0]
print(metadata_set[i])
print(label_set[i])
im = image_set[i]
im = np.resize(im, tuple(s))

m = mask_set[i]
m = np.resize(np.unpackbits(m), (s[0], s[1], 1))
# print(m)
plotRGBColorImage(im, (100, 100))<jupyter_output>((45, 40, 3), (900, 64), b'39')
3
{'fraction_non_alphanumeric': 0.13335295849278775, 'fraction_numerical': 0.03427393919004163, 'mean_word_length': 3.4382232176185146, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '66025', 'license': 'Apache-2.0', 'n_tokens_mistral': 8939, 'n_tokens_neox': 8070, 'n_words': 1663, 'path': 'segmentation/Data Management.ipynb', 'repo_name': 'michaeltangc/BacteriaRecognition', 'size': '23785'}