<jupyter_start><jupyter_text>5-8-19: 01-data_processing.ipynb is for initial data loading, image scaling/resizing, and vectorizing.<jupyter_code>import os
from PIL import Image

import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import os, sys
from pathlib import Path
import seaborn as sns
import glob #good for 
import cv2

import io, time
tm = time.time()

%load_ext autotime
date="05092019" #defining the date for saving files later
wd= os.getcwd() #'/Users/geickelb1/Documents/GitHub/mimiciii-antibiotics-modeling'
wd<jupyter_output><empty_output><jupyter_text>first pass all images through autocropper<jupyter_code>def auto_crop_process(file):
    global OUTPUT_DIR
    # Print out some file information
    image = Image.open(file)
    image_width = image.size[0]
    image_height = image.size[1]

#     print("Image type: " + image.format + "; mode: " + image.mode + "; dimensions: " + str(image_width) + "x" + str(image_height))
    
    ######   MODIFIED START  ##########
    if image.mode!="RGB":
        image= image.convert(mode='RGB')
    else: pass
    ######   MODIFIED END  ##########

    # Sample background color
    def rgb_tuple_to_str(tuple):
        return 'rgb(' + str(tuple[0]) + ', ' + str(tuple[1]) + ', ' + str(tuple[2]) + ')'

    def is_like_bg_color(color):
        color_r, color_g, color_b = color[0], color[1], color[2]
        bg_r, bg_g, bg_b = bg_color[0], bg_color[1], bg_color[2]
        r_similar, g_similar, b_similar = False, False, False

        if color_r in range(bg_r - TOLERANCE, bg_r + TOLERANCE):
            r_similar = True
            
        if color_g in range(bg_g - TOLERANCE, bg_g + TOLERANCE):
            g_similar = True
            
        if color_b in range(bg_b - TOLERANCE, bg_b + TOLERANCE):
            b_similar = True

        return r_similar and g_similar and b_similar

#     print("Sampling background color...")
    pixel_map = image.load()
    x_offset = image_width * 0.05
    y_offset = image_height * 0.05

    ul_color = pixel_map[x_offset, y_offset]
    ur_color = pixel_map[image_width - x_offset, y_offset]
    ll_color = pixel_map[x_offset, image_height - y_offset]
    lr_color = pixel_map[image_width - x_offset, image_height - y_offset]
    
    
    ######   MODIFIED START  ##########
    bg_color = image.getpixel((1, 1))#()#()  
    ###### MODIFIED END ###########


    if ul_color == ur_color and ur_color == ll_color and ll_color == lr_color:
        bg_color = ul_color
#         print("Sampled background color: " + rgb_tuple_to_str(ul_color))

    # Search for top edge
#     print("Searching for top edge...")
    top_edge_coords = []

    for i in range(0, image_width, int(image_width / 10)):
        for y in range(0, image_height - 1):
            if not is_like_bg_color(pixel_map[i, y]):
                top_edge_coords.append(y)
                break

    top_edge_coord = top_edge_coords[0]
    for c in top_edge_coords:
        if c < top_edge_coord:
            top_edge_coord = c

#     print("Found top edge at y = " + str(top_edge_coord))

    # Search for bottom edge
#     print("Searching for bottom edge...")
    bottom_edge_coords = []

    for i in range(0, image_width, int(image_width / 10)):
        for y in range(image_height - 1, 0, -1):
            if not is_like_bg_color(pixel_map[i, y]):
                bottom_edge_coords.append(y)
                break

    bottom_edge_coord = bottom_edge_coords[0]
    for c in bottom_edge_coords:
        if c > bottom_edge_coord:
            bottom_edge_coord = c

#     print("Found bottom edge at y = " + str(bottom_edge_coord))

    # Search for left edge
#     print("Searching for left edge...")
    left_edge_coords = []

    for i in range(0, image_height, int(image_height / 10)):
        for x in range(0, image_width - 1):
            if not is_like_bg_color(pixel_map[x, i]):
                left_edge_coords.append(x)
                break

    left_edge_coord = left_edge_coords[0]
    for c in left_edge_coords:
        if c < left_edge_coord:
            left_edge_coord = c

#     print("Found left edge at x = " + str(left_edge_coord))
    
    # Search for right edge
#     print("Searching for right edge...")
    right_edge_coords = []

    for i in range(0, image_height, int(image_height / 10)):
        for x in range(image_width - 1, 0, -1):
            try:
                if not is_like_bg_color(pixel_map[x, i]):
                    right_edge_coords.append(x)
                    break
            except IndexError:
                pass

    right_edge_coord = right_edge_coords[0]
    for c in right_edge_coords:
        if c > right_edge_coord:
            right_edge_coord = c

#     print("Found right edge at x = " + str(right_edge_coord))

    # Crop image
#     print("Cropping image...")
    cropped_image = image.crop((left_edge_coord, top_edge_coord, right_edge_coord, bottom_edge_coord))

    # Save image to output dir
    file_name, file_ext = os.path.splitext(file)
    output_file_name = os.path.basename(file_name) + '_processed' + file_ext
    output_file_path = os.path.join(os.getcwd(), OUTPUT_DIR, output_file_name)
#     print("Saving image to " + output_file_path)
    cropped_image.save(output_file_path)
def auto_crop(input_dir, output_dir):
    image_exts = [ '.jpg', '.jpeg', '.png', ".JPG",".JPEG", ".PNG" ]
    #input_dir = Path("../data/raw/no/") #os.path.join(os.getcwd(), 'img')
    #output_dir = Path("../data/processed/autotest/no/")#os.path.join(os.getcwd(), 'out')

    # Create output directory, if not present
    try:
        os.stat(output_dir)
    except:
        os.mkdir(output_dir)

    # Iterate over working directory
    for file in os.listdir(input_dir):
        file_path = os.path.join(input_dir, file)
        file_name, file_ext = os.path.splitext(file_path)

        # Check if file is an image file
        if file_ext not in image_exts:
            #print("Skipping " + file + " (not an image file)")
            continue
        else:
            #print()
            #print("Processing " + file + "...")
            auto_crop_process(file_path)<jupyter_output>time: 2.31 ms<jupyter_text>running autocrop on yes and no folder<jupyter_code>##running autocrop on yes folder
INPUT_DIR = Path("../data/raw/no/")
#OUTPUT_DIR = 'out'
OUTPUT_DIR=Path("../data/processed/no/")
TOLERANCE = 11

auto_crop(INPUT_DIR, OUTPUT_DIR)

##running autocrop on no folder
INPUT_DIR = Path("../data/raw/yes/")
#OUTPUT_DIR = 'out'
OUTPUT_DIR=Path("../data/processed/yes/")
TOLERANCE = 11

auto_crop(INPUT_DIR, OUTPUT_DIR)<jupyter_output>time: 2.26 s<jupyter_text>Second pass: my crop/scale algorithm<jupyter_code>def max_dimensions():
    """use to determine the max height and width of images to determine final image size to standardize to.
    the following functions will be tuned assuming this has been run and interpreted"""
    
    global allyes, allno
    #finding max dimensions
    dim_list=[]
    for i in range(0,len(allyes)):
        dim_list.append(cv2.imread(allyes[i]).shape)
    for i in range(0,len(allno)):
        dim_list.append(cv2.imread(allno[i]).shape)
    #dim_list
     
    largest_l=0 #longest height
    largest_w=0 #longst width
    i=0
    i_l=0 #index of tallest
    i_w=0 #index of widest
    for element in dim_list:
        if element[0]>largest_l:
            largest_l=element[0]
            i_l=i
        else: pass
        if element[1]>largest_w:
            largest_w=element[1]
            i_w=i
        else: pass
        i+=1
    
    return(largest_l, largest_w, i_l, i_w)

def img_std_size1(im, h):

    h=h #desired size decided based on output from max_dimensions
    
    ##scaling a pictures smallest dimension to 1500
    old_size=im.shape[:2]
    
    scale_ratio=h/min(old_size)
    
    new_size = tuple([int(round(x*scale_ratio)) for x in old_size])
    im = cv2.resize(im, (new_size[1], new_size[0]))
    
    #cropping the rescaled image to 1500x1500
    x1=int((im.shape[1]/2) -(h/2))
    y1=int((im.shape[0]/2) -(h/2))

    crop_img = im[y1:y1+h, x1:x1+h].copy() #the .copy() makes sure it doesn't change the image inline and saves it as new
    
    return(crop_img)


def img_std_size2(im, h):
    'scale the largest to 1500 and add black background'
    from cv2 import copyMakeBorder
#     print("img size:", img.shape)
    
    h=h #desired size
    ##scaling a pictures smallest dimension to 1500
    old_size=im.shape[:2]

    ##scaling image down by factor of 1500/max dimension
    scale_ratio=h/max(old_size) #scale it up or down to max=1500
    new_size = tuple([int(x*scale_ratio) for x in old_size])
    im = cv2.resize(im, (new_size[1], new_size[0]))

    ###adding appropriate amount of boarder
    #dealing with odd numbers by adding a pixel on top or left
    i=0
    if im.shape[0] %2 ==1: #if number is odd
        i=1
    j=0
    if im.shape[1] %2 ==1: #if number is odd
        j=1
    
    top=int((h-im.shape[0])/2)+i
    bottom=int((h-im.shape[0])/2)
    left=int((h-im.shape[1])/2)+j
    right=int((h-im.shape[1])/2)

    color = [0, 0, 0]
    border_img=copyMakeBorder(im, top, bottom, left, right, cv2.BORDER_CONSTANT,
        value=color)
    return(border_img)

def master_img_scale(im, h=1500):
    if max(im.shape)>h:
        scaled_im = img_std_size1(im, h=h)
    else:
        scaled_im = img_std_size2(im, h=h)
    return(scaled_im)
def main(newsize=1500, normalize=True):
    X=[]    #
    y=[]    #onehot encoded y label
    
    global allyes, allno
    
    #allyes,allno=gather_image_files()
    
    for myFile in allyes:
        image = cv2.imread(myFile)#, cv2.COLOR_BGR2RGB)
        scaled= master_img_scale(image, h=newsize)
        X.append (np.array(scaled))
        y.append([1., 0.])
    for myFile in allno:
        image = cv2.imread(myFile)#, cv2.COLOR_BGR2RGB)
        scaled= master_img_scale(image, h=newsize)
        X.append (np.array(scaled))
        y.append([0., 1.])
    
    if normalize==True:
        X = np.array(X,dtype='float32')  / 255.0 #convert to np array and scale values to between 0-1.
    else:
        X= np.array(X)
    #X= X[:,:,:,0] #each rgb layer provides same info since it's greyscaled already
    y = np.array(y,dtype='float64') #as mnist
    
    return(X,y)
##filepath of autocropped images
allyes=glob.glob("../data/processed/yes/"+"*")#'*.[pjJ][npP][gG]') #grabs all jpg and png files
allno=glob.glob("../data/processed/no/"+"*")#'*.[pjJ][npP][gG]')

allno[:3]
# plt.imshow(cv2.imread(allyes[0]))<jupyter_output>time: 791 µs<jupyter_text>run max dimensions to see the maximum width and height of all images in allyes and allno.* use this to determine h for main()<jupyter_code>max_dimensions()
X,y= main(newsize=400, normalize=False)#h=1250) 1250 had issue at number 48<jupyter_output>time: 876 ms<jupyter_text>Visualize cropping<jupyter_code># for i in range(0,50):
#     print(i)
#     plt.imshow(X[i])
#     plt.show()
# for i in range(160,247):
#     print(i)
#     plt.imshow(X[i])
#     plt.show()<jupyter_output>time: 287 µs<jupyter_text>Third pass: scale images down to desired size<jupyter_code>#test
#im = cv2.resize(im, (new_size[1], new_size[0]))
desired_pixel_size=224 #200
size=tuple([desired_pixel_size,desired_pixel_size])
# for i in range(160,247):
#     print(i)
#     plt.imshow(cv2.resize(X[i],size)) #plt.imshow(X[i])
#     plt.show()
# for i in range(0,len(X)):
#     #print(i)
#     X[i]=cv2.resize(X[i],size) #plt.imshow(X[i])
X.shape
from sklearn.model_selection import train_test_split
X_train,X_test,y_train,y_test = train_test_split(X, y, test_size=0.2, random_state=13)
y_test.shape
y_train_tindex = y_train[:,0]==True
y_train_findex = y_train[:,1]==True
y_test_tindex = y_test[:,0]==True
y_test_findex = y_test[:,1]==True
# imageio.imwrite(pn, im)
#scipy.misc.imsave(pn, im) #path,img
# basepath= "../data"

# if folder_name != None:
#     address = os.path.join(basepath, folder_name)
# else:
#     address = basepath
if not os.path.exists("../data/train/True/"):
    os.makedirs("../data/train/True/")
if not os.path.exists("../data/train/False/"):
    os.makedirs("../data/train/False/")
    
if not os.path.exists("../data/test/True/"):
    os.makedirs("../data/test/True/")
if not os.path.exists("../data/test/False/"):
    os.makedirs("../data/test/False/")
# import imageio
# imageio.imwrite("../data/train/{}.png".format(1), X_train[1])<jupyter_output>time: 776 µs<jupyter_text>save/load<jupyter_code>##setting images in format for Ellick's algorithm
import imageio

i=1
for im in X_train[y_train_tindex]:
    pn=("../data/train/True/{}.png".format(i))
    imageio.imwrite(pn, im)
    i+=1

i=1
for im in X_train[y_train_findex]:
    pn=("../data/train/False/{}.png".format(i))
    imageio.imwrite(pn, im)
    i+=1
i=1
for im in X_test[y_test_tindex]:
    pn=("../data/test/True/{}.png".format(i))
    imageio.imwrite(pn, im)
    i+=1
i=1
for im in X_test[y_test_findex]:
    pn=("../data/test/False/{}.png".format(i))
    imageio.imwrite(pn, im)
    i+=1

# np.save(Path("../data/processed/%s_X.npy" %(date)),X)
# np.save(Path("../data/processed/%s_y.npy" %(date)),y)

# #can load with:
# #X=np.load("../data/processed/%s_X.npy"%(date))
# #y=np.load("../data/processed/%s_y.npy"%(date))<jupyter_output>time: 191 µs
{'fraction_non_alphanumeric': 0.140647400588546, 'fraction_numerical': 0.01999547272315702, 'mean_word_length': 2.99096657633243, 'pattern_counts': {'":': 1, '<': 28, '<?xml version=': 0, '>': 31, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '220016', 'license': 'MIT', 'n_tokens_mistral': 5088, 'n_tokens_neox': 4862, 'n_words': 1087, 'path': 'notebooks/01-data_preprocessing.ipynb', 'repo_name': 'geickelb/dl_mri_project', 'size': '13253'}