<jupyter_start><jupyter_text>__At the command line, from the directory of this notebook__ipcluster start -n 8<jupyter_code>%matplotlib inline
from IPython.parallel import Client
c = Client()
%%px --local
from skimage import io
from os import path
from matplotlib import pyplot as plt
import numpy as np
import os
import shutil
%%px --local
%run edges.py


in_path = path.normpath('/home/boris/Kaggle/Retina/train/raw')
out_path = path.normpath('/home/boris/Kaggle/Retina/train/cropped')
def crop(file_name):
    in_file = path.join(in_path, file_name)
    out_file = path.join(out_path, file_name)
    
    eye = io.imread(in_file)
    cropped = crop_cv(eye)
    io.imsave(out_file, cropped)<jupyter_output><empty_output><jupyter_text>Test it out:<jupyter_code>file_name = '21_left.jpeg'

#if path.exists(out_path):
#    shutil.rmtree(out_path)
#os.mkdir(out_path)

crop(file_name)
eye = io.imread(path.join(out_path, file_name))
plt.figure()
ax = plt.subplot(111)
ax.imshow(eye)
plt.show()
%time
dv = Client()[:]
files = os.listdir(in_path)

if path.exists(out_path):
    shutil.rmtree(out_path)
os.mkdir(out_path)

asr = dv.map(crop, files)
asr.wait()
files_out = set(os.listdir(out_path))
diff = set(files).difference(files_out)
diff
asr = dv.map(crop, diff)
asr.wait()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13287250384024576, 'fraction_numerical': 0.004608294930875576, 'mean_word_length': 5.387254901960785, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '180883', 'license': 'MIT', 'n_tokens_mistral': 532, 'n_tokens_neox': 519, 'n_words': 104, 'path': 'Notebooks/Unused/Crop on IPython Cluster.ipynb', 'repo_name': 'fierval/retina', 'size': '1302'}