<jupyter_start><jupyter_text>Motivating example: Parallel image processing with scikit-image To get a sense of what IPython.parallel might be used for,we start with an example of some batch processing of image files with [scikit-image](http://scikit-image.org/).We will revisit pieces of this example as we learn about the different components of IPython.You can download images with [this notebook](../download-images.ipynb), or get a zip [here](https://s3.amazonaws.com/ipython-parallel-data/images.zip), or find any images on your computer.<jupyter_code>import sys
import requests
from zipfile import ZipFile, BadZipFile
from ipywidgets import IntProgress
from IPython.display import display

images = os.path.join('..', 'images')
images_url = "https://s3.amazonaws.com/ipython-parallel-data/images.zip"


def download_images():
    r = requests.get(images_url, stream=True)
    content_length = r.headers.get('content-length')
    print("Downloading images")
    sys.stdout.flush()
    p = IntProgress(max=content_length)
    display(p)
    with open(images_zip, 'wb') as f:
        for chunk in r.iter_content(chunk_size=8192):
            p.value += len(chunk)
            f.write(chunk)


if not os.path.exists(images):
    images_zip = images + '.zip'
    if os.path.exists(images_zip):
        try:
            zf = ZipFile(images_zip)
        except BadZipFile:
            os.remove(images_zip)
        else:
            zf.close()
    if not os.path.exists(images_zip):
        download_images()
    
    ZipFile(images_zip).extractall('..')

%matplotlib inline
import matplotlib.pyplot as plt
import sys,os,re,time
import urllib

import numpy as np

import ipyparallel as parallel
from skimage.io import imread
from skimage.feature import corner_harris, corner_peaks<jupyter_output><empty_output><jupyter_text>Define a function to find the corners of the image:<jupyter_code>def find_corners(path, min_distance=5):
    """Find corners in an image at path
    
    Returns the image and the corner lists.
    """
    img = imread(path, flatten=True)
    corners = corner_peaks(corner_harris(img), min_distance=min_distance)
    return img, corners<jupyter_output><empty_output><jupyter_text>and show the contours on the image with matplotlib:<jupyter_code>def plot_corners(img, corners, show=True):
    """Display the image and plot all contours found"""
    plt.imshow(img, cmap='gray')
    plt.plot(corners[:,1], corners[:,0], 'r+', markeredgewidth=1.5, markersize=8) # Plot corners
    plt.axis('image')
    plt.xticks([])
    plt.yticks([])
    if show:
        plt.show()

def get_corners_image(path):
    """Given a path, return a PNG of the image with contour lines
    
    Calls both find_contours and plot_contours
    """
    from IPython.core.pylabtools import print_figure
    
    img, corners = find_corners(path)
    plot_corners(img, corners, show=False)
    fig = plt.gcf()
    pngdata = print_figure(fig)
    plt.close(fig)
    return pngdata

import os
pictures_dir = os.path.join('..', 'images', 'portrait')

pictures = []
for directory, subdirs, files in os.walk(pictures_dir):
    for fname in files:
        if fname.lower().endswith(('.jpg', '.png')):
            pictures.append(os.path.join(directory, fname))

pictures[:5]<jupyter_output><empty_output><jupyter_text>Let's test our function locally, to see what it does.<jupyter_code>for p in pictures[:3]:
    img, corners = find_corners(p)
    plot_corners(img, corners)<jupyter_output><empty_output><jupyter_text>Now in parallel First, we connect our parallel Client<jupyter_code>rc = parallel.Client()
all_engines = rc[:]
view = rc.load_balanced_view()<jupyter_output><empty_output><jupyter_text>Then we initialize the namespace on all of the engines with imports<jupyter_code>%px import os; os.chdir("{os.getcwd()}")
%%px
%matplotlib inline
import matplotlib.pyplot as plt

from skimage.io import imread
from skimage.feature import corner_harris, corner_peaks<jupyter_output><empty_output><jupyter_text>and make sure some functions are defined everywhere (this is only necessary for the `contours_in_url` case)<jupyter_code>all_engines.push(dict(
    plot_corners=plot_corners,
    find_corners=find_corners,
))<jupyter_output><empty_output><jupyter_text>Now we can iterate through all of our pictures, and detect and display any corners we find<jupyter_code>from IPython.display import display, Image

amr = view.map_async(get_corners_image, pictures[:20], ordered=False)
for pngdata in amr:
    display(Image(data=pngdata))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10661683886568477, 'fraction_numerical': 0.003517256539898879, 'mean_word_length': 4.436081242532856, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '264601', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1521, 'n_tokens_neox': 1482, 'n_words': 392, 'path': 'parallel/examples/Parallel image processing.ipynb', 'repo_name': 'minrk/ipython-cse17', 'size': '4549'}