<jupyter_start><jupyter_text>This notebook tests the output of the bioimage-search image-preprocessing service.It assumes the "standard" organization of both the BBBC-021 dataset and the test output.<jupyter_code>!pip install shortuuid
!pip install s3fs
!pwd
import boto3
import sys
import pandas as pd
import numpy as np
import s3fs
# path setup for SageMaker Studio
sys.path.insert(0, "../../../../bioimage-search/datasets/bbbc-021/scripts")
import bbbc021common as bb
sys.path.insert(0, "../../../../bioimage-search/main/src/common")
import bioimageimage as bi
from PIL import Image
from skimage.exposure import histogram
import matplotlib.pyplot as plt
import math
s3c = boto3.client('s3')
s3f = s3fs.S3FileSystem(anon=False)
BBBC021_SOURCE_BUCKET='bioimagesearchbbbc021stack-bbbc021bucket544c3e64-10ecnwo51127'
BIOIMAGE_SEARCH_TEST_BUCKET='bioimagesearchbasestack-bioimagesearchtestbucket3-djdwcbvul5zb'
plateName = 'Week10_40111'
Week10_40111_Info = bb.Bbbc021PlateInfo(BBBC021_SOURCE_BUCKET, plateName)
dapiFileList = Week10_40111_Info.getDapiFileList()<jupyter_output><empty_output><jupyter_text>We use the DAPI channel as the index key for the full set of channels<jupyter_code>testImageKey = dapiFileList[0]
testImageKey<jupyter_output><empty_output><jupyter_text>Load generate MIP of source images<jupyter_code>def getBbbcImageFilesByDapiKey(dapiImageKey, plateInfo):
    fileKeys={}
    fileKeys['dapi']=dapiImageKey
    fileKeys['actin']=plateInfo.getActinFileByDapi(dapiImageKey)
    fileKeys['tubulin']=plateInfo.getTubulinFileByDapi(dapiImageKey)
    return fileKeys
t1 = getBbbcImageFilesByDapiKey(testImageKey, Week10_40111_Info)
t1
def getBbbcMipByDapiKey(dapiImageKey, bucket, plateInfo):
    input_arr=[]
    fileKeys = getBbbcImageFilesByDapiKey(dapiImageKey, plateInfo)
    fileArr = []
    fileArr.append(fileKeys['dapi'])
    fileArr.append(fileKeys['tubulin'])
    fileArr.append(fileKeys['actin'])
    for key in fileArr:
        fileObject = s3c.get_object(Bucket=bucket, Key=key)
        file_stream = fileObject['Body']
        im = Image.open(file_stream)
        pix = np.array(im)
        input_arr.append(pix)
    input_data = np.array(input_arr)
    input_data = bi.normImageData(input_data)

    bavgFill = np.zeros(shape=input_data[0].shape, dtype=input_data.dtype)
    for c in range(input_data.shape[0]):
        channelData = input_data[c]
        h1 = histogram(channelData, 100)
        bcut = bi.findHistCutoff(h1, 0.20)
        bavg = bi.findCutoffAvg(channelData, bcut)
        bavgFill.fill(bavg)
        bi.normalizeChannel(bavgFill, channelData)
        
    ca = bi.getColors(input_data.shape[0])
    mip = bi.calcMip(input_data, ca)
    return mip
mip = getBbbcMipByDapiKey(testImageKey, BBBC021_SOURCE_BUCKET, Week10_40111_Info)
mip.shape
def showNdFloatArrImage(ndarr):
    plt.figure(figsize=(20,15))
    plt.subplot(1,1,1)
    plt.xticks([])
    plt.yticks([])
    im=Image.fromarray(ndarr)
    print("shape=", im.size)
    print("format=", im.format)
    ip=plt.imshow(im)
    #plt.xlabel(0)
    plt.show()
showNdFloatArrImage(mip)<jupyter_output><empty_output><jupyter_text>Next, we load the ROIs and show outlines:ROI compute:    /ROI//-roi.npy (contains normalized multichannel ROI data ready for training)    /ROI//-roi.json (contains list of ROI coordinates wrt raw image, ordered wrt the npy file)<jupyter_code>def loadROIDataByDapiKey(dapiImageKey, bucket):
    roiKeyPrefix = 'ROI/' + dapiImageKey[:-4] + '-roi'
    roiDataKey = roiKeyPrefix + '.npy'
    roiJsonKey = roiKeyPrefix + '.json'
    roiData = bi.getNumpyArrayFromS3(bucket, roiDataKey)
    roiInfo = bi.loadJsonObjectFromS3(roiJsonKey, bucket)
    return roiData, roiInfo
roiData, roiInfo = loadROIDataByDapiKey(testImageKey, BIOIMAGE_SEARCH_TEST_BUCKET)
roiInfo
def roiAnnotate2DDataXY(data, roiInfo, color, index=-1):
    roiSize = roiInfo['roisize']
    roiX=roiSize['x']
    roiY=roiSize['y']
    roiArr = roiInfo['roi']
    i=0
    for roi in roiArr:
        if index<0 or i==index:
            x=roi['x']
            y=roi['y']
            for xi in range(roiX):
                data[y][x+xi]=color
                data[y+roiY][x+xi]=color
            for yi in range(roiY):
                data[y+yi][x]=color
                data[y+yi][x+roiX]=color
        i+=1
mip2=np.copy(mip)
roiAnnotate2DDataXY(mip2, roiInfo, [255.0, 0.0, 0.0], 0)
showNdFloatArrImage(mip2)
def displayRoiImages(mip, roiInfo):
    plt.figure(figsize=(25,35))
    roiSize = roiInfo['roisize']
    roiX=roiSize['x']
    roiY=roiSize['y']
    roiArr = roiInfo['roi']
    i=0
    l =len(roiArr)
    spx=10
    spy=math.ceil(l/spx)
    print("Count=", l)
    for roi in roiArr:
        x0=roi['x']
        y0=roi['y']
        x1 = x0 + roiX
        y1 = y0 + roiY
        mipData=mip[y0:y1,x0:x1]
        im=Image.fromarray(mipData)
        plt.subplot(spy,spx,i+1)
        plt.xticks([])
        plt.yticks([])
        plt.imshow(im)
        plt.xlabel(i)
        i+=1
displayRoiImages(mip, roiInfo)
def displayRoiChannelImages(mip, roiInfo, roiData):
    #plt.figure(figsize=(25,600))
    roiSize = roiInfo['roisize']
    roiX=roiSize['x']
    roiY=roiSize['y']
    roiArr = roiInfo['roi']
    i=0
    #s=0
    #l =len(roiArr)
    w = roiData[0].shape[0] + 1
    spx=w
    #spy=l
    for roi in roiArr:
        s=0
        plt.figure(figsize=(25,35))
        l = 1
        spy = l
        # Mip
        x0=roi['x']
        y0=roi['y']
        x1 = x0 + roiX
        y1 = y0 + roiY
        mipData=mip[y0:y1,x0:x1]
        im=Image.fromarray(mipData)
        plt.subplot(spy,spx,s+1)
        plt.xticks([])
        plt.yticks([])
        plt.imshow(im)
        plt.xlabel(i)
        s+=1
        chanData=roiData[i]
        for c in range(chanData.shape[0]):
            chan=chanData[c]
            mc = bi.calcMip(chan, bi.colors)
            ci=Image.fromarray(mc)
            plt.subplot(spy,spx,s+1)
            plt.xticks([])
            plt.yticks([])
            plt.imshow(ci)
            plt.xlabel(i)
            s+=1
        i+=1
displayRoiChannelImages(mip, roiInfo, roiData)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12483660130718954, 'fraction_numerical': 0.029901960784313727, 'mean_word_length': 3.2330567081604427, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '605922', 'license': 'Apache-2.0', 'n_tokens_mistral': 2426, 'n_tokens_neox': 2370, 'n_words': 404, 'path': 'cli/bioims/test/test-image-preprocessing.ipynb', 'repo_name': 'aws-samples/bioimage-search', 'size': '6120'}