<jupyter_start><jupyter_text>Generate data with small beads and Poisson noise from experimental PSF<jupyter_code>import numpy as np
import tifffile
import napari
#Create a 2d array to do FFT
ashape = (256,256,256)

a = np.zeros(ashape, dtype=float)

#Add a few cubes in grid-like locations
cubesize=2
cubespacing=60
for iz in range(int(cubespacing/2),ashape[0],cubespacing):
    for iy in range(int(cubespacing/2),ashape[1],cubespacing):
        for ix in range(int(cubespacing/2),ashape[2],cubespacing):
            a[iz:iz+cubesize , iy:iy+cubesize , ix:ix+cubesize] = np.ones((cubesize,cubesize,cubesize))
nview_data  = napari.view_image(a, ndisplay=3)
#OK
#Optionally save the data
tifffile.imsave('test/gendata1_raw.tif', a)<jupyter_output><empty_output><jupyter_text>Convolve data with the experimental 'Rosalind' Psf.Read data first<jupyter_code>psfdata=tifffile.imread('PSF_RFI_8bit.tif')
type(psfdata)
psfdata.dtype
psfdata.shape
psfdata_norm = (psfdata.astype(float) - psfdata.min() ) / (psfdata.max() - psfdata.min())
nview_psf = napari.view_image(psfdata_norm, ndisplay=3)<jupyter_output>Exception in callback BaseAsyncIOLoop._handle_events(4036, 1)
handle: <Handle BaseAsyncIOLoop._handle_events(4036, 1)>
Traceback (most recent call last):
  File "C:\Users\Luis\miniconda3\envs\dev\lib\asyncio\events.py", line 81, in _run
    self._context.run(self._callback, *self._args)
  File "C:\Users\Luis\miniconda3\envs\dev\lib\site-packages\tornado\platform\asyncio.py", line 189, in _handle_events
    handler_func(fileobj, events)
  File "C:\Users\Luis\miniconda3\envs\dev\lib\site-packages\zmq\eventloop\zmqstream.py", line 452, in _handle_events
    self._handle_recv()
  File "C:\Users\Luis\miniconda3\envs\dev\lib\site-packages\zmq\eventloop\zmqstream.py", line 481, in _handle_recv
    self._run_callback(callback, msg)
  File "C:\Users\Luis\miniconda3\envs\dev\lib\site-packages\zmq\eventloop\zmqstream.py", line 431, in _run_callback
    callback(*args, **kwargs)
  File "C:\Users\Luis\miniconda3\envs\dev\lib\site-packages\jupyter_client\threaded.py", line 121, in _handle_recv
    m[...]<jupyter_text>Convolve<jupyter_code>import scipy.signal
data_convolved = scipy.signal.convolve(a, psfdata_norm, mode='same')
data_convolved.shape
#normalises to 0-255 range
data_convolved = (data_convolved - data_convolved.min()) / (data_convolved.max() - data_convolved.min())*255
print(data_convolved.max())
print(data_convolved.min())
nview_dataconv = napari.view_image(data_convolved,ndisplay=3)<jupyter_output><empty_output><jupyter_text>Add Poisson noise<jupyter_code>#data_convolved_noised = data_convolved + np.random.poisson(256 , size=ashape).astype(np.float32)/80
#This method of adding does not look right. The original intensity should be the lambda poisson parameter in the function
rng = np.random.default_rng()
data_convolved_noised = rng.poisson(lam = data_convolved)
nview_data_noised = napari.view_image(data_convolved_noised,ndisplay=3)
data_convolved_noised_uint8 = ((data_convolved_noised - data_convolved_noised.min()) / ( data_convolved_noised.max() - data_convolved_noised.min() ) *255 ).astype(np.uint8)
tifffile.imsave('test/gendata_psfconv_poiss.tif', data_convolved_noised_uint8)<jupyter_output><empty_output><jupyter_text>Create large data<jupyter_code>import numpy as np
import tifffile
import napari
import scipy.signal

#Create a 2d array to do FFT
ashape = (60,1026,1544) # Casper LM size
a = np.zeros(ashape, dtype=float)
#a = np.random.poisson(256 , size=(size0,size0,size0)).astype(np.float32)/2000

#Add a few cubes in grid-like locations
cubesize=2
cubespacing=67
for iz in range(5,ashape[0],cubespacing):
    for iy in range(5,ashape[1],cubespacing):
        for ix in range(5,ashape[2],cubespacing):
            a[iz:iz+cubesize , iy:iy+cubesize , ix:ix+cubesize] = np.ones((cubesize,cubesize,cubesize))

#Read psf
psfdata=tifffile.imread('PSF_RFI_8bit.tif')
psfdata_norm = (psfdata.astype(float) - psfdata.min() ) / (psfdata.max() - psfdata.min())

#Convolve
data_convolved = scipy.signal.convolve(a, psfdata_norm, mode='same')

#Adjust max/min and intensity
data_convolved = (data_convolved - data_convolved.min()) / (data_convolved.max() - data_convolved.min())*255

#Noisify with Poisson
rng = np.random.default_rng()
data_convolved_noised = rng.poisson(lam = data_convolved)
data_convolved_noised_uint8 = ((data_convolved_noised - data_convolved_noised.min()) / ( data_convolved_noised.max() - data_convolved_noised.min() ) *255 ).astype(np.uint8)
tifffile.imsave('gendata_psfconv_poiss_large.tif', data_convolved_noised_uint8)<jupyter_output><empty_output><jupyter_text>Create very large data<jupyter_code>import numpy as np
import tifffile
import napari
import scipy.signal

#Create a 2d array to do FFT
ashape = (51,2048,2048) # <NAME>
a = np.zeros(ashape, dtype=float)
#a = np.random.poisson(256 , size=(size0,size0,size0)).astype(np.float32)/2000

#Add a few cubes in grid-like locations
cubesize=2
cubespacing=67
for iz in range(5,ashape[0],cubespacing):
    for iy in range(5,ashape[1],cubespacing):
        for ix in range(5,ashape[2],cubespacing):
            a[iz:iz+cubesize , iy:iy+cubesize , ix:ix+cubesize] = np.ones((cubesize,cubesize,cubesize))

#Read psf
psfdata=tifffile.imread('PSF_RFI_8bit.tif')
psfdata_norm = (psfdata.astype(float) - psfdata.min() ) / (psfdata.max() - psfdata.min())

#Convolve
data_convolved = scipy.signal.convolve(a, psfdata_norm, mode='same')

#Adjust max/min and intensity
data_convolved = (data_convolved - data_convolved.min()) / (data_convolved.max() - data_convolved.min())*255

#Noisify with Poisson
rng = np.random.default_rng()
data_convolved_noised = rng.poisson(lam = data_convolved)
data_convolved_noised_uint8 = ((data_convolved_noised - data_convolved_noised.min()) / ( data_convolved_noised.max() - data_convolved_noised.min() ) *255 ).astype(np.uint8)
nview_data_noised = napari.view_image(data_convolved_noised_uint8,ndisplay=3)
tifffile.imsave('gendata_psfconv_poiss_vlarge.tif', data_convolved_noised_uint8)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12878161679616465, 'fraction_numerical': 0.027111919325508348, 'mean_word_length': 6.405140758873929, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '518380', 'license': 'Apache-2.0', 'n_tokens_mistral': 2561, 'n_tokens_neox': 2369, 'n_words': 390, 'path': 'test/testdata/GenerateConvolutedData.ipynb', 'repo_name': 'DragaDoncila/RedLionfish', 'size': '6056'}