<jupyter_start><jupyter_text>load image from path<jupyter_code>image_list[:10] # list of image in file
mask_list[:10] # list of mask image in file<jupyter_output><empty_output><jupyter_text>image value type is unsigned int The lungs image shape is 16bit(grayscale(8), alpha(8))<jupyter_code>img = cv2.imread(image_list[0], 0)
img.shape
np.unique(img)
plt.imshow(img, cmap = 'gray')
plt.show()
IMG_H, IMG_W = 512, 512
dataX, dataY = np.empty((2, len(image_list), IMG_H, IMG_W, 1), dtype=np.float32)
dataX.shape<jupyter_output><empty_output><jupyter_text>image resize<jupyter_code>for i, image in enumerate(image_list):
    
    # changing value type uint to int, float is changed to calculate
    img = cv2.imread(image, 0).astype("int16").astype('float32')
    #print(img.shape)        
    #img = cv2.resize(img, dsize=(IMG_W, IMG_H))
    img = (img - np.min(img)) / (np.max(img) - np.min(img))
    
    dataX[i] = np.expand_dims(img, axis = 2)    

dataX.shape
mask = cv2.imread(mask_list[0], 0)
print(mask.shape)
np.unique(mask)

plt.imshow(mask, cmap = 'gray')
plt.show()
mask[mask>=1] = 1 
np.unique(mask)

plt.imshow(mask, cmap = 'gray')
plt.show()
for i, image in enumerate(mask_list):
    img = cv2.imread(image, 0) #  .astype('float32')
    #img = cv2.resize(img, dsize=(IMG_W, IMG_H))    
    img[img>=1] = 1 #lung mask     
    dataY[i] = np.expand_dims(img, axis = 2)

dataY.shape
np.unique(dataX[0,0,:10])
np.unique(dataY)<jupyter_output><empty_output><jupyter_text>rescale of image plot image and mask of image<jupyter_code>fig, ax = plt.subplots(1, 2)
ax[0].imshow(dataX[0].squeeze(), cmap='gray')
ax[1].imshow(dataY[0].squeeze(), cmap='gray')
plt.show()<jupyter_output><empty_output><jupyter_text>Divide train and test data<jupyter_code>#load previous data  

save_dir = data_dir + "dataset/" # path where you want to save

trainX1 = np.load(save_dir+'x_train_512.npy')
trainY1 = np.load(save_dir+'y_train_512.npy')
testX1 = np.load(save_dir+'x_test_512.npy')
testY1 = np.load(save_dir+'y_test_512.npy')

print(trainX1.shape,trainY1.shape)
print(testX1.shape,testY1.shape)

dataX1 = np.concatenate((trainX1, testX1), axis=0)
dataY1 = np.concatenate((trainY1, testY1), axis=0)
print(dataX1.shape,dataY1.shape)

dataX2 =  np.concatenate((dataX, dataX1), axis=0)
dataY2 =  np.concatenate((dataY, dataY1), axis=0)
print(dataX2.shape,dataY2.shape)

trainX, testX = train_test_split(dataX2, test_size = 0.05, random_state=50)
trainY, testY = train_test_split(dataY2, test_size = 0.05, random_state=50)
save_dir = data_dir + "dataset/" # path where you want to save

if not os.path.exists(save_dir): # if there is no exist, make the path
    os.makedirs(save_dir)
    
np.save(save_dir+'x_train_512.npy', trainX)
np.save(save_dir+'y_train_512.npy', trainY)
np.save(save_dir+'x_test_512.npy', testX)
np.save(save_dir+'y_test_512.npy', testY)

print(trainX.shape, trainY.shape)
print(testX.shape, testY.shape)<jupyter_output>(966, 512, 512, 1) (966, 512, 512, 1)
(51, 512, 512, 1) (51, 512, 512, 1)
{'fraction_non_alphanumeric': 0.15873544093178035, 'fraction_numerical': 0.05058236272878536, 'mean_word_length': 5.450643776824034, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '146304', 'license': 'MIT', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1280, 'n_words': 242, 'path': 'segmentation/preprocessing_lungs-CNCB.ipynb', 'repo_name': 'deepcam-cn/3D-CNN-BERT-COVID19', 'size': '3005'}