<jupyter_start><jupyter_text>Data Augmentation SetsLoad COVIDx dataset according to the protocol proposed in [1], code available in the [link](https://https://github.com/lindawangg/COVID-Net), and includes augmneted data. Files are saved in h5 format to speed up future loading.The metadata can be download in [COVID-Net repositiry](https://https://github.com/lindawangg/COVID-Net).[1] <NAME>., & <NAME>. (2020). COVID-Net: A tailored deep convolutional neural network design for detection of COVID-19 cases from chest radiography images. arXiv preprint arXiv:2003.09871.<jupyter_code>from sklearn.metrics import confusion_matrix
import numpy as np
import tensorflow as tf
import os, argparse
import cv2
import numpy as np

mapping = {'normal': 0, 'pneumonia': 1, 'COVID-19': 2}

# number of images for each class
NUM_IMGS_N = 1000
NUM_IMGS_P = 1000
NUM_IMGS_COV = 1000  # the COVID-19 class is the only one augmented<jupyter_output><empty_output><jupyter_text><jupyter_code>y_train = []
x_train = []

count = 0
imgs_path = '/<path to COVID-19 augmented images>/'
for f in os.listdir(imgs_path): 
    if count < NUM_IMGS_COV:
      x = cv2.imread(os.path.join(imgs_path,f))
      x = cv2.resize(x, (224, 224))
      x = x.astype('float32') / 255.0
      y_train.append(mapping['COVID-19'])
      x_train.append(x)
      count = count + 1         
    
dim0x = np.array(x_train).shape
dim0y = np.array(y_train).shape
np.array(y_train).shape
np.array(x_train).shape
train_folder = '/<path to train data>/train'
train_file_paths = '/<path to COVIDx metadata files>/train_COVIDx.txt'
file = open(train_file_paths, 'r')
testfile = file.readlines()
count_P = 0
count_N = 0
for i in range(len(testfile)):
    line = testfile[i].split()
    try:
      
      if mapping[line[2]] == 0 and count_N < NUM_IMGS_N:
        x = cv2.imread(train_folder +'/' + line[1])
        x = cv2.resize(x, (224, 224))
        x = x.astype('float32') / 255.0
        y_train.append(mapping[line[2]])
        x_train.append(x)
        count_N = count_N+1

      if mapping[line[2]] == 1 and count_P < NUM_IMGS_P:
        x = cv2.imread(train_folder +'/' + line[1])
        x = cv2.resize(x, (224, 224))
        x = x.astype('float32') / 255.0
        y_train.append(mapping[line[2]])
        x_train.append(x)
        count_P = count_P+1

    except:
      None
    
    
dim1 = np.array(x_train).shape
dim2 = np.array(y_train).shape
test_folder = '/<path to test data>/test'
test_file_paths = '/<path to COVIdx metadata files>/test_COVIDx.dat'
file = open(test_file_paths, 'r')
testfile = file.readlines()
y_test = []
x_test = []
count = 0
for i in range(len(testfile)):
    line = testfile[i].split()
    try:
      x = cv2.imread(test_folder +'/' + line[1])
      x = cv2.resize(x, (224, 224))
      x = x.astype('float32') / 255.0
      y_test.append(mapping[line[2]])
      x_test.append(x)
      count = count+1
    except:
      None
dim3 = np.array(x_test).shape
dim4 = np.array(y_test).shape
x_train = np.array(x_train)
x_train.shape
y_train = np.array(y_train)
y_train.shape
x_test = np.array(x_test)
x_test.shape
y_test = np.array(y_test)
y_test.shape
uniqueValues, occurCount = np.unique(y_train, return_counts=True)
print("Unique Values for Train Set: " , uniqueValues)
print("Occurrence Count For Train Set: ", occurCount)
uniqueValues, occurCount = np.unique(y_test, return_counts=True)
print("Unique Values for Train Set: " , uniqueValues)
print("Occurrence Count For Train Set: ", occurCount)
import h5py
h5f = h5py.File('/<path to save files>/h5/dataset_COVIx_BALANCED_augmented.h5', 'w')
h5f.create_dataset('train_X', dim1, data= np.array(x_train), chunks=True)
h5f.create_dataset('train_y', dim2, data= np.array(y_train), chunks=True)
h5f.create_dataset('test_X', dim3, data= np.array(x_test), chunks=True)
h5f.create_dataset('test_y', dim4, data= np.array(y_test), chunks=True)
h5f.close()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13803680981595093, 'fraction_numerical': 0.034764826175869123, 'mean_word_length': 3.89125, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 17, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '228936', 'license': 'MIT', 'n_tokens_mistral': 1538, 'n_tokens_neox': 1446, 'n_words': 338, 'path': 'src/ia/covid19/save_dataset_balanced_flat_classification.ipynb', 'repo_name': 'ufopcsilab/covid-19', 'size': '3914'}