<jupyter_start><jupyter_text>Splitting the data-set into three sets — train, validation and test sets.<jupyter_code>for _set in ['train', 'val', 'test']:
    n_normal = len(os.listdir(data + _set + '/NORMAL'))
    n_infect = len(os.listdir(data + _set + '/PNEUMONIA'))
    print('Set: {}, normal images: {}, pneumonia images: {}'.format(_set, n_normal, n_infect))<jupyter_output>Set: train, normal images: 1342, pneumonia images: 3876
Set: val, normal images: 9, pneumonia images: 9
Set: test, normal images: 234, pneumonia images: 390<jupyter_text>Defining a couple of data generators and setting batch_size = 64<jupyter_code>def process_data(img_dims, batch_size):
    # Data generation objects
    train_datagen = ImageDataGenerator(rescale=1./255, zoom_range=0.3, vertical_flip=True)
    test_val_datagen = ImageDataGenerator(rescale=1./255)
    
    # This is fed to the network in the specified batch sizes and image dimensions
    train_gen = train_datagen.flow_from_directory(
    directory=input_path+'train', 
    target_size=(img_dims, img_dims), 
    batch_size=batch_size, 
    class_mode='binary', 
    shuffle=True)

    test_gen = test_val_datagen.flow_from_directory(
    directory=input_path+'test', 
    target_size=(img_dims, img_dims), 
    batch_size=batch_size, 
    class_mode='binary', 
    shuffle=True)
    
    # I will be making predictions off of the test set in one batch size
    # This is useful to be able to get the confusion matrix
    test_data = []
    test_labels = []

    for cond in ['/NORMAL/', '/PNEUMONIA/']:
        for img in (os.listdir(input_path + 'test' + cond)):
            img = plt.imread(input_path+'test'+cond+img)
            img = cv2.resize(img, (img_dims, img_dims))
            img = np.dstack([img, img, img])
            img = img.astype('float32') / 255
            if cond=='/NORMAL/':
                label = 0
            elif cond=='/PNEUMONIA/':
                label = 1
            test_data.append(img)
            test_labels.append(label)
        
    test_data = np.array(test_data)
    test_labels = np.array(test_labels)
    
    return train_gen, test_gen, test_data, test_labels

    #  Defining some constants
    img_dims = 150
    epochs = 10
    batch_size = 32

    train_gen, test_gen, test_data, test_labels = process_data(img_dims, batch_size)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11012282930961458, 'fraction_numerical': 0.018212621770436255, 'mean_word_length': 3.2103386809269163, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '236074', 'license': 'MIT', 'n_tokens_mistral': 822, 'n_tokens_neox': 774, 'n_words': 191, 'path': '.ipynb_checkpoints/pneumonia-checkpoint.ipynb', 'repo_name': 'Shuaibu78/pneumonia-detection', 'size': '2361'}