<jupyter_start><jupyter_text>Import the data<jupyter_code># checking directory as well counting content
L = 0
for dirpath, dirnames, filenames in os.walk("./Dataset_A"):
    print(f"There are {len(dirnames)} directories and {len(filenames)} images in '{dirpath}'")
    L = L + len(filenames)
data_dir = pathlib.Path("./Dataset_A/") 
class_names = np.array(sorted([item.name for item in data_dir.glob('*')]))
print(class_names)<jupyter_output>['Covid' 'Normal']<jupyter_text>Functions for Preprocessing<jupyter_code>EXAMPLE_IMG_PATH = "./Dataset_A/Covid/2edb88df42cab5e5fbc18b3965e0bd_jumbo.jpeg"
img1 = cv2.imread(EXAMPLE_IMG_PATH)
print(f"Image example shape: {img1.shape}")
plt.imshow(img1)
plt.title("Example image")
plt.axis("off")
plt.show()
# Grayscale
def rgb2gray(rgb_img):
    output_img = cv2.cvtColor(rgb_img, cv2.COLOR_BGR2GRAY)
    return output_img
# Scale to 0 to 1
def scale(image):
    return image / 255
#resize
def resize_img(image, rows=224, cols=224):
    return cv2.resize(image, dsize=(rows, cols), interpolation=cv2.INTER_CUBIC)
# resize the shape
def reshape(image, axis):
    return np.expand_dims(image.mean(axis=axis), axis=1)
# Function to call other Preprocessing Functions
def preprocessed_img(input_img):
    output_img = rgb2gray(input_img)
    output_img = scale(output_img)
    output_img = resize_img(output_img)
    output_img = reshape(output_img, 1)
    return output_img
# Image converted in array
preprocessed_img(img1)<jupyter_output><empty_output><jupyter_text>Processing the whole dataset<jupyter_code># matrix containing L vectors of shape (224, 1)
mean_vector_matrix = np.zeros(shape=(L, 224, 1))
# Target vector containing the classes for L images
target_vector = np.zeros(shape=(L, 1))
# fill the matrix with normal class images. Class "1"
n = 0
for root, dirnames, filenames in os.walk("./Dataset_A/Normal"):
    n_total = len(filenames)
    for filename in filenames:
        file_path = os.path.join(root, filename)
        img = cv2.imread(file_path)
        img = preprocessed_img(img)
        mean_vector_matrix[n] = img
        target_vector[n] = 1
        if n % 20 == 0:
            print(f"File {n} {filename}")
        n = n + 1  
# fill the matrix with covid class images. Class "0"
for root, dirnames, filenames in os.walk("./Dataset_A/Covid"):
    n_total = len(filenames)
    for filename in filenames:
        file_path = os.path.join(root, filename)
        img = cv2.imread(file_path)
        img = preprocessed_img(img)
        mean_vector_matrix[n] = img
        target_vector[n] = 0
        if n % 20 == 0:
            print(f"File {n} {filename}")
        n = n + 1<jupyter_output>File 200 1-s2.0-S1684118220300608-main.pdf-001.jpg
File 220 10.1016-slash-j.crad.2020.04.002-b.png
File 240 16865_1_1.jpg
File 260 41591_2020_819_Fig1_HTML.webp-day10.png
File 280 82a78c13.jpg
File 300 auntminnie-a-2020_01_28_23_51_6665_2020_01_28_Vietnam_coronavirus.jpeg
File 320 covid-19-caso-95-3-14.png
File 340 covid-19-pneumonia-rapidly-progressive-12-hours.jpg
File 360 f8335316.jpg
File 380 ryct.2020003.fig2-a.png<jupyter_text>Train Test split<jupyter_code>SEED = 0
x_train, x_test, y_train, y_test = train_test_split(mean_vector_matrix, target_vector, test_size=0.2, random_state=SEED)
tf.random.set_seed(SEED)
np.random.seed(SEED)
input_shape = x_train.shape
input_shape<jupyter_output><empty_output><jupyter_text><jupyter_code>model = tf.keras.Sequential([
   tf.keras.layers.Flatten(),
   tf.keras.layers.Dense(units=64, activation='softmax', input_shape = input_shape),
   tf.keras.layers.Dense(units=64, activation='softmax', input_shape = input_shape),
   tf.keras.layers.Dense(units=64, activation='softmax', input_shape = input_shape),
   tf.keras.layers.Dense(units=2, activation='softmax')
])
model.compile(
  loss='sparse_categorical_crossentropy',
  optimizer = 'adam',
  metrics = ['accuracy']
)
history = model.fit(
  x = x_train,
  y = y_train,
  epochs = 100
)
plt.plot(history.history['accuracy'], label='accuracy')

plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.ylim([0.5, 1])
plt.legend(loc='lower right')

test_loss, test_acc = model.evaluate(x_test,  y_test, verbose=2)<jupyter_output>3/3 - 0s - loss: 0.2516 - accuracy: 0.9747 - 111ms/epoch - 37ms/step
{'fraction_non_alphanumeric': 0.12697663441113996, 'fraction_numerical': 0.057587915978286525, 'mean_word_length': 4.6431424766977365, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '505394', 'license': 'Apache-2.0', 'n_tokens_mistral': 1786, 'n_tokens_neox': 1606, 'n_words': 321, 'path': 'covid_x-ray_by_rounak.ipynb', 'repo_name': 'Rounak-Paul/covid_x-ray', 'size': '4237'}