<jupyter_start><jupyter_text>Import the data from Edge Impulse. You can obtain the URL from the Dashboard, right-click on the download icon next to 'Spectral features data' and 'Spectral features labels', and click **Copy link location**.<jupyter_code>import numpy as np
import requests

API_KEY = 'PROVIDE YOUR 68 DIGIT API KEY FROM EDGE IMPULSE STUDIO PROJECT'

X = (requests.get('https://studio.edgeimpulse.com/v1/api/PROJECT ID/training/8/x', headers={'x-api-key': API_KEY})).content
Y = (requests.get('https://studio.edgeimpulse.com/v1/api/PROJECT ID/training/8/y', headers={'x-api-key': API_KEY})).content<jupyter_output><empty_output><jupyter_text>Store the data in a temporary file, and load it back through Numpy.<jupyter_code>with open('x_train.npy', 'wb') as file:
    file.write(X)
with open('y_train.npy', 'wb') as file:
    file.write(Y)
X = np.load('x_train.npy')
Y = np.load('y_train.npy')[:,0]<jupyter_output><empty_output><jupyter_text>Define our labels and split the data up in a test and training set:<jupyter_code>import sys, os, random
import tensorflow as tf
from sklearn.model_selection import train_test_split

import logging
tf.get_logger().setLevel(logging.ERROR)
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '2'

# Set random seeds for repeatable results
RANDOM_SEED = 3
random.seed(RANDOM_SEED)
np.random.seed(RANDOM_SEED)
tf.random.set_seed(RANDOM_SEED)

classes_values = [ "bacteria", "normal", "virus" ]
classes = len(classes_values)

Y = tf.keras.utils.to_categorical(Y - 1, classes)

X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2, random_state=1)

input_length = X_train[0].shape[0]

train_dataset = tf.data.Dataset.from_tensor_slices((X_train, Y_train))
validation_dataset = tf.data.Dataset.from_tensor_slices((X_test, Y_test))

def set_batch_size(batch_size, train_dataset, validation_dataset):
    train_dataset = train_dataset.batch(batch_size, drop_remainder=True).prefetch(tf.data.experimental.AUTOTUNE)
    validation_dataset = validation_dataset.batch(batch_size, drop_remainder=True).prefetch(tf.data.experimental.AUTOTUNE)
    return train_dataset, validation_dataset

callbacks = []<jupyter_output><empty_output><jupyter_text>Train the model:<jupyter_code>import tensorflow as tf
from tensorflow.keras import Model
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, InputLayer, Dropout, Conv1D, Flatten, Reshape, MaxPooling1D, BatchNormalization, Conv2D, GlobalMaxPooling2D, Lambda
from tensorflow.keras.optimizers import Adam, Adadelta
from tensorflow.keras.losses import categorical_crossentropy

INPUT_SHAPE = (96, 96, 3)

base_model = tf.keras.applications.MobileNetV2(
    input_shape=INPUT_SHAPE, alpha=0.35,
    weights='./transfer-learning-weights/keras/mobilenet_v2_weights_tf_dim_ordering_tf_kernels_0.35_96.h5',
    include_top=True
)
base_model.trainable = False

model = Sequential()
model.add(InputLayer(input_shape=INPUT_SHAPE, name='x_input'))
model.add(Model(inputs=base_model.inputs, outputs=base_model.layers[-3].output))
model.add(Dense(16))
model.add(Dropout(0.1))
model.add(Flatten())
model.add(Dense(classes, activation='softmax'))

model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0005),
                loss='categorical_crossentropy',
                metrics=['accuracy'])

BATCH_SIZE = 32
train_dataset, validation_dataset = set_batch_size(BATCH_SIZE, train_dataset, validation_dataset)

# Set the data to the expected input shape
def reshape(image, label):
    return tf.reshape(image, INPUT_SHAPE), label
train_dataset = train_dataset.map(reshape, tf.data.experimental.AUTOTUNE)
validation_dataset = validation_dataset.map(reshape, tf.data.experimental.AUTOTUNE)
model.fit(train_dataset, validation_data=validation_dataset, epochs=10, verbose=2, callbacks=callbacks)

print('')
print('Initial training done.', flush=True)

# How many epochs we will fine tune the model
FINE_TUNE_EPOCHS = 10
# What percentage of the base model's layers we will fine tune
FINE_TUNE_PERCENTAGE = 65

print('Fine-tuning model for {} epochs...'.format(FINE_TUNE_EPOCHS), flush=True)

# Determine which layer to begin fine tuning at
model_layer_count = len(model.layers)
fine_tune_from = math.ceil(model_layer_count * ((100 - FINE_TUNE_PERCENTAGE) / 100))

# Allow the entire base model to be trained
model.trainable = True
# Freeze all the layers before the 'fine_tune_from' layer
for layer in model.layers[:fine_tune_from]:
    layer.trainable = False

model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.000045),
                loss='categorical_crossentropy',
                metrics=['accuracy'])

model.fit(train_dataset,
                epochs=FINE_TUNE_EPOCHS,
                verbose=2,
                validation_data=validation_dataset,
                callbacks=callbacks)

# Save the model to disk
model.save('saved_model')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11049949031600408, 'fraction_numerical': 0.013863404689092763, 'mean_word_length': 5.720547945205479, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '327121', 'license': 'MIT', 'n_tokens_mistral': 1738, 'n_tokens_neox': 1679, 'n_words': 350, 'path': 'transfer_learning/ei-pneumonia-detection-using-edgeml-transfer-learning.ipynb', 'repo_name': 'arijitdas123student/pneumonia-detection-edgeml', 'size': '4905'}