<jupyter_start><jupyter_text>Load and Pre-process data  load from pickle<jupyter_code>#full_data = pd.read_pickle('dataset_df.pickle')
full_data = pd.read_pickle('dataset_df_120x187.pickle')
#del full_data
full_data.head(2)<jupyter_output><empty_output><jupyter_text>get equal amounts of entries (benign, malignant)<jupyter_code>malignant_df = full_data.loc[full_data['benign_malignant'] == 'malignant', :]
benign_df = full_data.loc[full_data['benign_malignant'] == 'benign', :]

print(f"Total malignant entries: {str(malignant_df.shape[0])}")
print(f"Total benign entries: {str(benign_df.shape[0])}")

sample_malignant_df_df = benign_df.sample(n=malignant_df.shape[0])

dataset_df = malignant_df.append(sample_malignant_df_df)

benign_total = dataset_df.loc[dataset_df['benign_malignant'] == 'benign', :].shape[0]
malignant_total = dataset_df.loc[dataset_df['benign_malignant'] == 'malignant', :].shape[0]
print(f"Total dataset_df entries: {str(dataset_df.shape[0])} | malignan: {malignant_total} | benign: {benign_total}")<jupyter_output>Total malignant entries: 581
Total benign entries: 32120
Total dataset_df entries: 1162 | malignan: 581 | benign: 581<jupyter_text>set X and y<jupyter_code># set y
y = np.array(dataset_df['benign_malignant'])
# set X
X = np.array([entry for entry in dataset_df['image_data']])
X.size
print(y[11])
plt.imshow(X[11])
print(X[11].shape)
#tmp = np.array([entry for entry in X_train])
#tmp.reshape(24525, -1).shape<jupyter_output><empty_output><jupyter_text>Split data into training and test sets<jupyter_code>X_train, X_test, y_train, y_test = train_test_split(X, y, random_state=1000, stratify=y)
print(X_train[0].shape)
print(X_train.shape)
np.unique(y_train, return_counts=True)
train_label_encoder = LabelEncoder()
train_label_encoder.fit(y_train)
y_train_encoded = train_label_encoder.transform(y_train)

test_label_encoder = LabelEncoder()
test_label_encoder.fit(y_test)
y_test_encoded = test_label_encoder.transform(y_test)<jupyter_output><empty_output><jupyter_text>One hot encoding<jupyter_code>num_classes = 2
y_train_hencoded = to_categorical(y_train_encoded, num_classes)
y_test_hencoded = to_categorical(y_test_encoded, num_classes)
print(y_train_hencoded.shape)
print(y_test_hencoded.shape)<jupyter_output>(871, 2)
(291, 2)<jupyter_text>Build CNN model<jupyter_code>from tensorflow import keras
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from tensorflow.keras.layers.experimental.preprocessing import Resizing
from tensorflow.keras.layers.experimental.preprocessing import Rescaling
from tensorflow.keras.layers import Conv2D
from tensorflow.keras.layers import MaxPooling2D
from tensorflow.keras.layers import Flatten
from tensorflow.math import argmax<jupyter_output><empty_output><jupyter_text>Standarize the data<jupyter_code>print(X_train.shape)
#X_train.map(lambda x, y: ())
#X_train_norm = normalization_layer(X_train)
model = Sequential([
    Rescaling(1./255),
    Conv2D(X_train.shape[0], X_train.shape[-1], activation='relu'),
    MaxPooling2D(),
    
    Conv2D(X_train.shape[0], X_train.shape[-1], activation='relu'),
    MaxPooling2D(),
    
    Flatten(),
    #Dense(10, activation='relu'),
    Dense(units=len(train_label_encoder.classes_), activation='softmax')
])
# Compile the model
model.compile(optimizer='adam',
              loss='categorical_crossentropy',
              metrics=['accuracy'])
print(X_train.shape[-1])
print(y_train.shape[0])
# Fit the model to the training data
model.fit(
    X_train,
    y_train_hencoded,
    epochs=30,
    validation_split=0.1,
    shuffle=True,
    verbose=2
)
model.summary()<jupyter_output>Model: "sequential"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
rescaling (Rescaling)        (None, 120, 187, 3)       0         
_________________________________________________________________
conv2d (Conv2D)              (None, 118, 185, 871)     24388     
_________________________________________________________________
max_pooling2d (MaxPooling2D) (None, 59, 92, 871)       0         
_________________________________________________________________
conv2d_1 (Conv2D)            (None, 57, 90, 871)       6828640   
_________________________________________________________________
max_pooling2d_1 (MaxPooling2 (None, 28, 45, 871)       0         
_________________________________________________________________
flatten (Flatten)            (None, 1097460)           0         
________________________________________________________[...]<jupyter_text>Model Performance<jupyter_code># Evaluate the model using the training data
model_loss, model_accuracy = model.evaluate(X_test, y_test_hencoded, verbose=2)
print(f"Loss: {model_loss}, Accuracy: {model_accuracy}")<jupyter_output>10/10 - 2s - loss: 0.9225 - accuracy: 0.6838
Loss: 0.9225172400474548, Accuracy: 0.6838487982749939<jupyter_text>Validation<jupyter_code>import os
def validate(img_name):
    img = cv.imread(os.path.join('.', 'validation', img_name))
    # (240, 320, 3)
    #dim = (320, 240)
    width = X_train.shape[2]
    height = X_train.shape[1]
    dim = (width, height)
    new_img = cv.resize(img, dim)

    plt.imshow(new_img)

    x_prime = new_img.reshape((1, height, width, 3))
    x_prime.shape
    prediction = model.predict(x_prime)
    print(prediction[0])
    print( test_label_encoder.inverse_transform( argmax(prediction, axis=1) ) )
validate('ISIC_0463621-b.jpg')
validate('ISIC_0362853-b.jpg')
validate('ISIC_4012047-m.jpg')
validate('ISIC_4619903-m.jpg')
# Save the model
model.save("em_model-rgb.h5")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12711421098517872, 'fraction_numerical': 0.043243243243243246, 'mean_word_length': 6.038036809815951, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '475041', 'license': 'CC0-1.0', 'n_tokens_mistral': 2138, 'n_tokens_neox': 1956, 'n_words': 336, 'path': 'Resources/em_model-rgb.ipynb', 'repo_name': 'molobit619/isic-challenge', 'size': '5735'}