<jupyter_start><jupyter_text>print(df.head())<jupyter_code>train_df, val_df = train_test_split(df, test_size=0.033)

neg_train_samples = train_df[train_df.label == 0]
pos_train_samples = train_df[train_df.label == 1]

neg_val_samples = val_df[val_df.label == 0]
pos_val_samples = val_df[val_df.label == 1]

neg_train_samples = neg_train_samples.reset_index(drop=True)
pos_train_samples = pos_train_samples.reset_index(drop=True)

neg_val_samples = neg_val_samples.reset_index(drop=True)
pos_val_samples = pos_val_samples.reset_index(drop=True)


print("# Of Negative Training Samples : {}".format(len(neg_train_samples)))
print("# Of Positive Training Samples : {}".format(len(pos_train_samples)))
print("# Of Negative Validation Samples : {}".format(len(neg_val_samples)))
print("# Of Positive Validation Samples : {}".format(len(pos_val_samples)))
# Code here to move images into the appropriate directories as they start sitting in the train and test folders
# and need to be moved into train/0, train/1 and test/0, test/1

#[shutil.move(x, train_dir + "0/") for x in neg_train_samples['path']]
#[shutil.move(x, train_dir + "1/") for x in pos_train_samples['path']]
#[shutil.move(x, val_dir + "0/") for x in neg_val_samples['path']]
#[shutil.move(x, val_dir + "1/") for x in pos_val_samples['path']]

#for filename in os.listdir("./data/train/0/"):
#    if filename.endswith(".tif"):
#        shutil.move("./data/train/0/"+filename, train_dir)
#
#for filename in os.listdir("./data/train/1/"):
#    if filename.endswith(".tif"):
#        shutil.move("./data/train/1/"+filename, train_dir)
# Data augmentation
training_data_generator = ImageDataGenerator(rescale=1./255,
                                             horizontal_flip=True,
                                             vertical_flip=True,
                                             rotation_range=180,
                                             zoom_range=0.3, 
                                             width_shift_range=0.3,
                                             height_shift_range=0.3,
                                             shear_range=0.3,
                                             channel_shift_range=0.3)
# Data augmentation
val_data_generator = ImageDataGenerator(rescale=1./255)
# Data generation
training_generator = training_data_generator.flow_from_directory(train_dir,
                                                                 target_size=(IMAGE_SIZE,IMAGE_SIZE),
                                                                 batch_size=BATCH_SIZE,
                                                                 class_mode='binary')
validation_generator = val_data_generator.flow_from_directory(val_dir,
                                                              target_size=(IMAGE_SIZE,IMAGE_SIZE),
                                                              batch_size=BATCH_SIZE,
                                                              class_mode='binary')
# Model Ver. 1.0 - model_V_1

# Define what the input will look like
image_shape = (IMAGE_SIZE, IMAGE_SIZE, 3)

# Define the VGG16 model to use the imagenet weights without a top (note the input shape as image_shape)
vgg_conv = VGG16(weights='imagenet',
                  include_top=False,
                  input_shape=(image_shape))


# Create the model
model_V_1 = Sequential()
 
# Add the vgg convolutional base model
model_V_1.add(vgg_conv)
 
# Add new layers
model_V_1.add(Flatten())
model_V_1.add(Dense(512, activation='relu'))
model_V_1.add(Dropout(0.4))
model_V_1.add(Dense(1, activation='sigmoid'))
 
# Show a summary of the model. Check the number of trainable parameters
model_V_1.summary()

model_V_1.compile(optimizer=Adam(lr=0.0002, decay=0.00001),
              loss='binary_crossentropy',
              metrics=['accuracy'])
#  Training
history_V_1 = model_V_1.fit_generator(training_generator,
                                      steps_per_epoch=len(training_generator), 
                                      validation_data=validation_generator,
                                      validation_steps=len(validation_generator),
                                      epochs=EPOCHS,
                                      verbose=VERBOSITY)
plt.plot(history_V_1.history['loss'], color='blue', label="training_loss")
plt.plot(history_V_1.history['val_loss'], color='red', label="validation_loss")
plt.legend(loc='best')
plt.title('training')
plt.xlabel('epoch')
plt.plot(history_V_1.history['acc'], color='blue', label="training_accuracy")
plt.plot(history_V_1.history['val_acc'], color='red',label="validation_accuracy")
plt.legend(loc='best')
plt.title('validation')
plt.xlabel('epoch')
# ROC testing plot
predictions = model_V_1.predict_generator(validation_generator, steps=len(validation_generator), verbose=VERBOSITY)
false_positive_rate, true_positive_rate, threshold = roc_curve(validation_generator.classes, predictions)
area_under_curve = auc(false_positive_rate, true_positive_rate)

plt.plot([0, 1], [0, 1], 'k--')
plt.plot(false_positive_rate, true_positive_rate, label='AUC = {:.3f}'.format(area_under_curve))
plt.xlabel('False positive rate')
plt.ylabel('True positive rate')
plt.title('ROC curve')
plt.legend(loc='best')<jupyter_output>29/29 [==============================] - 4s 121ms/step
{'fraction_non_alphanumeric': 0.11864406779661017, 'fraction_numerical': 0.01770244821092279, 'mean_word_length': 2.5716207128446538, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 33, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '369515', 'license': 'MIT', 'n_tokens_mistral': 1621, 'n_tokens_neox': 1525, 'n_words': 285, 'path': 'histopathologic-cancer-detection-model.ipynb', 'repo_name': 'darien-schettler/histopathologic-cancer-detection', 'size': '5310'}