<jupyter_start><jupyter_text>Load dataset<jupyter_code>from data import CamVid
dataset = CamVid('datasets/CamVid')
TRAIN_LENGTH = dataset.num_train
BATCH_SIZE = 64
BUFFER_SIZE = 1000
STEPS_PER_EPOCH = TRAIN_LENGTH // BATCH_SIZE
train = dataset.train.map(load_image_train, num_parallel_calls=tf.data.experimental.AUTOTUNE)
val = dataset.val.map(load_image_test)
test = dataset.test.map(load_image_test)
train_dataset = train.cache().shuffle(BUFFER_SIZE).batch(BATCH_SIZE).repeat()
train_dataset = train_dataset.prefetch(buffer_size=tf.data.experimental.AUTOTUNE)
val_dataset = val.batch(BATCH_SIZE)
test_dataset = test.batch(BATCH_SIZE)
def display(display_list):
    plt.figure(figsize=(15, 15))

    title = ['Input Image', 'True Mask', 'Predicted Mask']

    for i in range(len(display_list)):
        plt.subplot(1, len(display_list), i+1)
        plt.title(title[i])
        plt.imshow(tf.keras.preprocessing.image.array_to_img(display_list[i]))
        plt.axis('off')
    plt.show()
for image, mask in dataset.train.take(1):
    sample_image, sample_mask = image, mask
    sample_mask = dataset.label_to_image(mask)
display([sample_image, sample_mask])<jupyter_output><empty_output><jupyter_text>Load model<jupyter_code>OUTPUT_CHANNELS = len(dataset.ID2LABEL)
pass<jupyter_output><empty_output><jupyter_text>model complie<jupyter_code>model.compile(optimizer='adam',
              loss=tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True),
              metrics=['accuracy'])
tf.keras.utils.plot_model(model, show_shapes=True)
def create_mask(pred_mask):
    pred_mask = tf.argmax(pred_mask, axis=-1)
    pred_mask = pred_mask[..., tf.newaxis]
    return pred_mask[0]
def show_predictions(dataset=None, num=1):
    if dataset:
        for image, mask in dataset.take(num):
            pred_mask = model.predict(image)
            display([image[0], dataset.label_to_image(mask[0]), dataset.label_to_image(create_mask(pred_mask))])
    else:
        display([sample_image, dataset.label_to_image(sample_mask),
                 dataset.label_to_image(create_mask(model.predict(sample_image[tf.newaxis, ...])))])
show_predictions()
class DisplayCallback(tf.keras.callbacks.Callback):
    def on_epoch_end(self, epoch, logs=None):
        clear_output(wait=True)
        show_predictions()
        print ('\nSample Prediction after epoch {}\n'.format(epoch+1))
EPOCHS = 20
VAL_SUBSPLITS = 5
VALIDATION_STEPS = dataset.num_val//BATCH_SIZE//VAL_SUBSPLITS

model_history = model.fit(train_dataset, epochs=EPOCHS,
                          steps_per_epoch=STEPS_PER_EPOCH,
                          validation_steps=VALIDATION_STEPS,
                          validation_data=val_dataset,
                          callbacks=[DisplayCallback()])
loss = model_history.history['loss']
val_loss = model_history.history['val_loss']

epochs = range(EPOCHS)

plt.figure()
plt.plot(epochs, loss, 'r', label='Training loss')
plt.plot(epochs, val_loss, 'bo', label='Validation loss')
plt.title('Training and Validation Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss Value')
plt.ylim([0, 1])
plt.legend()
plt.show()
show_predictions(dataset.test, 3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12661417322834645, 'fraction_numerical': 0.008188976377952756, 'mean_word_length': 4.5524475524475525, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '300805', 'license': 'MIT', 'n_tokens_mistral': 1121, 'n_tokens_neox': 1088, 'n_words': 161, 'path': 'main.ipynb', 'repo_name': 'haofengsiji/segmentation', 'size': '3175'}