<jupyter_start><jupyter_text>Uni-Modal Image Sequence Classifier Base Model<jupyter_code>import numpy as np
from collections import Counter
from sklearn import model_selection, metrics
import tensorflow
from tensorflow import keras
import pandas as pd
import os
import PIL
import sklearn
from matplotlib import pyplot as plt
from utils import data_utils, model_utils, models
tensorflow.config.list_physical_devices('GPU')<jupyter_output><empty_output><jupyter_text>Load & Pre-Process Data<jupyter_code>label_map = {
    'geol_geow': 0, 
    'geol_sed': 1, 
    'gphys_gen': 2, 
    'log_sum': 3, 
    'pre_site': 4, 
    'vsp_file': 5
}<jupyter_output><empty_output><jupyter_text>Page Image Classifier - CNN<jupyter_code>doc_data = data_utils.DocumentData(label_map, 2020, drop_nans='image')
doc_data.load_image_data(image_size=200, n_pages=1)
early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=2, mode='min', restore_best_weights=True)
cnn = models.image_cnn_model()
history = cnn.fit(doc_data.image_train, doc_data.y_train, epochs=400, validation_data=(doc_data.image_val, doc_data.y_val), callbacks=[early_stopping])
model_utils.plot_history(history, 'Image CNN')
cnn.evaluate(doc_data.image_test, doc_data.y_test)
y_probs = cnn.predict(doc_data.image_test)
y_hat = np.argmax(y_probs, axis=-1)
y = np.argmax(doc_data.y_test, axis=-1)
model_utils.confusion_matrix(y, y_hat, label_map, 'Page Image CNN Classifier Base Model')
labels = [label for label in label_map]
print(metrics.classification_report(y, y_hat, target_names=labels))
average_metrics = model_utils.iterate_training(models.image_cnn_model, doc_data, 10, y, data_type='image')
average_metrics<jupyter_output><empty_output><jupyter_text>Page Image Classifier - CNN (Multipage)  Load multi-page data<jupyter_code>doc_data = data_utils.DocumentData(label_map, 2020, drop_nans='image')
doc_data.load_image_data(image_size=200, n_pages=10, sequential=True)
y_train = doc_data.y_train
y_val = doc_data.y_val
y_test = doc_data.y_test

doc_data.y_train = model_utils.multi_y(doc_data.y_train)
doc_data.y_val = model_utils.multi_y(doc_data.y_val)
doc_data.y_test = model_utils.multi_y(doc_data.y_test)<jupyter_output><empty_output><jupyter_text>Build and train multi-page CNN model<jupyter_code>early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=2, mode='min', restore_best_weights=True)
cnn = models.image_cnn_model()
history = cnn.fit(doc_data.image_train, doc_data.y_train, epochs=400, validation_data=(doc_data.image_val, doc_data.y_val), callbacks=[early_stopping])
cnn.evaluate(doc_data.image_test, doc_data.y_test)
y_pred = cnn.predict(doc_data.image_test)
y_hat = np.argmax(y_pred, axis=-1)
y = np.argmax(doc_data.y_test, axis=-1)
labels = [label for label in label_map]
print(metrics.classification_report(y, y_hat, target_names=labels))
keras.utils.plot_model(cnn, show_layer_names=False)<jupyter_output><empty_output><jupyter_text>Majority Vote Method<jupyter_code>predicted_votes = model_utils.majority_vote(y_pred)
y = np.argmax(y_test, axis=-1)
model_utils.confusion_matrix(y, predicted_votes, label_map, 'Page Image CNN Multipage Classifier Base Model')
labels = [label for label in label_map]
print(metrics.classification_report(y, predicted_votes, target_names=labels))<jupyter_output>precision    recall  f1-score   support

   geol_geow       0.71      0.71      0.71       322
    geol_sed       0.88      0.41      0.56       203
   gphys_gen       0.83      0.19      0.31       208
     log_sum       0.28      0.96      0.44       208
    pre_site       0.83      0.25      0.39       231
    vsp_file       0.90      0.40      0.55       134

    accuracy                           0.51      1306
   macro avg       0.74      0.49      0.49      1306
weighted avg       0.73      0.51      0.51      1306<jupyter_text>ANN Aggregation<jupyter_code>train_probs = cnn.predict(doc_data.image_train)
train_probs = model_utils.aggregate_probabilities(train_probs)
predicted_val = cnn.predict(doc_data.image_val)
predicted_val = model_utils.aggregate_probabilities(predicted_val)
predicted_test = cnn.predict(doc_data.image_test)
predicted_test = model_utils.aggregate_probabilities(predicted_test)
input_layer = keras.layers.Input(shape=60)
dense1 = keras.layers.Dense(10, activation='relu')(input_layer)
output = keras.layers.Dense(6, activation='softmax')(dense1)

ann = keras.models.Model(inputs=[input_layer], outputs=[output])
ann.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
keras.utils.plot_model(ann, show_layer_names=False, rankdir='TB')
ann.fit(train_probs, y_train, epochs=400, validation_data=(predicted_val, y_val), callbacks=[early_stopping])
ann.evaluate(predicted_test, y_test)
y_probs = ann.predict(predicted_test)
y_hat = np.argmax(y_probs, axis=-1)
y = np.argmax(y_test, axis=-1)
model_utils.confusion_matrix(y, y_hat, label_map, 'Page Image ANN Classifier Base Model')
labels = [label for label in label_map]
print(metrics.classification_report(y, y_hat, target_names=labels))<jupyter_output>precision    recall  f1-score   support

   geol_geow       0.74      0.75      0.74       322
    geol_sed       0.72      0.68      0.70       203
   gphys_gen       0.74      0.73      0.73       208
     log_sum       0.58      0.71      0.64       208
    pre_site       0.78      0.71      0.74       231
    vsp_file       0.96      0.84      0.89       134

    accuracy                           0.73      1306
   macro avg       0.75      0.74      0.74      1306
weighted avg       0.74      0.73      0.73      1306<jupyter_text>Page Image Sequence Classifier - LSTM CNN<jupyter_code>doc_data = data_utils.DocumentData(label_map, 2020, drop_nans='image')
doc_data.load_image_data(image_size=200, n_pages=10)<jupyter_output><empty_output><jupyter_text>Build CNN LSTM page image sequence classifier<jupyter_code>early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=2, mode='min', restore_best_weights=True)
lstm_cnn = models.image_cnn_lstm_model()
history = lstm_cnn.fit(doc_data.image_train, doc_data.y_train, epochs=400, validation_data=(doc_data.image_val, doc_data.y_val), callbacks=[early_stopping])
lstm_cnn.evaluate(doc_data.image_test, doc_data.y_test)
model_utils.plot_history(history, 'Page Image Sequence Classifier Base Model')
y_probs = lstm_cnn.predict(doc_data.image_test)
y_hat = np.argmax(y_probs, axis=-1)
y = np.argmax(doc_data.y_test, axis=-1)
model_utils.confusion_matrix(y, y_hat, label_map, 'Page Image Sequence Classifier Base Model')
labels = [label for label in label_map]
print(metrics.classification_report(y, y_hat, target_names=labels))
cnn_lstm_metrics = model_utils.iterate_training(models.image_cnn_lstm_model, doc_data, 10, y, data_type='image')
cnn_lstm_metrics
keras.utils.plot_model(lstm_cnn, show_layer_names=False, rankdir='TB')
bi_cnn_lstm_metrics
bi_cnn_lstm_metrics = model_utils.iterate_training(models.image_cnn_lstm_model, doc_data, 10, y, data_type='image', model_params={'bi_directional': True})<jupyter_output>---- Training model iteration 0:
Epoch 1/100
131/131 - 43s - loss: 1.6859 - accuracy: 0.3388 - val_loss: 1.4483 - val_accuracy: 0.4526
Epoch 2/100
131/131 - 26s - loss: 1.4221 - accuracy: 0.4379 - val_loss: 1.2344 - val_accuracy: 0.5407
Epoch 3/100
131/131 - 25s - loss: 1.1422 - accuracy: 0.5660 - val_loss: 1.1176 - val_accuracy: 0.5904
Epoch 4/100
131/131 - 25s - loss: 0.9651 - accuracy: 0.6464 - val_loss: 1.0476 - val_accuracy: 0.6230
Epoch 5/100
131/131 - 25s - loss: 0.7873 - accuracy: 0.7163 - val_loss: 0.9779 - val_accuracy: 0.6794
Epoch 6/100
131/131 - 25s - loss: 0.6309 - accuracy: 0.7709 - val_loss: 0.9716 - val_accuracy: 0.6727
Epoch 7/100
131/131 - 25s - loss: 0.4681 - accuracy: 0.8372 - val_loss: 0.9500 - val_accuracy: 0.7024
Epoch 8/100
131/131 - 24s - loss: 0.3521 - accuracy: 0.8803 - val_loss: 1.0330 - val_accuracy: 0.7062
Epoch 9/100
131/131 - 26s - loss: 0.2630 - accuracy: 0.9093 - val_loss: 0.9923 - val_accuracy: 0.7349
41/41 [==============================] - 4s 91ms/[...]
{'fraction_non_alphanumeric': 0.11314908910645681, 'fraction_numerical': 0.07150824141777172, 'mean_word_length': 4.445344129554656, 'pattern_counts': {'":': 0, '<': 34, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '546521', 'license': 'MIT', 'n_tokens_mistral': 3391, 'n_tokens_neox': 3005, 'n_words': 568, 'path': 'experimental_models/image_base_model.ipynb', 'repo_name': 'justinbt1/Multimodal-Document-Classification', 'size': '8069'}