<jupyter_start><jupyter_text>Hybrid Fusion Multi-Modal Classifier<jupyter_code>import numpy as np
from collections import Counter
import pandas as pd
import os
import PIL
import sklearn
import pickle
from sklearn import model_selection, metrics
from matplotlib import pyplot as plt
import tensorflow
from tensorflow import keras
from utils import data_utils, model_utils, models<jupyter_output><empty_output><jupyter_text>Data Loading and Pre-processing<jupyter_code>label_map = label_map = {
    'geol_geow': 0, 
    'geol_sed': 1, 
    'gphys_gen': 2, 
    'log_sum': 3, 
    'pre_site': 4, 
    'vsp_file': 5
}
doc_data = data_utils.DocumentData(label_map, 2020, drop_nans='or')
doc_data.load_text_data()
doc_data.text_train.shape
doc_data.load_image_data(image_size=200, n_pages=10)
doc_data.image_train.shape
text_train = np.array([np.array([text] * 10) for text in doc_data.text_train])
text_val = np.array([np.array([text] * 10) for text in doc_data.text_val])
text_test = np.array([np.array([text] * 10) for text in doc_data.text_test])<jupyter_output><empty_output><jupyter_text>Hybrid Fusion Combining Image C-LSTM & Text CNN Classifier with Early Fusion C-LSTM  Input Layers<jupyter_code>late_fusion_text_input = keras.layers.Input(shape=2000)
early_fusion_text_input = keras.layers.Input(shape=(10, 2000))
image_input = keras.layers.Input(shape=(10, 200, 200, 1))<jupyter_output><empty_output><jupyter_text>Late fusion text learning pathway.Text feature extraction & classification:<jupyter_code>lf_embeddings = keras.layers.Embedding(doc_data.vocab_length, 150, input_length=2000)(late_fusion_text_input)
lf_conv_1d = keras.layers.Conv1D(filters=200, kernel_size=7, activation='relu')(lf_embeddings)
lf_global_pooling = keras.layers.GlobalMaxPool1D()(lf_conv_1d)
lf_flatten = keras.layers.Flatten()(lf_global_pooling)
lf_dense_layer = keras.layers.Dense(50, activation='relu', kernel_regularizer=keras.regularizers.l2(0.5))(lf_flatten)
lf_text_features = keras.layers.Dropout(0.3)(lf_dense_layer)<jupyter_output><empty_output><jupyter_text>Late fusion image learning pathway.Image feature extraction & classification:<jupyter_code>lf_conv_2d_1 = keras.layers.TimeDistributed(keras.layers.Conv2D(20, 7, activation='relu', padding='same'))(image_input)
lf_pool_2d_1 = keras.layers.TimeDistributed(keras.layers.MaxPooling2D(4))(lf_conv_2d_1)
lf_conv_2d_2 = keras.layers.TimeDistributed(keras.layers.Conv2D(50, 5, activation='relu', padding='valid'))(lf_pool_2d_1)
lf_pool_2d_2 = keras.layers.TimeDistributed(keras.layers.MaxPooling2D(4))(lf_conv_2d_2)
image_features = keras.layers.TimeDistributed(keras.layers.Flatten())(lf_pool_2d_2)
lf_lstm_1 = keras.layers.LSTM(1000, return_sequences=True)(image_features)
lf_image_features = keras.layers.LSTM(1000, dropout=0.5)(lf_lstm_1)<jupyter_output><empty_output><jupyter_text>Merge Late Fusion Output<jupyter_code>lf_merge_features = keras.layers.concatenate([lf_text_features, lf_image_features])
late_fusion_features = keras.layers.Flatten()(lf_merge_features)<jupyter_output><empty_output><jupyter_text>Early fusion learning pathway. Text modality feature extraction:<jupyter_code>ef_embeddings = keras.layers.TimeDistributed(keras.layers.Embedding(doc_data.vocab_length, 150, input_length=2000), name='word_embeddings')(early_fusion_text_input)
ef_conv_1d = keras.layers.TimeDistributed(keras.layers.Conv1D(filters=200, kernel_size=7, activation='relu'), name='1d_convolutional_layer')(ef_embeddings)
ef_global_pooling = keras.layers.TimeDistributed(keras.layers.GlobalMaxPool1D(), name='max_pooling_layer')(ef_conv_1d)
ef_text_features = keras.layers.TimeDistributed(keras.layers.Flatten(), name='text_features')(ef_global_pooling)<jupyter_output><empty_output><jupyter_text>Image modality feature extraction:<jupyter_code>ef_conv_2d_1 = keras.layers.TimeDistributed(keras.layers.Conv2D(20, 7, activation='relu', padding='same'))(image_input)
ef_pool_2d_1 = keras.layers.TimeDistributed(keras.layers.MaxPooling2D(4))(ef_conv_2d_1)
ef_conv_2d_2 = keras.layers.TimeDistributed(keras.layers.Conv2D(50, 5, activation='relu', padding='valid'))(ef_pool_2d_1)
ef_pool_2d_2 = keras.layers.TimeDistributed(keras.layers.MaxPooling2D(4))(ef_conv_2d_2)
ef_image_features = keras.layers.TimeDistributed(keras.layers.Flatten())(ef_pool_2d_2)<jupyter_output><empty_output><jupyter_text>Joint feature representation:<jupyter_code>ef_joint_features = keras.layers.concatenate([ef_text_features, ef_image_features])<jupyter_output><empty_output><jupyter_text>Classifier layers:<jupyter_code>ef_lstm_1 = keras.layers.LSTM(450, return_sequences=True)(ef_joint_features)
ef_lstm_2 = keras.layers.LSTM(1000)(ef_lstm_1)
early_fusion_features = keras.layers.Dropout(0.5)(ef_lstm_2)<jupyter_output><empty_output><jupyter_text>Classification Output.Concatenate early and late fusion output:<jupyter_code>hybrid_representation = keras.layers.concatenate([late_fusion_features, early_fusion_features])
output = keras.layers.Dense(6, activation='softmax')(hybrid_representation)<jupyter_output><empty_output><jupyter_text>Build and compile model:<jupyter_code>model = keras.models.Model(inputs=[late_fusion_text_input, early_fusion_text_input, image_input], outputs=[output])
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
model.summary()<jupyter_output><empty_output><jupyter_text>Train and evaluate model:<jupyter_code>early_stopping = keras.callbacks.EarlyStopping(monitor='val_loss', patience=2, mode='min', restore_best_weights=True)
history = model.fit(
    [doc_data.text_train, text_train, doc_data.image_train], 
    doc_data.y_train, 
    epochs=400, 
    validation_data=([doc_data.text_val, text_val, doc_data.image_val], doc_data.y_val), 
    callbacks=[early_stopping]
)
model.evaluate([doc_data.text_test, text_test, doc_data.image_test], doc_data.y_test)
y_hat = model.predict([doc_data.text_test, text_test, doc_data.image_test])
y = np.argmax(doc_data.y_test, axis=-1)
y_hat = np.argmax(y_hat, axis=-1)
labels = [label for label in label_map]
print(metrics.classification_report(y, y_hat, target_names=labels))
keras.utils.plot_model(model, show_layer_names=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11998706338939198, 'fraction_numerical': 0.02684346701164295, 'mean_word_length': 10.184448462929476, 'pattern_counts': {'":': 0, '<': 53, '<?xml version=': 0, '>': 53, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '617972', 'license': 'MIT', 'n_tokens_mistral': 2438, 'n_tokens_neox': 2273, 'n_words': 290, 'path': 'experimental_models/hybrid_fusion_model.ipynb', 'repo_name': 'justinbt1/Multimodal-Document-Classification', 'size': '6184'}