<jupyter_start><jupyter_text>Mount Drive<jupyter_code>from google.colab import drive
drive.mount('/content/drive', force_remount=True)<jupyter_output>Mounted at /content/drive<jupyter_text>Install packages<jupyter_code>!pip install -q transformers
# !pip install -q tensorflow==2.2-rc1
!pip install -q tf-models-official==2.2.0
!pip install keras-lr-multiplier<jupyter_output>Collecting keras-lr-multiplier
  Downloading https://files.pythonhosted.org/packages/7d/78/0eed4862a7274fb491b50881dd2f0dac996ff5774dc4a30c4b628fb78b25/keras-lr-multiplier-0.8.0.tar.gz
Requirement already satisfied: numpy in /usr/local/lib/python3.6/dist-packages (from keras-lr-multiplier) (1.19.5)
Requirement already satisfied: Keras in /usr/local/lib/python3.6/dist-packages (from keras-lr-multiplier) (2.4.3)
Requirement already satisfied: pyyaml in /usr/local/lib/python3.6/dist-packages (from Keras->keras-lr-multiplier) (3.13)
Requirement already satisfied: h5py in /usr/local/lib/python3.6/dist-packages (from Keras->keras-lr-multiplier) (2.10.0)
Requirement already satisfied: scipy>=0.14 in /usr/local/lib/python3.6/dist-packages (from Keras->keras-lr-multiplier) (1.4.1)
Requirement already satisfied: six in /usr/local/lib/python3.6/dist-packages (from h5py->Keras->keras-lr-multiplier) (1.15.0)
Building wheels for collected packages: keras-lr-multiplier
  Building wheel for keras-lr-m[...]<jupyter_text>Import libraries<jupyter_code>import os
import time
import datetime
import random
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
from tqdm import tqdm
from sklearn.metrics import confusion_matrix, classification_report, accuracy_score, roc_auc_score
from sklearn.preprocessing import LabelBinarizer
from sklearn.model_selection import train_test_split
from transformers import TFAutoModel, AutoTokenizer, TFBertForSequenceClassification,AutoConfig
import tensorflow as tf
from tensorflow.keras.utils import to_categorical
from tensorflow.keras.layers import Maximum,LayerNormalization,GlobalMaxPooling2D,Average,Dot, Dense, Input, GlobalAveragePooling1D, BatchNormalization, Activation, Concatenate, Flatten, Dropout, Conv1D, MaxPooling1D, Add, Lambda, GlobalAveragePooling2D, Reshape, RepeatVector, UpSampling1D 
from tensorflow.keras.models import Model
from keras.layers import LSTM, Bidirectional
from official import nlp
import official.nlp.optimization

tf.keras.backend.clear_session()
# Set seed value
seed_value = 56
os.environ['PYTHONHASHSEED']=str(0)
# 2. Set `python` built-in pseudo-random generator at a fixed value
random.seed(seed_value)
# 3. Set `numpy` pseudo-random generator at a fixed value
np.random.seed(seed_value)
# 4. Set `tensorflow` pseudo-random generator at a fixed value
tf.random.set_seed(seed_value)
# for later versions: 
# tf.compat.v1.set_random_seed(seed_value)
# 5. Configure a new global `tensorflow` session
from keras import backend as K
# session_conf = tf.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
# sess = tf.Session(graph=tf.get_default_graph(), config=session_conf)
# K.set_session(sess)
# for later versions:
session_conf = tf.compat.v1.ConfigProto(intra_op_parallelism_threads=1, inter_op_parallelism_threads=1)
sess = tf.compat.v1.Session(graph=tf.compat.v1.get_default_graph(), config=session_conf)
tf.compat.v1.keras.backend.set_session(sess)<jupyter_output><empty_output><jupyter_text>Parameters<jupyter_code>base_dir    = '/content/drive/MyDrive/mediaeval2016'
train_path  = os.path.join(base_dir, 'train-ekphrasis.csv')
val_path    = os.path.join(base_dir, 'dev-ekphrasis.csv')
test_path    = os.path.join(base_dir, 'test-ekphrasis.csv')
img_train_path = os.path.join(base_dir, 'train-img-224.npy')
img_val_path = os.path.join(base_dir, 'dev-img-224.npy')
img_test_path = os.path.join(base_dir, 'test-img-224.npy')
MAX_LENGTH  = 32
MODEL       = 'vinai/bertweet-base'
MODEL_NAME  = 'vinai/bertweet-base'
N_LABELS    = 1<jupyter_output><empty_output><jupyter_text>Read data<jupyter_code>df_train = pd.read_csv(train_path)
print(df_train.shape)
print(df_train.info())

display(df_train.head())
df_val = pd.read_csv(val_path)
print(df_val.shape)
print(df_val.info())

display(df_val.head())
df_test = pd.read_csv(test_path)
print(df_test.shape)
print(df_test.info())

display(df_test.head())
# Get the lists of sentences and their labels.
train_sent      = df_train.cleaned_text.values
train_labels    = df_train.enc_label.values
val_sent        = df_val.cleaned_text.values
val_labels      = df_val.enc_label.values 
test_sent        = df_test.cleaned_text.values
test_labels      = df_test.enc_label.values 

#Bertweet tokens
import re

for i in range(train_sent.shape[0]):
  train_sent[i] = re.sub(r'<url>','HTTPURL',train_sent[i])
  train_sent[i] = re.sub(r'<user>','@USER',train_sent[i])
for i in range(val_sent.shape[0]):
  val_sent[i] = re.sub(r'<url>','HTTPURL',val_sent[i])
  val_sent[i] = re.sub(r'<user>','@USER',val_sent[i])
for i in range(test_sent.shape[0]):
  test_sent[i] = re.sub(r'<url>','HTTPURL',test_sent[i])
  test_sent[i] = re.sub(r'<user>','@USER',test_sent[i])
train_sent = np.append(train_sent,val_sent)
train_labels = np.append(train_labels,val_labels)
#Image
img_train = np.load(img_train_path)
img_val = np.load(img_val_path)
img_test = np.load(img_test_path)
img_train.shape,  img_test.shape, img_val.shape
#Use both train and val to train like other papers
img_train = np.vstack((img_train,img_val))
img_train.shape
print(len(train_sent), len(train_labels))
print(len(val_sent), len(val_labels))
print(len(test_sent), len(test_labels))<jupyter_output>13762 13762
1386 1386
1117 1117<jupyter_text>Tokenization & Input Formatting<jupyter_code>!pip install emoji
# Load the BERT tokenizer.
print('Loading BERT tokenizer...')
tokenizer = AutoTokenizer.from_pretrained(MODEL, do_lower_case=False)
print(' Original: ', train_sent[4])
print('Tokenized: ', tokenizer.tokenize(train_sent[4]))
print('Token IDs: ', tokenizer.convert_tokens_to_ids(tokenizer.tokenize(train_sent[4])))<jupyter_output>Original:  <NAME> passed through New York and destroyed everything, or rather, almost everything. See this BBC photo. Virgin Mary . . . HTTPURL 
Tokenized:  ['Hurricane', 'Sandy', 'passed', 'through', 'New', 'York', 'and', 'destroyed', 'everything@@', ',', 'or', 'ra@@', 'ther@@', ',', 'almost', 'everything@@', '.', 'See', 'this', 'BBC', 'photo@@', '.', 'Virgin', 'Mary', '.', '.', '.', 'HTTPURL']
Token IDs:  [11623, 11741, 2262, 292, 210, 1092, 13, 5537, 42345, 7, 72, 1127, 3270, 7, 683, 42345, 4, 740, 33, 2674, 9844, 4, 11342, 3555, 4, 4, 4, 10]<jupyter_text>Find max length<jupyter_code>max_len = 0
stat = []
for sent in tqdm(train_sent):
    input_ids   = tokenizer.encode(sent, add_special_tokens=True)
#    if(len(input_ids)>96):
#      print(sent)
    max_len     = max(max_len, len(input_ids))
    stat.append(len(input_ids))

print('\nMax sentence length in train data: ', max_len)

# for sent in tqdm(val_sent):
#     input_ids   = tokenizer.encode(sent, add_special_tokens=True)
#  #   if(len(input_ids)>96):
#  #     print(sent)    
#     max_len     = max(max_len, len(input_ids))
#     stat.append(len(input_ids))

# print('\nMax sentence length in both train and val data: ', max_len)

for index,sent in tqdm(enumerate(test_sent)):
   input_ids   = tokenizer.encode(sent, add_special_tokens=True)
   if(len(input_ids)>96):
     print(index, sent)
   max_len     = max(max_len, len(input_ids))
   stat.append(len(input_ids))

print('\nMax sentence length in both train and val data: ', max_len)<jupyter_output>100%|██████████| 13762/13762 [00:01<00:00, 6951.94it/s]
1117it [00:00, 6148.61it/s]<jupyter_text>Tokenize   Train<jupyter_code>input_ids       = []
attention_masks = []

for sent in tqdm(train_sent):
    encoded_dict = tokenizer.encode_plus(
                        sent,                      
                        add_special_tokens = True, 
                        max_length = MAX_LENGTH,         
                        pad_to_max_length = True,
                        return_attention_mask = True,   
                        return_tensors = 'np',   
                        truncation = True,
                   )
    input_ids.append(encoded_dict['input_ids'])
    attention_masks.append(encoded_dict['attention_mask'])

  
id_train        = np.concatenate(input_ids)
mask_train      = np.concatenate(attention_masks)
y_train         = train_labels 
id_train.shape, mask_train.shape, y_train.shape<jupyter_output>0%|          | 0/13762 [00:00<?, ?it/s]/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2143: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).
  FutureWarning,
100%|██████████| 13762/13762 [00:01<00:00, 7031.45it/s]<jupyter_text>Val<jupyter_code>input_ids       = []
attention_masks = []

for sent in tqdm(val_sent):
    encoded_dict = tokenizer.encode_plus(
                        sent,                      
                        add_special_tokens = True, 
                        max_length = MAX_LENGTH,         
                        pad_to_max_length = True,
                        return_attention_mask = True,   
                        return_tensors = 'np',   
                        truncation = True,
                   )
    input_ids.append(encoded_dict['input_ids'])
    attention_masks.append(encoded_dict['attention_mask'])


id_val          = np.concatenate(input_ids)
mask_val        = np.concatenate(attention_masks)
y_val           = val_labels
id_val.shape, mask_val.shape, y_val.shape<jupyter_output>0%|          | 0/1386 [00:00<?, ?it/s]/usr/local/lib/python3.6/dist-packages/transformers/tokenization_utils_base.py:2143: FutureWarning: The `pad_to_max_length` argument is deprecated and will be removed in a future version, use `padding=True` or `padding='longest'` to pad to the longest sequence in the batch, or use `padding='max_length'` to pad to a max length. In this case, you can give a specific length with `max_length` (e.g. `max_length=45`) or leave max_length to None to pad to the maximal input size of the model (e.g. 512 for Bert).
  FutureWarning,
100%|██████████| 1386/1386 [00:00<00:00, 6899.43it/s]<jupyter_text>Create iterator for data<jupyter_code>BATCH_SIZE      = 256 

X_train         = [
    id_train,
    mask_train,
    img_train
]
X_val           = [
    id_val,
    mask_val,
    img_val  
]<jupyter_output><empty_output><jupyter_text>Train TFBertForSequenceClassification Model<jupyter_code>from tensorflow.keras.applications import VGG19                                  #Change here for different models
from tensorflow.keras.applications.vgg19 import preprocess_input                 #Change here for different models

def create_cnn(input_shape):
  inputs = Input(shape=input_shape)
  x = Lambda(preprocess_input)(inputs)
  # load the VGG16 network, ensuring the head FC layer sets are left
  # off                                                                           - Change here for different models
  baseModel = VGG19(weights="imagenet", include_top=True, input_tensor=x)

  # construct the head of the model that will be placed on top of the
  # the base model                                           
  headModel_cnn = baseModel.layers[22].output
  headModel_fc  = baseModel.layers[25].output
  ###Attention
  headModel_cnn = Reshape((-1,headModel_cnn.shape[-1]))(headModel_cnn)

  ###Non-attention
  # headModel = Flatten()(headModel)                                 
  # headModel = Dense(512)(headModel)
  # headModel = BatchNormalization()(headModel)
  # headModel = Activation("relu")(headModel)
  # headModel = Dropout(0.2)(headModel)

  # headModel = Dense(512)(headModel)
  # headModel = BatchNormalization()(headModel)
  # headModel = Activation("relu")(headModel)
  # headModel = Dropout(0.2)(headModel)

  model = Model(inputs=baseModel.input, outputs=[headModel_cnn, headModel_fc])

  # loop over all layers in the base model and freeze them so they will
  # *not* be updated during the first training process
  for layer in baseModel.layers:
    layer.trainable = False

  return model

def create_model(transformer, max_len=256):
    merge = []

    input_ids           = Input(shape=(max_len,), dtype=tf.int32, name='input_ids')
    attention_mask      = Input(shape=(max_len,), dtype=tf.int32, name='attention_mask')
    image               = create_cnn((224,224,3))
    sequence_output_1     = transformer(input_ids, attention_mask=attention_mask)[2][-1]
    sequence_output_2     = transformer(input_ids, attention_mask=attention_mask)[2][-2]
    sequence_output_3     = transformer(input_ids, attention_mask=attention_mask)[2][-3]
    sequence_output_4     = transformer(input_ids, attention_mask=attention_mask)[2][-4]
    sequence_output       = Concatenate(axis=2)([sequence_output_1, sequence_output_2, sequence_output_3, sequence_output_4])

   #cls_token           = sequence_output[:, 0, :]
    
    #merge.append(cls_token)
  
   # Yoon Kim model (https://arxiv.org/abs/1408.5882)
    convs = []
    filter_sizes = [2,3,4,5]
    size_pool = 3
    drop_rate = 0.2
    final_hid = 32

    for filter_size in filter_sizes:
        l_conv = Conv1D(filters=768, kernel_size=filter_size)(sequence_output)
        l_conv = BatchNormalization()(l_conv)
        l_conv = Activation('relu')(l_conv)
        # l_pool = MaxPooling1D(pool_size=max_len-filter_size+1)(l_conv)
        l_pool = MaxPooling1D(pool_size=size_pool)(l_conv)
        convs.append(l_pool)
        #merge.append(Flatten()(l_pool))

    l2_pool = Concatenate(axis=1)(convs)
    # l2_pool = BatchNormalization()(l2_pool)
    for _ in range(2):
        origin  = l2_pool
        l2_conv = Conv1D(filters=768, kernel_size=size_pool,padding='same')(l2_pool)
        l2_conv = BatchNormalization()(l2_conv)
        l2_conv = Activation('relu')(l2_conv)
        #print(origin.shape, l2_conv.shape)
        # l2_conv = Add()([Lambda(lambda x: x[0]*x[1])([origin,0.1]), l2_conv])
        l2_pool = MaxPooling1D(pool_size=size_pool)(l2_conv)

    text = Flatten()(l2_pool)
    #append to merge
    text_append = Dense(512)(text)
    text_append = BatchNormalization()(text_append)
    text_append = Activation('relu')(text_append)
    text_append = Dropout(drop_rate)(text_append)
    text_append = Dense(final_hid)(text_append)
    text_append = BatchNormalization()(text_append)
    text_append = Activation('relu')(text_append)

    # merge.append(text_append)
    # text_append = l2_pool

    #text for attention
    # text = Dropout(0.2)(text)
    # text = Dense(128)(text)
    # text = BatchNormalization()(text)
    # text = Activation("tanh")(text)                  #change to tanh?
    # text = Dropout(0.2)(text)
    image_append = Dense(2048)(image.output[1])
    image_append = BatchNormalization()(image_append)
    image_append = Activation('relu')(image_append)
    image_append = Dropout(drop_rate)(image_append)
    image_append = Dense(final_hid)(image_append)
    image_append = BatchNormalization()(image_append)
    image_append = Activation('relu')(image_append)

    img = Dense(final_hid)(image.output[0])
    img = BatchNormalization()(img)
    img = Activation('relu')(img)


    # image_append = image.output[1]
    # merge.append(image_append)
    # image_append = image.output[0]
    
    head = 1
    att_layers = 1
    att_hid = 32
    ##With attention - text on image
    inpAttImg_key = img
    inpAttImg_query = text_append
    for layer in range(1):
      att_img = []
      concat_key = []
      for _ in range(head):
        img_key = Dense(att_hid/head, use_bias=False)(inpAttImg_key) #change to tanh?
        text_query = Dense(att_hid/head, use_bias=False)(inpAttImg_query) 
        # img_key = BatchNormalization()(img_key)
        # img_key = Activation('tanh')(img_key)
        # text_query = BatchNormalization()(text_query)
        # text_query = Activation('tanh')(text_query)

        img_value = Dense(att_hid/head, use_bias=False)(inpAttImg_key)
        # img_value = BatchNormalization()(img_value)
        # img_value = Activation('tanh')(img_value)

        attention = Dot(axes=(1,2))([text_query, img_key])
        attention = Lambda(lambda x: x[0]/x[1])([attention,np.sqrt(att_hid/head)])
        attention = Activation("softmax")(attention)
        head_att_img = Dot(axes=(1,1))([attention, img_value])
        att_img = head_att_img
        concat_key = img_key
      # att_img = Concatenate(axis=1)(att_img)
      # att_img = Dense(512, use_bias=False)(att_img)
      # att_img = Dropout(0.3)(att_img)
      # att_img = Add()([att_img, inpAttImg_query])
      # att_img = LayerNormalization()(att_img)
      # att_img2 = Dense(1024,activation='relu')(att_img)
      # att_img2 = Dense(512)(att_img)
      # att_img2 = Dropout(0.1)(att_img2)
      # att_img = Add()([att_img, att_img2])
      # att_img = LayerNormalization()(att_img)
      att_img2 = Average()([ Activation('relu')(BatchNormalization()(Dense(final_hid)(att_img))),
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(att_img))),
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(att_img))), 
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(att_img)))])      
      att_img2 = Dense(final_hid)(att_img2)
      att_img2 = BatchNormalization()(att_img2)
      att_img2 = Activation('relu')(att_img2)
      att_img2 = Dropout(drop_rate)(att_img2)
      att_img = Add()([inpAttImg_query, att_img2])
      att_img = LayerNormalization()(att_img)



      # concat_key = Concatenate(axis=1)(concat_key)
      # concat_key = Dense(512)(concat_key)
      # concat_key = Dropout(0.3)(inpAttImg_key)
      # concat_key = Add()([concat_key, inpAttImg_key])
      # concat_key = LayerNormalization()(concat_key)
      # concat_key2 = Dense(1024,activation='relu')(concat_key)
      # concat_key2 = Dense(512)(concat_key)
      # concat_key2 = Dropout(0.1)(concat_key2)
      # concat_key = Add()([concat_key, concat_key2])
      # concat_key = LayerNormalization()(concat_key)

      inpAttImg_query = att_img
      inpAttImg_key = concat_key

    # merge.append(att_img)

    ##attention image on text
    inpAttText_key = l2_pool
    inpAttText_query = image_append
    for layer in range(1):
      att_text = []
      concat_key = []
      for _ in range(head):
        img_query = Dense(att_hid/head, use_bias=False)(inpAttText_query)
    #     # # img_query = BatchNormalization()(img_query)
    #     # # img_query = Activation('tanh')(img_query)
        text_key = Dense(att_hid/head, use_bias=False)(inpAttText_key)
    #     # # text_key = BatchNormalization()(text_key)
    #     # # text_key = Activation('tanh')(text_key)
        text_value = Dense(att_hid/head, use_bias=False)(inpAttText_key)
        attention = Dot(axes=(1,2))([img_query, text_key])
        attention = Lambda(lambda x: x[0]/x[1])([attention,np.sqrt(att_hid/head)])
        attention = Activation("softmax")(attention)
        head_att_text = Dot(axes=(1,1))([attention, text_value])
        att_text = head_att_text
        concat_key = text_key
      # att_text = Concatenate(axis=1)(att_text)
      # att_text = Dense(512, use_bias=False)(att_text)
      # att_text = Dropout(0.3)(att_text)
      # att_text = Add()([att_text, inpAttText_query])
      # att_text = LayerNormalization()(att_text)
      # att_text2 = Dense(1024,activation='relu')(att_text)
      # att_text2 = Dense(512)(att_text)
      # att_text2 = Dropout(0.1)(att_text2)
      # att_text = Add()([att_text, att_text2])
      # att_text = LayerNormalization()(att_text)

      # concat_key = Concatenate(axis=1)(concat_key)
      # concat_key = Dense(512)(concat_key)
      # concat_key = Dropout(0.3)(concat_key)
      # concat_key = Add()([concat_key, inpAttText_key])
      # concat_key = LayerNormalization()(concat_key)
      # concat_key2 = Dense(1024,activation='relu')(concat_key)
      # concat_key2 = Dense(512)(concat_key)
      # concat_key2 = Dropout(0.1)(concat_key2)
      # concat_key = Add()([concat_key, concat_key2])
      # concat_key = LayerNormalization()(concat_key)
      att_text2 = Average()([Activation('relu')(BatchNormalization()(Dense(final_hid)(att_text))),
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(att_text))),
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(att_text))), 
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(att_text)))])
      att_text2 = Dense(final_hid)(att_text2)
      att_text2 = BatchNormalization()(att_text2)
      att_text2 = Activation('relu')(att_text2)
      att_text2 = Dropout(drop_rate)(att_text2)
      att_text = Add()([inpAttText_query, att_text2])
      att_text = LayerNormalization()(att_text)

      inpAttText_query = att_text
      inpAttText_key = concat_key
    
    head = 1
    ###Self attention image
    inpAttSelf_key = img
    inpAttSelf_query = img
    for layer in range(att_layers):
      self_img = []
      for _ in range(head):
        self_query = Dense(att_hid/head, use_bias=False)(inpAttSelf_query)
        self_key = Dense(att_hid/head, use_bias=False)(inpAttSelf_key)
        self_value = Dense(att_hid/head, use_bias=False)(inpAttSelf_key)

        attention = Dot(axes=(2,2))([self_query, self_key])
        attention = Lambda(lambda x: x[0]/x[1])([attention,np.sqrt(att_hid/head)])
        attention = Activation("softmax")(attention)
        head_att_self = Dot(axes=(2,1))([attention, self_value])
        self_img = head_att_self

      # self_img = Concatenate(axis=2)(self_img)
      # self_img = Dense(512, use_bias=False)(self_img)
      # self_img = Dropout(0.3)(self_img)
      # self_img = Add()([self_img, inpAttSelf_query])
      # self_img = LayerNormalization()(self_img)
      # self_img2 = Dense(1024,activation='relu')(self_img)
      # self_img2 = Dense(512)(self_img2)
      # self_img2 = Dropout(0.3)(self_img2)
      # self_img = Add()([self_img, self_img2])
      # self_img = LayerNormalization()(self_img)
      self_img2 = Average()([Activation('relu')(BatchNormalization()(Dense(final_hid)(self_img))),
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(self_img))),
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(self_img))), 
                             Activation('relu')(BatchNormalization()(Dense(final_hid)(self_img)))])      
      self_img2 = Dense(final_hid)(self_img2)
      self_img2 = BatchNormalization()(self_img2)
      self_img2 = Activation('relu')(self_img2)
      self_img2 = Dropout(drop_rate)(self_img2)
      self_img = Add()([inpAttSelf_query, self_img2])
      self_img = LayerNormalization()(self_img)

      inpAttSelf_query = self_img
      inpAttSelf_key = self_img


    # merge.append(att_text)
    self_img =  Flatten()(self_img)
    # att_img =  Dense(32)(Flatten()(att_img))
    # # # att_img = BatchNormalization()(att_img)
    # att_img = Activation('relu')(att_img)

    # att_text = Flatten()(att_text)
    # # # att_text = Concatenate(axis=1)([att_text,self_img])
    # att_text = Dense(32)(att_text)
    # # # att_text = BatchNormalization()(att_text)
    # att_text = Activation('relu')(att_text)

    self_img =  Dense(final_hid)(self_img)
    self_img = BatchNormalization()(self_img)
    self_img = Activation('relu')(self_img)
    
    # self_img =  Dense(32)(self_img)
    # # # self_img = BatchNormalization()(self_img)
    # self_img = Activation('relu')(self_img)

    # text_append =  Dense(512)(Flatten()(text_append))
    # text_append = BatchNormalization()(text_append)
    # text_append = Activation('relu')(text_append)

    # image_append =  Dense(512)(Flatten()(image_append))
    # image_append = BatchNormalization()(image_append)
    # image_append = Activation('relu')(image_append)

    # merge.append(att_img)
    # merge.append(att_text)
    merge.append(self_img)

    # merge.append(Dense(512,activation='relu')(Flatten()(self_img)))
    # merge.append(text_append)
    merge.append(image_append)
    l_merge             = Concatenate(axis=1)(merge)
    # l_merge = image_append
    l_merge = Dropout(drop_rate)(l_merge)
    l_merge = Dense(final_hid)(l_merge)
    l_merge = BatchNormalization()(l_merge)
    l_merge = Activation('relu')(l_merge)
    l_merge = Dropout(drop_rate)(l_merge)
    # l_merge             = Average()(merge)   
    out                 = Dense(N_LABELS, activation='sigmoid')(l_merge) 
    model               = Model(inputs=[input_ids, attention_mask, image.input], 
                            outputs=out)
    return model
%%time
EPOCHS          = 8
total_steps     = len(y_train) * BATCH_SIZE
train_data_size = len(y_train)
steps_per_epoch = int(train_data_size / BATCH_SIZE) + 1
num_train_steps = steps_per_epoch * EPOCHS
# warmup_steps    = int(steps_per_epoch * 1)
warmup_steps    = 0

# Create the learning rate scheduler.
decay_schedule = tf.keras.optimizers.schedules.PolynomialDecay(
      initial_learning_rate=1e-4,
      decay_steps=num_train_steps,
      end_learning_rate=0)

warmup_schedule = nlp.optimization.WarmUp(
        initial_learning_rate=7e-5,
        decay_schedule_fn=decay_schedule,
        warmup_steps=warmup_steps)

optimizer       = nlp.optimization.AdamWeightDecay(
        learning_rate=warmup_schedule,
        epsilon=1e-8,
        exclude_from_weight_decay=['LayerNorm', 'layer_norm', 'bias'])

#Load bert4news
# config = AutoConfig.from_pretrained('/content/drive/MyDrive/bert4news/config.json')
# transformer = TFAutoModel.from_pretrained('/content/drive/MyDrive/bert4news/pytorch_model.bin', from_pt=True, config=config)

transformer = TFAutoModel.from_pretrained(MODEL,output_attentions=False,output_hidden_states=True,return_dict =True)
transformer.trainable = False
model = create_model(transformer, max_len=MAX_LENGTH)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=1e-4),
                loss='binary_crossentropy',
                metrics='accuracy')

model.summary()
from sklearn.utils import class_weight

class_weights = class_weight.compute_class_weight('balanced', np.unique(np.argmax(train_labels, axis=1)), np.argmax(train_labels, axis=1))
class_weights = {i : class_weights[i] for i in range(3)}
class_weights<jupyter_output><empty_output><jupyter_text>Train model<jupyter_code>from keras.utils import np_utils
from keras.callbacks import Callback, EarlyStopping
from sklearn.metrics import f1_score
from sklearn.datasets import make_classification

class roc_auc_callback(Callback):
    def __init__(self,training_data,validation_data):
        self.x = training_data[0]
        self.y = training_data[1]
        self.x_val = validation_data[0]
        self.y_val = validation_data[1]


    def on_train_begin(self, logs={}):
        return

    def on_train_end(self, logs={}):
        return

    def on_epoch_begin(self, epoch, logs={}):
        return

    def on_epoch_end(self, epoch, logs={}):
        # y_pred_train = self.model.predict(self.x, verbose=0)
        # y_pred_train = (y_pred_train>=0.5)*1
        # y_true_train = self.y
        # f1_train_micro = f1_score(y_true_train, y_pred_train, average='micro')
        # f1_train_macro = f1_score(y_true_train, y_pred_train, average='macro')
        
        y_pred_val = self.model.predict(self.x_val, verbose=0)
        y_pred_val = (y_pred_val>0.5)*1
        y_true_val = self.y_val
        f1_val_micro = f1_score(y_true_val, y_pred_val, average='micro')
        f1_val_macro = f1_score(y_true_val, y_pred_val, average='macro')
        print('\ f1_val: micro - %s macro - %s' % (str(round(f1_val_micro,4)),str(round(f1_val_macro,4))),end=100*' '+'\n')
        return

    def on_batch_begin(self, batch, logs={}):
        return

    def on_batch_end(self, batch, logs={}):
        return

n_steps = int(np.ceil(y_train.shape[0] / BATCH_SIZE))

# Checkpoint path
ckpt_path     = f'/content/drive/My Drive/mediaeval2016/checkpoint-test/{MODEL_NAME}/'
if not os.path.exists(ckpt_path):
    os.makedirs(ckpt_path)
ckpt_path     += 'cp-{epoch:02d}.h5'

# Callback
my_callbacks  = [tf.keras.callbacks.ModelCheckpoint(filepath=ckpt_path, 
                                                    monitor='val_loss', 
                                                    save_weights_only=True,
                                                    save_freq='epoch'),
                 tf.keras.callbacks.EarlyStopping(monitor='val_loss', patience=10, verbose=1),
                 roc_auc_callback(training_data=(X_train, y_train),validation_data=(X_test_data, test_labels))]

H = model.fit(
    X_train, y_train,
    validation_data=(X_test_data, test_labels),
    batch_size=BATCH_SIZE,
    epochs=EPOCHS,
    #steps_per_epoch=n_steps,
    # class_weight=class_weights,
    shuffle=False,
    callbacks=my_callbacks
)<jupyter_output>Epoch 1/8
54/54 [==============================] - 35s 596ms/step - loss: 0.3178 - accuracy: 0.8770 - val_loss: 0.5263 - val_accuracy: 0.7287
\ f1_val: micro - 0.7287 macro - 0.7172                                                                                                    
Epoch 2/8
54/54 [==============================] - 32s 588ms/step - loss: 0.1122 - accuracy: 0.9883 - val_loss: 0.5316 - val_accuracy: 0.6992
\ f1_val: micro - 0.6992 macro - 0.6926                                                                                                    
Epoch 3/8
54/54 [==============================] - 32s 587ms/step - loss: 0.0805 - accuracy: 0.9951 - val_loss: 0.5404 - val_accuracy: 0.7019
\ f1_val: micro - 0.7019 macro - 0.6974                                                                                                    
Epoch 4/8
54/54 [==============================] - 32s 591ms/step - loss: 0.0635 - accuracy: 0.9973 - val_loss: 0.5626 - val_accuracy: 0.6947
\ f1_val: mi[...]<jupyter_text>Saving Fine-Tuned Model<jupyter_code>model.save_weights(os.path.join(base_dir, f'model/{MODEL_NAME}.h5'))
EPOCHS = 20
%%time
EPOCHS          = EPOCHS
total_steps     = len(y_train) * BATCH_SIZE
train_data_size = len(y_train)
steps_per_epoch = int(train_data_size / BATCH_SIZE) + 1
num_train_steps = steps_per_epoch * EPOCHS
# warmup_steps    = int(num_train_steps * 0.1)
warmup_steps    = 0

# Create the learning rate scheduler.
decay_schedule = tf.keras.optimizers.schedules.PolynomialDecay(
      initial_learning_rate=2e-5,
      decay_steps=num_train_steps,
      end_learning_rate=0)

warmup_schedule = nlp.optimization.WarmUp(
        initial_learning_rate=2e-5,
        decay_schedule_fn=decay_schedule,
        warmup_steps=warmup_steps)

optimizer       = nlp.optimization.AdamWeightDecay(
        learning_rate=warmup_schedule,
        epsilon=1e-8,
        exclude_from_weight_decay=['LayerNorm', 'layer_norm', 'bias'])


#Load bert4news
# config = AutoConfig.from_pretrained('/content/drive/MyDrive/bert4news/config.json')
# transformer = TFAutoModel.from_pretrained('/content/drive/MyDrive/bert4news/pytorch_model.bin', from_pt=True, config=config)

transformer = TFAutoModel.from_pretrained(MODEL,output_attentions=False,output_hidden_states=True,return_dict =True)
transformer.trainable = False
model2      = create_model(transformer, max_len=MAX_LENGTH)
model2.compile(optimizer=optimizer,
                loss='binary_crossentropy',
                metrics='accuracy')
    
model2.summary()
model2.load_weights(os.path.join(base_dir, f'checkpoint-test/{MODEL_NAME}/cp-03.h5'))
# model2.evaluate(X_val,y_val)<jupyter_output><empty_output><jupyter_text>Predict<jupyter_code>input_ids       = []
attention_masks = []

for sent in tqdm(test_sent):
    encoded_dict = tokenizer.encode_plus(
                        sent,                      
                        add_special_tokens = True, 
                        max_length = MAX_LENGTH,         
                        pad_to_max_length = True,
                        return_attention_mask = True,   
                        return_tensors = 'np',   
                        truncation = True,
                   )
    input_ids.append(encoded_dict['input_ids'])
    attention_masks.append(encoded_dict['attention_mask'])

  
id_test_data        = np.concatenate(input_ids)
mask_test_data      = np.concatenate(attention_masks)
id_test_data.shape, mask_test_data.shape
X_test_data         = [
    id_test_data,
    mask_test_data,
    img_test
]
pred = model2.predict(X_test_data, verbose=1)
test_true = test_labels
test_true.shape, pred.shape
from sklearn import metrics
theta = 0.4
# Print the confusion matrix
print(metrics.confusion_matrix(test_true,  (pred>theta).astype(int)))

# Print the precision and recall, among other metrics

print(metrics.classification_report(test_true, (pred>theta).astype(int), digits=3))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12566217192391044, 'fraction_numerical': 0.028112208042379003, 'mean_word_length': 3.5241013071895426, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 69, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '56806', 'license': 'MIT', 'n_tokens_mistral': 12178, 'n_tokens_neox': 11535, 'n_words': 1878, 'path': 'mediaeval2016_model.ipynb', 'repo_name': 'dt024/RIVF2021_fakenews', 'size': '33233'}