<jupyter_start><jupyter_text>Model<jupyter_code>def get_class_weights(training_labels):
    class_weights = class_weight.compute_class_weight('balanced',np.unique(training_labels),training_labels)
    uni = list(np.unique(training_labels))

    labelset = ['AFTER', 'BEFORE', 'SIMULTANEOUS','INCLUDED_IN', 'INCLUDES', 'VAGUE']

    weights = []

    for i in labelset:
      idx = uni.index(i)
      weights.append(class_weights[idx])
      
    return weights
def defineModel(l1,l2,l3,l4,d1,out,d):

    embedding_layer1 = Embedding(len(word_index2) + 1,EMBEDDING_DIM,weights=[embedding_matrix2],input_length=MAX_SEQUENCE_LENGTH,trainable=False)
    embedding_layer2 = Embedding(len(word_index1) + 1,28,weights=[embedding_matrix1],input_length=MAX_SEQUENCE_LENGTH,trainable=False)
    embedding_layer3 = Embedding(len(word_index3) + 1,77,weights=[embedding_matrix3],input_length=MAX_SEQUENCE_LENGTH,trainable=False)

    wi = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32',name="inp1")
    wi2 = embedding_layer1(wi)

    pi_sen = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32',name="inp22")
    pi2_sen = embedding_layer2(pi_sen)

    di_sen = Input(shape=(MAX_SEQUENCE_LENGTH,), dtype='int32',name="inp33")
    di2_sen = embedding_layer3(di_sen)

    lstm1_sen = Bidirectional(LSTM(l1, activation='tanh', dropout=d, return_sequences=True), name = 'bid1temp_sen')(pi2_sen)  #  pos  features
    lstm2_sen = Bidirectional(LSTM(l2, activation='tanh', dropout=d, return_sequences=True), name= 'bid2temp_sen')(di2_sen)   #  dep features
    lstm3 = Bidirectional(LSTM(l4, activation='tanh', dropout=d+0.1, return_sequences=True), name = 'bid3')(wi2)  #  word features

    hid_sen = concatenate([lstm1_sen, lstm2_sen, lstm3])    
    
    lstm5 = Bidirectional(LSTM(l4, activation='tanh', dropout=d), name = 'bid3templstm2_sen')(hid_sen)

    yii = Dense(d1, activation='relu', name='dense1')(lstm5)
    drop1 = Dropout(d+0.05)(yii)
    yi = Dense(out, activation="softmax", name='dense2')(yii)
    model = Model(inputs=[pi_sen,di_sen,wi],outputs=yi)
    return model<jupyter_output><empty_output><jupyter_text>Train<jupyter_code>def trainModel():
    num_classes = 6

    epochs = 50
    batchsize = 64
    lr = 0.001
    file1 = 'TBDense/chkpt/'
    
    out = num_classes

    training_data, y_train, training_labels, val_data, y_val, val_labels = [seq11, seq13, seq12] , Y_train, labels_train, [seq11_val, seq13_val, seq12_val], Y_val, labels_val
    weights = get_class_weights(training_labels)

    set_nodes = [32, 32, 64, 64, 32, 0.30]
    l1 = set_nodes[0]
    l2 = set_nodes[1]
    l3 = set_nodes[2]
    l4 = set_nodes[3]
    d1 = set_nodes[4]
    d = set_nodes[5]
    optimizer = tf.keras.optimizers.Adam(learning_rate=lr)

    checkpoint_filepath = file1 + f'model'   
    model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath,save_weights_only=True,monitor='val_accuracy',mode='max',save_best_only=True)
    callback = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)

    model = defineModel(l1,l2,l3,l4,d1,out,d)
    model.compile(loss='categorical_crossentropy', optimizer=optimizer, metrics=['accuracy'], loss_weights=weights)
    model.fit(x = training_data, y = y_train, epochs = epochs, batch_size = batchsize,validation_data=(val_data,y_val), callbacks=[callback, model_checkpoint_callback], verbose=0)
    model.load_weights(checkpoint_filepath)
    return model
model = trainModel()
model.save("TBDense/model.h5")
batchsize = 64
lr = 0.001
epochs = 50

training_data, y_train, training_labels, val_data, y_val, val_labels = [seq11, seq13, seq12] , Y_train, labels_train, [seq11_val, seq13_val, seq12_val], Y_val, labels_val
weights = get_class_weights(training_labels)

file1 = 'TBDense/chkpt/'
checkpoint_filepath = file1 + f'model'
optimizer =  tf.keras.optimizers.Adam(learning_rate=lr)
model_checkpoint_callback = tf.keras.callbacks.ModelCheckpoint(filepath=checkpoint_filepath,save_weights_only=True,monitor='val_accuracy',mode='max',save_best_only=True)
callback = tf.keras.callbacks.EarlyStopping(monitor='val_accuracy', patience=20)
model.fit(x = training_data, y = y_train, epochs = epochs, batch_size = batchsize,validation_data=(val_data,y_val), callbacks=[callback,model_checkpoint_callback],verbose=0)<jupyter_output><empty_output><jupyter_text>Best Model Results<jupyter_code>def format_report(report, scores):
  # ['AFTER', 'BEFORE', 'SIMULTANEOUS','INCLUDED_IN', 'INCLUDES', 'VAGUE']

  print(f"              {'{0:>10}'.format('precision')} {'{0:>10}'.format('recall')} {'{0:>10}'.format('f1-score')}")
  print(f"        AFTER {'{0:>10}'.format(round(report['0']['precision']*100.0, 1))} {'{0:>10}'.format(round(report['0']['recall']*100.0, 1))} {'{0:>10}'.format(round(report['0']['f1-score']*100.0, 1))}")
  print(f"       BEFORE {'{0:>10}'.format(round(report['1']['precision']*100.0, 1))} {'{0:>10}'.format(round(report['1']['recall']*100.0, 1))} {'{0:>10}'.format(round(report['1']['f1-score']*100.0, 1))}")
  print(f" SIMULTANEOUS {'{0:>10}'.format(round(report['2']['precision']*100.0, 1))} {'{0:>10}'.format(round(report['2']['recall']*100.0, 1))} {'{0:>10}'.format(round(report['2']['f1-score']*100.0, 1))}")
  print(f"  INCLUDED_IN {'{0:>10}'.format(round(report['3']['precision']*100.0, 1))} {'{0:>10}'.format(round(report['3']['recall']*100.0, 1))} {'{0:>10}'.format(round(report['3']['f1-score']*100.0, 1))}")
  print(f"     INCLUDES {'{0:>10}'.format(round(report['4']['precision']*100.0, 1))} {'{0:>10}'.format(round(report['4']['recall']*100.0, 1))} {'{0:>10}'.format(round(report['4']['f1-score']*100.0, 1))}")
  print(f"        VAGUE {'{0:>10}'.format(round(report['5']['precision']*100.0, 1))} {'{0:>10}'.format(round(report['5']['recall']*100.0, 1))} {'{0:>10}'.format(round(report['5']['f1-score']*100.0, 1))}")
  print("")
  print(f"    micro avg {'{0:>10}'.format(round(scores[0]*100.0, 1))} {'{0:>10}'.format(round(scores[1]*100.0, 1))} {'{0:>10}'.format(round(scores[2]*100.0, 1))}")
data_test = [seq11_test,seq13_test,seq12_test]
model = defineModel(32,32,64,64,32,6,0.3)
model.load_weights('TBDense/model.h5', by_name=True)
classes = np.argmax(model.predict(x = data_test), axis=-1)
y_test_classes = Y_test.argmax(1)
y_pred_classes = classes

report = classification_report(y_true=y_test_classes, y_pred=y_pred_classes, zero_division=0, output_dict=True, digits= 3, labels=[0,1,2,3,4,5])
scores = precision_recall_fscore_support(y_true=y_test_classes, y_pred=y_pred_classes, average='micro', labels=[0,1,2,3,4,5])
format_report(report, scores)<jupyter_output>precision     recall   f1-score
        AFTER       52.1       20.9       29.8
       BEFORE       54.5       27.3       36.4
 SIMULTANEOUS       11.1        3.3        5.1
  INCLUDED_IN       17.1        7.6       10.5
     INCLUDES       15.4        5.3        7.8
        VAGUE       50.8       83.5       63.2

    micro avg       50.0       50.0       50.0
{'fraction_non_alphanumeric': 0.18261620377032667, 'fraction_numerical': 0.06446970787163621, 'mean_word_length': 5.568998109640832, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '525765', 'license': 'Apache-2.0', 'n_tokens_mistral': 2951, 'n_tokens_neox': 2727, 'n_words': 358, 'path': 'SERC-t-TBDense.ipynb', 'repo_name': 'kracr/Temporal-Causal-Relation-Classification', 'size': '6949'}