<jupyter_start><jupyter_text>Path<jupyter_code>#PATH
dataset_dev_path           = root_project + "dataset/haspeede2/preprocessed/dev/dev.csv"
dataset_test_tweets_path   = root_project + "dataset/haspeede2/preprocessed/reference/reference_tweets.csv"
#w2v_bin_path               = "/content/twitter128.bin"
w2v_bin_path               = root_project + 'results/model/word2vec/twitter128.bin'<jupyter_output><empty_output><jupyter_text>Embedding<jupyter_code>#load word2vec and embedding_matrix
w2v = KeyedVectors.load_word2vec_format(datapath(w2v_bin_path), binary=True)
index_to_key, key_to_index = get_index_key_association(w2v)
embedding_matrix, vocab_size = build_keras_embedding_matrix(w2v, index_to_key)

WORD_EMB_SIZE = 128
VOCAB_SIZE = vocab_size<jupyter_output>Vocab_size is 1170776
Embedding_matrix with unk word loaded
Shape (1170777, 128)<jupyter_text>DATA<jupyter_code>#load dataset dictionary
dataset_dev = load_csv_to_dict(dataset_dev_path)
dataset_test_tweets = load_csv_to_dict(dataset_test_tweets_path)
def load_data(dataset_dict, w2v, key_to_index, embedding_matrix, max_text_len):
    #TODO: deve ritornare anche tutto il resto, extra, lemma, stem, ...
    senteces = dataset_dict["tokens"]
    X = dataset_dict["tokens"]
    X = set_unkmark_token(X, w2v)
    X = get_int_seq(X, key_to_index)
    X = pad_sequences(X, maxlen=MAX_TEXT_LEN, padding='post', truncating='post')
    X = np.array(X)
    y = np.array(dataset_dict["stereotype"])
    return X, y

def to_emb(X):
    return np.array([ [ embedding_matrix[index_word] for index_word in sentence] for sentence in X])
#load dev/test
MAX_TEXT_LEN = 65

X, y = load_data(dataset_dev, w2v, key_to_index,embedding_matrix, MAX_TEXT_LEN)
X_e = to_emb(X)
X_test, y_test = load_data(dataset_test_tweets, w2v, key_to_index, embedding_matrix, MAX_TEXT_LEN)
X_test_e = to_emb(X_test)<jupyter_output><empty_output><jupyter_text>Split<jupyter_code>X_train, X_val, y_train, y_val = train_test_split(X_e, y, test_size=0.10, random_state=128)<jupyter_output><empty_output><jupyter_text>KIM-CNN<jupyter_code>def kim_cnn(input_shape, filters = 256, filter_sizes = [2,4,6], dropout = 0.5, hn=128, lr=0.01, print_model=False):
    import tensorflow as tf
    from tensorflow.keras.models import Model
    from tensorflow.keras.layers import Dropout
    from tensorflow.keras.layers import Conv1D, GlobalMaxPool1D
    from tensorflow.keras.layers import Dense, Input, Flatten, Activation
    from tensorflow.keras.layers import Concatenate
    from tensorflow.keras.regularizers import l2

    """ HyperParameters """

    text_seq_input = Input(shape=input_shape, name="text")

    convs = []
    for filter_size in filter_sizes:
        l_conv = Conv1D(filters=filters, kernel_size=filter_size)(text_seq_input)
        l_relu = Activation("relu")(l_conv)
        l_pool = GlobalMaxPool1D()(l_relu)   
        convs.append(l_pool)

    l_merge     = Concatenate(axis=1)(convs)

    l_flat      = Flatten()(l_merge)
    l_drop      = Dropout(dropout)(l_flat)
    l_hidden    = Dense(hn, activation='relu')(l_drop)
    l_drop      = Dropout(dropout)(l_hidden)
    l_out_st    = Dense(1, activation='sigmoid', name="st")(l_drop)  #dims output

    model_cnn   = Model(inputs=text_seq_input, outputs=l_out_st)
    if print_model:
        model_cnn.summary()
        tf.keras.utils.plot_model(model_cnn, "my_first_model.png", show_shapes=True)
    
    model_cnn.compile(
        loss= tf.keras.losses.BinaryCrossentropy(),#(from_logits=True),
        optimizer=tf.keras.optimizers.Adam(learning_rate=lr, amsgrad=False),
        metrics= [f1_macro]
    )

    return model_cnn
#del model
model = kim_cnn((65,128, ), filters = 256, filter_sizes = [2,4,6], dropout = 0.5, hn=128, lr=0.01, print_model=False)
model.compile(
    loss= tf.keras.losses.BinaryCrossentropy(from_logits=True),
    optimizer=tf.keras.optimizers.Adam(learning_rate=0.001, amsgrad=False),
    #optimizer = tf.keras.optimizers.SGD(learning_rate=0.07, momentum=0, nesterov=False),
    metrics=[f1_macro]
)
#mc = ModelCheckpoint('best_model.h5', monitor='val_f1_macro', mode='max', save_best_only=True, verbose=1)
#es = EarlyStopping(monitor="val_f1_macro", min_delta=0, patience=200, verbose=1, mode="max", restore_best_weights=False)
f1_callback = FCallback(validation = (X_test_e, y_test), verbose=True)                                   
best_callback = ReturnBestEarlyStopping(monitor="val_f1_macro", min_delta=0, patience=100, verbose=1, mode="max", restore_best_weights=True)

history = model.fit(X_train, y_train, batch_size=128, epochs=200, validation_data=(X_val, y_val), callbacks=[f1_callback, best_callback], verbose = 1)
#model_cnn = load_model('best_model.h5', custom_objects={'f1_macro': f1_macro})
y_val_pred = np.where(model.predict(X_val) >0.5,1,0)
print(f1_score(y_val, y_val_pred, average="macro"))
print(classification_report(y_val, y_val_pred))
y_test_pred = np.where(model.predict(X_test_e) > 0.5, 1, 0)
print(y_test.shape)
print(f1_score(y_test, y_test_pred, average="macro"))
print(classification_report(y_test, y_test_pred))<jupyter_output>(1263,)
0.6780973784652471
              precision    recall  f1-score   support

           0       0.78      0.58      0.67       694
           1       0.61      0.79      0.69       569

    accuracy                           0.68      1263
   macro avg       0.69      0.69      0.68      1263
weighted avg       0.70      0.68      0.68      1263<jupyter_text>|        	|        	|              	|    0   	|        	|        	|    1   	|        	|        	|          	||--------	|--------	|--------------	|:------:	|:------:	|:------:	|:------:	|:------:	|:------:	|:--------:	||        	|        	|              	|    P   	|    R   	|   F1   	|    P   	|    R   	|   F1   	| **Macro-F1** 	|| **TASK B** 	|  **News**  	| Baseline_MFC 	|   0,65 	|      1 	| 0,7878 	|      0 	|      0 	|      0 	|   0,3939 	||        	|        	| Baseline_SVC 	| 0,7467 	| 0,8707 	| 0,8039 	| 0,6528 	| 0,4514 	| 0,5337 	|   0,6688 	||        	| **Tweets** 	| Baseline_MFC 	| 0,5494 	|      1 	| 0,7092 	|      0 	|      0 	|      0 	|   0,3546 	||        	|        	| Baseline_SVC 	| 0,7869 	| 0,6599 	| 0,7178 	| 0,6534 	|  0,782 	|  0,712 	|   0,7149 	|<jupyter_code>print(history.history.keys())
# summarize history for accuracy

# summarize history for loss
plt.plot(history.history['loss'])
plt.plot(history.history['val_loss'])
plt.title('model loss')
plt.ylabel('loss')
plt.xlabel('epoch')
plt.legend(['train', 'test'], loc='upper left')
plt.show()<jupyter_output>dict_keys(['loss', 'f1_macro', 'val_loss', 'val_f1_macro', 'val_f1'])<jupyter_text>Result GridSearch<jupyter_code>import pickle

grid_result0 = pickle.load( open(root_project + "script/grid_search/kim_cnn/result_kim_cnn_0.p", "rb" ) )
grid_result1 = pickle.load( open(root_project + "script/grid_search/kim_cnn/result_kim_cnn_1.p", "rb" ) )
grid_result2 = pickle.load( open(root_project+  "script/grid_search/kim_cnn/result_kim_cnn_2.p", "rb" ) )
grid_result0[0]
grid_result_d = {
    'hyper_parm' : [],
    'mean': [],
    'std': [],
    'folds': []
}
for elem in grid_result0:
    for k in grid_result_d:
        grid_result_d[k].append(elem[k])
for elem in grid_result1:
    for k in grid_result_d:
        grid_result_d[k].append(elem[k])
for elem in grid_result2:
    for k in grid_result_d:
        grid_result_d[k].append(elem[k])
df_result = pd.DataFrame(grid_result_d)
pd.set_option('display.max_colwidth', None)
df_result.sort_values(by=['mean'], ascending=False)[0:20][["hyper_parm","mean", "std"]]
grid_result_final = pickle.load( open( root_project + "script/grid_search/kim_cnn/result_kim_cnn.p", "rb" ) )
grid_result_final[0]
df_result_final = pd.DataFrame(grid_result_final)
df_result_final_sorted = df_result_final.sort_values(by=['val_f1_macro_mean'], ascending=False)[0:3][["hyper_parm","loss_mean","loss_std","f1_macro_mean", "f1_macro_std","val_loss_mean","val_loss_std","val_f1_macro_mean","val_f1_macro_std"]]
df_result_final_sorted
print(df_result_final_sorted.to_latex(index=False))<jupyter_output>\begin{tabular}{lrrrrrrrr}
\toprule
                                                                          hyper\_parm &  loss\_mean &  loss\_std &  f1\_macro\_mean &  f1\_macro\_std &  val\_loss\_mean &  val\_loss\_std &  val\_f1\_macro\_mean &  val\_f1\_macro\_std \\
\midrule
 \{'filters': 256, 'filter\_sizes': [2, 3, 4], 'dropout': 0.5, 'hn': 64, 'lr': 0.0001\} &   0.376552 &  0.035755 &       0.852610 &      0.018248 &       0.560757 &      0.011836 &           0.702131 &          0.009217 \\
  \{'filters': 256, 'filter\_sizes': [2, 3, 4], 'dropout': 0.5, 'hn': 64, 'lr': 0.001\} &   0.383705 &  0.068300 &       0.852852 &      0.037940 &       0.571303 &      0.013317 &           0.699694 &          0.009102 \\
\{'filters': 256, 'filter\_sizes': [2, 3, 4], 'dropout': 0.5, 'hn': 124, 'lr': 0.0001\} &   0.380719 &  0.025508 &       0.849036 &      0.019435 &       0.562171 &      0.005032 &           0.696124 &          0.007857 \\
\bottomrule
\end{tabular}<jupyter_text>Retrain the final model and save<jupyter_code>model = kim_cnn((65,128, ), filters = 256, filter_sizes = [2,3,4], dropout = 0.4, hn=64, lr=0.0001, print_model=False)
f1_callback = FCallback(validation = (X_val, y_val), verbose=True)    
# gap_callback = GapCallback(train = (X_train, y_train), validation = (X_val, y_val), verbose=True)                               
best_callback = ReturnBestEarlyStopping(monitor="val_loss", min_delta=0, patience=50, verbose=1, mode="min", restore_best_weights=True)
history = model.fit(X_train, y_train, batch_size=128, epochs=200, validation_data=(X_val, y_val), callbacks=[f1_callback, best_callback], verbose = 2)
y_train_pred = np.where(model.predict(X_train) > 0.5,1,0)
print(f1_score(y_train, y_train_pred, average="macro"))

y_val_pred = np.where(model.predict(X_val) >0.5,1,0)
print(f1_score(y_val, y_val_pred, average="macro"))

y_test_pred = np.where(model.predict(X_test_e) > 0.5,1,0)
print(f1_score(y_test, y_test_pred, average="macro"))
plotHistory(history.history)
model.save(root_project + 'results/model/kcnn/kcnn.h5')
save_data_p(history.history, root_project + 'results/model/kcnn/kcnn_history.p')<jupyter_output><empty_output><jupyter_text>Final model<jupyter_code># Save the entire model as a SavedModel.
#!mkdir -p saved_model
#model.save('saved_model/kcnn')
#model = tf.keras.models.load_model('saved_model/kcnn')
dependencies = {
    'f1_macro': f1_macro
}
model = tf.keras.models.load_model(root_project + 'results/model/kcnn/kcnn.h5', custom_objects=dependencies)
model.compile(loss= tf.keras.losses.BinaryCrossentropy(), metrics= [f1_macro])
history = load_data_p(root_project + 'results/model/kcnn/kcnn_history.p')
model.evaluate(X_train, y_train)
model.evaluate(X_val, y_val)
model.evaluate(X_test_e, y_test)
print(history.keys())
y_train_pred = np.where(model.predict(X_train) > 0.5,1,0)
print(f1_score(y_train, y_train_pred, average="macro"))
print(classification_report(y_train, y_train_pred))
y_val_pred = np.where(model.predict(X_val) >0.5,1,0)
print(f1_score(y_val, y_val_pred, average="macro"))
print(classification_report(y_val, y_val_pred))
y_test_pred = np.where(model.predict(X_test_e) > 0.5,1,0)
print(f1_score(y_test, y_test_pred, average="macro"))
print(classification_report(y_test, y_test_pred))
plotHistory(history)
plotLoss(history)
plotF1_macro(history)
for sentence in dataset_test_tweets["processed_text"][10:15]:
    print(sentence)
    print()
y_test[10:15]
np.where(model.predict(X_test_e[10:15]) > 0.5,1,0)<jupyter_output><empty_output><jupyter_text>----------------------------------------------  ensemble predictions<jupyter_code>from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score
import scipy
cv_kfold = StratifiedKFold(n_splits=5, shuffle=True)

models = []

for train_index, test_index in cv_kfold.split(X_e, y):
  input_train_k = X_e[train_index]
  input_val_k   = X_e[test_index]
  y_train_k = y[train_index]
  y_val_k = y[test_index]

  model = kim_cnn((65,128, ), filters = 256, filter_sizes = [2,3,4], dropout = 0.4, hn=64, lr=0.0001, print_model=False)

  f1_callback = FCallback(validation = (input_val_k, y_val_k), verbose=False)    
  best_callback = ReturnBestEarlyStopping(monitor="val_loss", min_delta=0, patience=50, verbose=1, mode="min", restore_best_weights=True)

  history = model.fit(input_train_k, y_train_k, batch_size=128, epochs=200, validation_data=(input_val_k, y_val_k), callbacks=[f1_callback, best_callback], verbose = 0)
  
  y_val_pred_k = np.where(model.predict(input_val_k) >0.5,1,0)
  print(f1_score(y_val_k,y_val_pred_k,average="macro"))
  models.append(model)

def ensemble_predictions(members, X):
  # make predictions
  results = []
  yhats = [np.squeeze(np.where(models[0].predict(X) > 0.5, 1,0).reshape(1,-1)) for model in members]
  # sum across ensemble members
  yhats = np.array(yhats)
  print(yhats[:,0])
  print(yhats.shape)

  for i in range(yhats.shape[1]):
    counts = np.bincount(yhats[:,i])
    results.append(np.argmax(counts))
  # argmax across classes
  return results

def ensemble_predictions2(models, test):
  # Predict labels with models
  labels = []
  for m in models:
      predicts = np.argmax(m.predict(test), axis=1)
      labels.append(predicts)
      
  # Ensemble with voting
  labels = np.array(labels)
  labels = np.transpose(labels, (1, 0))
  labels = scipy.stats.mode(labels, axis=-1)[0]
  labels = np.squeeze(labels)
y_pred = ensemble_predictions(models, X_test_e)
print(classification_report(y_test,y_pred))<jupyter_output>[0 0 0 0 0]
(5, 1263)
              precision    recall  f1-score   support

           0       0.81      0.55      0.66       694
           1       0.61      0.84      0.70       569

    accuracy                           0.68      1263
   macro avg       0.71      0.70      0.68      1263
weighted avg       0.72      0.68      0.68      1263
{'fraction_non_alphanumeric': 0.1434105746466748, 'fraction_numerical': 0.0557428796900782, 'mean_word_length': 3.7511929107021134, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '402069', 'license': 'MIT', 'n_tokens_mistral': 5857, 'n_tokens_neox': 5400, 'n_words': 856, 'path': 'notebooks/ReStCnn/Kim_CNN.ipynb', 'repo_name': 'alessandrocuda/ReSt', 'size': '13939'}