<jupyter_start><jupyter_text><jupyter_code># # get extra dependencies
 
!pip install transformers
!wget http://nlp.stanford.edu/data/glove.6B.zip
!unzip glove.6B.zip -d embeddings
!rm glove.6B.zip
!rm embeddings/glove.6B.50d.txt
!rm embeddings/glove.6B.100d.txt
!rm embeddings/glove.6B.200d.txt
from google.colab import drive
drive.mount('/content/drive', force_remount=True)<jupyter_output>Mounted at /content/drive<jupyter_text>Importing<jupyter_code>import os
import pandas as pd
import time
import tensorflow as tf
import numpy as np
import pandas as pd
from tqdm.notebook import tqdm
from transformers import BertTokenizer, TFBertModel
from collections import namedtuple
from sklearn.model_selection import train_test_split
from sklearn import preprocessing
from keras.preprocessing.text import Tokenizer
from keras.layers.embeddings import Embedding
FOLDER_PATH = "/content/drive/MyDrive/CS60075-Team28-Task-1/"
DATA_FOLDER = os.path.join(FOLDER_PATH,"data/preprocessed")
# import evaluate function
import sys
sys.path.append(FOLDER_PATH)
from eval import evaluate
data = pd.read_csv(os.path.join(DATA_FOLDER, "lcp_single_train_preprocessed.csv"), index_col=0)
data['token'] = data['token'].astype(str)
data['sentence'] = data['sentence'].astype(str)

data.head()
data_multi = pd.read_csv(os.path.join(DATA_FOLDER, "lcp_multi_train_preprocessed.csv"), index_col=0)
data_multi['token'] = data_multi['token'].astype(str)
data_multi['sentence'] = data_multi['sentence'].astype(str)

data_multi.head()
data = pd.concat([data, data_multi])
print(len(data))<jupyter_output>9179<jupyter_text>Features - load data - only single data<jupyter_code>f1 = pd.read_csv(os.path.join(FOLDER_PATH, "data/extra_features/lcp_single_train_features.csv"), index_col=0)
f1['token'] = f1['token'].astype(str)
f1['sentence'] = f1['sentence'].astype(str)
f1.set_index("id", inplace=True)

# drop unwanted features
f1.drop(['parse', 'lemma'], axis=1, inplace=True)

print(f1.columns)

f2 = pd.read_csv(os.path.join(FOLDER_PATH, "data/added_corpus_presence/lcp_single_train_preprocessed.csv"), index_col=0)
f2['token'] = f2['token'].astype(str)
f2['sentence'] = f2['sentence'].astype(str)
print(f2.columns)

features = f1.merge(f2, on=['id','sentence', 'corpus', 'token', 'complexity'])
# fill pos nan by NN, as they are in majority
features['pos'] = features['pos'].fillna('NN')
features['token_length'] = features['token_length'].fillna(0)

# categorical encoding
labels = dict(features['pos'].value_counts())
labels = { k:i for i,k in enumerate(labels)}
labels['POS'] = len(labels)
print(labels)

features['pos'] = features['pos'].apply((lambda x: labels[x]))

def get_vowels(word):
    val = 0
    for w in word:
        if(w in ['A', 'a', 'E', 'e', 'I', 'i', 'O', 'o', 'U','u']):
            val+=1
    return val

features['token_vowels'] = features['token'].apply(lambda x: get_vowels(x) )


scaler = preprocessing.StandardScaler()
features[['token_length', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'familarity', 'token_vowels']] =  \
    scaler.fit_transform(features[['token_length', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'familarity', 'token_vowels']])

features.head()<jupyter_output>{'NN': 0, 'JJ': 1, 'NNP': 2, 'NNS': 3, 'VBG': 4, 'VB': 5, 'RB': 6, 'VBP': 7, 'UH': 8, 'CD': 9, 'VBN': 10, 'FW': 11, 'VBZ': 12, 'IN': 13, 'JJR': 14, 'VBD': 15, 'PRP': 16, 'GW': 17, 'WRB': 18, 'MD': 19, 'AFX': 20, 'JJS': 21, 'LS': 22, 'POS': 23}<jupyter_text>Splitting the Dataset into Test/Train<jupyter_code>train, test = train_test_split(features, test_size=0.1, random_state=12)
train.head()
sentences_train_list = list(train['sentence'])
complexity_train_list = list(train['complexity'])
tokens_train_list = list(train['token'])

sentences_test_list = list(test['sentence'])
complexity_test_list = list(test['complexity'])
tokens_test_list = list(test['token'])<jupyter_output><empty_output><jupyter_text>Getting embeddings (300 dimensional GloVe) and Preprocessing to I/P in Model<jupyter_code>def read_glove_vector(glove_vec):
  with open(glove_vec, 'r', encoding='UTF-8') as f:
    words = set()
    word_to_vec_map = {}
    for line in f:
      w_line = line.split()
      curr_word = w_line[0]
      word_to_vec_map[curr_word] = np.array(w_line[1:], dtype=np.float64)

  return word_to_vec_map

word_to_vec_map = read_glove_vector('embeddings/glove.6B.300d.txt')

# get embeddings, and pad till max_len
def get_embeddings(sentences, max_len=0):
    sent_emb = []
    for s in sentences:
        
        # fill unk by nan
        # calculate mean over non nan embeddings
        # fill unk by the mean embedding of sentence
        # pad 0 vectors till max_len

        temp_sent_emb = [ word_to_vec_map[x] if x in word_to_vec_map else np.full((300,), np.nan) for x in s.split() ]
        mean_emb = np.nanmean(np.array(temp_sent_emb), axis=0)

        temp_sent_emb = np.array([ mean_emb if np.isnan(x[0]) else x for x in temp_sent_emb ])
        temp_sent_emb = np.concatenate((temp_sent_emb, np.zeros((max_len-temp_sent_emb.shape[0],300))))

        sent_emb.append(temp_sent_emb)

    return np.array(sent_emb)
# max len for the sentence used for padding
max_len_sent = max(max([len(s.split()) for s in sentences_train_list]) , max([len(s.split()) for s in sentences_test_list])) 
print(max_len_sent)

max_len_token = max(max([len(s.split()) for s in tokens_train_list]) , max([len(s.split()) for s in tokens_test_list])) 
print(max_len_token)
train_sent_emb = get_embeddings(sentences_train_list, max_len_sent)
test_sent_emb = get_embeddings(sentences_test_list, max_len_sent)

print("sent emb shape train : {}".format(train_sent_emb.shape))
print("sent emb shape test : {}".format(test_sent_emb.shape))


# token_position_data
# for each token we should have [start index, lenght of token]

train_token_data =   [ 
                        [ 
                            len(s.split(t)[0].split()), 
                            len(t.split())
                        ]
                        for s,t in zip(sentences_train_list, tokens_train_list) 
                    ]

test_token_data =   [ 
                        [ 
                            len(s.split(t)[0].split()), 
                            len(t.split())
                        ]
                        for s,t in zip(sentences_test_list, tokens_test_list) 
                    ]

features_train = train[['token_length', 'token_vowels', 'syllables', 'pos', 'dep num', 
                        'synonyms', 'hypernyms', 'hyponyms', 
                        'google frequency', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

features_test = test[['token_length', 'token_vowels', 'syllables', 'pos', 'dep num', 
                        'synonyms', 'hypernyms', 'hyponyms', 
                        'google frequency', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values


# features_train = train[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
#                         'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
#                         'google frequency1', 'google frequency2', 
#                         'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

# features_test = test[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
#                         'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
#                         'google frequency1', 'google frequency2', 
#                         'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

print("feature emb shape train : {}".format(features_train.shape))
print("feature emb shape test : {}".format(features_test.shape))<jupyter_output>sent emb shape train : (6895, 123, 300)
sent emb shape test : (767, 123, 300)
feature emb shape train : (6895, 14)
feature emb shape test : (767, 14)<jupyter_text>Model Class<jupyter_code># -------------------------------------------------------------------------------------
# feed fwd neural network :
# 1 hidden size layer, 1 output_size layer  and a dropout between them
# used for calculating weights as well as final complexity prediction 
#-------------------------------------------------------------------------------------

class FFN(tf.keras.layers.Layer):
    def __init__(
        self, 
        hidden_size,
        output_size,
        rate,
        softmax=False,
    ):
        super(FFN, self).__init__()

        self.layer1 = tf.keras.layers.Dense(hidden_size, activation="relu")  # (batch_size, hidden_size)
        self.dropout = tf.keras.layers.Dropout(rate)
        self.layer2 = tf.keras.layers.Dense(output_size, activation= "softmax" if softmax is True else None)  # (batch_size, output_size)

    def call(self, x, training):
        return self.layer2(self.dropout(self.layer1(x), training=training) ) 
        # return self.layer2(self.dropout(x, training=training)) 
        
# -------------------------------------------------------------------------------------
# Custom Model for predicting complexity
#-------------------------------------------------------------------------------------

class OurModelBiLSTM(tf.keras.Model):
    def __init__(
        self,
        lstm_units,
        hidden_size,
        random_seed,
        seq_len,
        embedding_size,
        rate=0.25
        ):
        """
        hidden_size - for FFN
        """

        super(OurModelBiLSTM, self).__init__()

        tf.random.set_seed(random_seed)
        self.random_seed = random_seed
        self.lstm_units = lstm_units

        # self.dense1 = tf.keras.layers.Dense(2*self.lstm_units)
        self.bilstm = tf.keras.layers.Bidirectional(
                tf.keras.layers.LSTM(self.lstm_units, return_sequences=True), input_shape=(seq_len,embedding_size)
            )
        # self.bilstm = tf.keras.layers.LSTM(self.lstm_units, return_sequences=True)
        self.dropout = tf.keras.layers.Dropout(rate=rate)

        self.layernorm = tf.keras.layers.LayerNormalization()
        self.dense = tf.keras.layers.Dense(2*self.lstm_units)
        self.getWeights = FFN(hidden_size, 3, rate, softmax=True)
        self.final_layer = FFN(hidden_size, 1, rate)

    
    # one fwd pass on the model
    def call(self, input_seq, token_position, feature_emb, training=False):
        """
        input_seq : [batch_size, seq_len, embedding_size]
        token_position = [batch_size, 2] - [start,length] for tokens for which complexity is to be predicted
        """

        # bi-lstm pass
        bilstm_output = self.bilstm(input_seq)

        # bilstm_output = self.bilstm(input_seq)      #(batch_size, seq_len, 2*lstm_units)
        bilstm_output = self.dropout(bilstm_output, training=training)

        # extract token embeddings    
        # for each example we have start token and length
        # we take mean of these embeddings position

        token_emb = tf.stack(
                [tf.reduce_mean(tf.gather(i, tf.range(j[0],j[0]+j[1]), axis=0), axis=0) for i,j in zip(bilstm_output,token_position) ]
            )   # (batch_size, 2*lstm_units)

        # combine all embeddings - take mean
        added_emb = tf.reduce_mean(bilstm_output, axis=1)    # (batch_size, 2*lstm_units)
        
        # feature_emb - (batch_shape, features)
        feature_emb = self.dense(feature_emb)           # (batch_size, 2*lstm_units)
        # feature_emb = tf.cast(feature_emb, tf.float32)

        # get weights
        # weights = self.getWeights(self.layernorm(tf.add_n([token_emb,added_emb, feature_emb])), training)    # (batch_size, 3)
        weights = self.getWeights(self.layernorm(tf.concat([token_emb,added_emb, feature_emb], axis=1)), training)    # (batch_size, 3)

        # generate attenton-based final embeddings
        # final_emb = weights[0]*token_emb + weights[1]*added_emb + weights[3]*features_emb

        final_emb = tf.zeros(shape=(), dtype=tf.dtypes.float32)   # (bacth_size, 2*lstm_units)
        final_emb += tf.expand_dims(weights[:, 0], axis=1) * token_emb
        final_emb += tf.expand_dims(weights[:, 1], axis=1) * added_emb
        final_emb += tf.expand_dims(weights[:, 2], axis=1) * feature_emb

        # output complexity
        final_output = self.final_layer(final_emb, training)

        return final_output

    # loss function
    def loss_function(self, real, pred):
        loss_ = tf.keras.losses.MSE(real, pred)
        l = tf.reduce_mean(loss_)
        return l

    # set optimizer
    def compile(self, optimizer):
        super(OurModelBiLSTM, self).compile()
        self.optimizer = optimizer


    # train step - fwd pass + back prop + update model weights
    # @tf.function() - may not work beacuse zip is used
    def train_step(self, input_seq, token_position, y, feature_emb):
        training=True
        with tf.GradientTape() as tape:
            # Forward pass
            y_pred = self(input_seq, token_position, feature_emb, training)  
            loss = self.loss_function(tf.reshape(y,(-1,1)), y_pred)

        # calculate gradients
        gradients = tape.gradient(loss, self.trainable_variables)

        # update model weights using gradients
        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))

        return loss

    # fit funtion of model
    def _fit(
        self, 
        input_seq_data, 
        token_position_data, 
        feature_emb_data,
        complexity, 
        epochs, 
        batch_size,
        val_input_seq_data=None,
        val_token_position_data=None,
        val_feature_emb_data=None,
        val_complexity=None
        ):

        # create tensorflow dataset
        tf_data = tf.data.Dataset.from_tensor_slices((input_seq_data, token_position_data, feature_emb_data, complexity))

        # shuffle and batch
        tf_data = tf_data.shuffle(100000, seed=self.random_seed).batch(batch_size)

        train_loss = tf.keras.metrics.Mean(name="train_loss")
        loss_train = {"MSE": []}
        loss_val = {"MSE": []}

        #training starts
        for epoch in range(epochs):
            start = time.time()
            
            # reset state for mean loss
            train_loss.reset_states()

            # loop over batches
            for step, x in tf_data.enumerate():

                # get inputs for model
                input_seq = x[0]
                token_position = x[1]
                feature_emb = x[2]
                y = x[3]

                # call trainstep
                loss = self.train_step(input_seq, token_position, y, feature_emb)
                train_loss(loss)

            loss_train["MSE"].append(train_loss.result().numpy())
            print("Epoch {} loss  MSE: {}, time taken: {:.2f}s".format(epoch + 1, loss_train["MSE"][-1], time.time() - start))

            # validation if provided
            if (val_input_seq_data is not None):
                assert val_token_position_data is not None and val_complexity is not None, "incorrect validation data"
                val_pred = self._predict(val_input_seq_data, val_token_position_data, val_feature_emb_data)
                loss_val["MSE"].append(self.loss_function(tf.reshape(val_complexity, (-1,1)), val_pred).numpy())
                print("Validation loss MSE : {}".format(loss_val["MSE"][-1]))

            # SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/approach1/single")

            # if( not os.path.exists(SUBMISSION_FOLDER)):
            #     os.makedirs(SUBMISSION_FOLDER)

            # # predict 
            # pred = model._predict(single_sent_emb, single_token_data, features_single)
            # pred = pd.DataFrame({"ID":test_features.index, "complexity":pred.numpy().reshape(-1,)})
            # pred.to_csv(SUBMISSION_FOLDER+"/app1.csv", index=False, header=False)

            # evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_single_test_labelled_preprocessed.csv")

            
            # SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/approach1/multi")

            # if( not os.path.exists(SUBMISSION_FOLDER)):
            #     os.makedirs(SUBMISSION_FOLDER)

            # # predict 
            # pred = model._predict(multi_sent_emb, multi_token_data, features_multi)
            # pred = pd.DataFrame({"ID":test_multi_features.index, "complexity":pred.numpy().reshape(-1,)})
            # pred.to_csv(SUBMISSION_FOLDER+"/app1.csv", index=False, header=False)


            # evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_multi_test_labelled_preprocessed.csv")
        
        if (val_input_seq_data is not None):
            return loss_train, loss_val
        return loss_train

    
    # predict function
    def _predict(self, input_seq_data, token_position_data, feature_emb_data):
        
        # create tensorflow dataset
        tf_data = tf.data.Dataset.from_tensor_slices((input_seq_data, token_position_data, feature_emb_data))

        # batch for memory constraints
        tf_data = tf_data.batch(512)

        pred_list = []
        for step, x in tf_data.enumerate():
            # get inputs for model
            input_seq = x[0]
            token_position = x[1]
            feature_emb = x[2]

            pred_list.append(self(input_seq, token_position, feature_emb))
        return tf.concat(pred_list, axis=0)<jupyter_output><empty_output><jupyter_text>Prepare test data features<jupyter_code>test_f1 = pd.read_csv(os.path.join(FOLDER_PATH, "data/extra_features/lcp_single_test_features.csv"), index_col=0)
test_f1['token'] = test_f1['token'].astype(str)
test_f1['sentence'] = test_f1['sentence'].astype(str)
test_f1.set_index("id", inplace=True)

# drop unwanted features
test_f1.drop(['parse', 'lemma'], axis=1, inplace=True)

print(test_f1.columns)

test_f2 = pd.read_csv(os.path.join(FOLDER_PATH, "data/added_corpus_presence/lcp_single_test_preprocessed.csv"), index_col=0)
test_f2['token'] = test_f2['token'].astype(str)
test_f2['sentence'] = test_f2['sentence'].astype(str)
print(test_f2.columns)

test_features = test_f1.merge(test_f2, on=['id','sentence', 'corpus', 'token'])
# fill pos nan by NN, as they are in majority
test_features['pos'] = test_features['pos'].fillna('NN')
test_features['token_length'] = test_features['token_length'].fillna(0)

test_features['pos'] = test_features['pos'].apply((lambda x: labels[x]))

def get_vowels(word):
    val = 0
    for w in word:
        if(w in ['A', 'a', 'E', 'e', 'I', 'i', 'O', 'o', 'U','u']):
            val+=1
    return val

test_features['token_vowels'] = test_features['token'].apply(lambda x: get_vowels(x) )

test_features[['token_length', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'familarity', 'token_vowels']] = \
    scaler.transform(test_features[['token_length', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'familarity', 'token_vowels']])
# test_features['pos1'] = test_features['pos'].copy()
# test_features['pos2'] = test_features['pos'] 

# test_features['dep num1'] = test_features['dep num'] 
# test_features['dep num2'] = test_features['dep num'] 

# test_features['synonyms1'] = test_features['synonyms'] 
# test_features['synonyms2'] = test_features['synonyms'] 

# test_features['hypernyms1'] = test_features['hypernyms'] 
# test_features['hypernyms2'] = test_features['hypernyms'] 

# test_features['hyponyms1'] = test_features['hyponyms'] 
# test_features['hyponyms2'] = test_features['hyponyms'] 

# test_features['google frequency1'] = test_features['google frequency'] 
# test_features['google frequency2'] = test_features['google frequency'] 

# test_features.drop(['pos','dep num', 'synonyms', 'hyponyms', 'hypernyms', 'google frequency'], axis=1, inplace=True)


# test_features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']] =  \
#     scaler.transform(test_features[['token_length', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 'google frequency1', 'google frequency2', 'familarity', 'token_vowels']])

test_features.head()
single_sent_emb = get_embeddings(list(test_features['sentence']), max_len_sent)
# multi_sent_emb = get_embeddings(list(test_multi_features['sentence']), max_len_sent)

print("single emb shape train : {}".format(single_sent_emb.shape))
# print("multi emb shape test : {}".format(multi_sent_emb.shape))


# token_position_data
# for each token we should have [start index, lenght of token]

single_token_data =   [ 
                        [ 
                            len(s.split(t)[0].split()), 
                            len(t.split())
                        ]
                        for s,t in zip(list(test_features['sentence']), list(test_features['token'])) 
                    ]

# multi_token_data =   [ 
#                         [ 
#                             len(s.split(t)[0].split()), 
#                             len(t.split())
#                         ]
#                         for s,t in zip(list(test_multi_features['sentence']), list(test_multi_features['token'])) 
#                     ]

features_single = test_features[['token_length', 'token_vowels', 'syllables', 'pos', 'dep num', 'synonyms', 'hypernyms', 'hyponyms', 'google frequency', 'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']]
# features_single = test_features[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
#                         'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
#                         'google frequency1', 'google frequency2', 
#                         'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

# features_multi = test_multi_features[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
#                         'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
#                         'google frequency1', 'google frequency2', 
#                         'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

print("feature emb shape single : {}".format(features_single.shape))
# print("feature emb shape multi : {}".format(features_multi.shape))<jupyter_output>single emb shape train : (917, 123, 300)
feature emb shape single : (917, 14)<jupyter_text>Run model<jupyter_code>model = OurModelBiLSTM(lstm_units=20, hidden_size=10, random_seed=12, seq_len=max_len_sent, embedding_size=300, rate=0.4)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.005))

loss = model._fit ( train_sent_emb, 
                    train_token_data,
                    feature_emb_data = features_train,
                    complexity = complexity_train_list,
                    epochs=10, 
                    batch_size=32,
                    val_input_seq_data=test_sent_emb,
                    val_token_position_data = test_token_data,
                    val_feature_emb_data = features_test,
                    val_complexity = complexity_test_list
                )
SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/approach1/single")

if( not os.path.exists(SUBMISSION_FOLDER)):
    os.makedirs(SUBMISSION_FOLDER)

# predict 
pred = model._predict(single_sent_emb, single_token_data, features_single)
pred = pd.DataFrame({"ID":test_features.index, "complexity":pred.numpy().reshape(-1,)})
pred.to_csv(SUBMISSION_FOLDER+"/app1.csv", index=False, header=False)

evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_single_test_labelled_preprocessed.csv")
model.summary()<jupyter_output>Model: "our_model_bi_lstm_1"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
bidirectional_1 (Bidirection multiple                  51360     
_________________________________________________________________
dropout_3 (Dropout)          multiple                  0         
_________________________________________________________________
layer_normalization_1 (Layer multiple                  240       
_________________________________________________________________
dense_5 (Dense)              multiple                  600       
_________________________________________________________________
ffn_2 (FFN)                  multiple                  1243      
_________________________________________________________________
ffn_3 (FFN)                  multiple                  421       
===============================================[...]<jupyter_text>Commom Code - other experiments  sample test bilstm code<jupyter_code># ----------- test model output shape

sample_model = OurModelBiLSTM(lstm_units=2, hidden_size=2001, random_seed=12, seq_len=10, embedding_size=10)

#(bs, seq_len, embedding_shape)
input = tf.random.uniform((32, 10, 10), dtype=tf.float32, minval=0, maxval=200)

# (start, len)
token_position_data = tf.constant([(3,2) for _ in range(32)])

output = sample_model(input, token_position_data)
print(output.shape)

#  ----------- test fit and predict

sample_model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001))

input_seq_data = tf.random.uniform((100, 10, 10), dtype=tf.float32, minval=0, maxval=200)
token_position_data = [(3,5) for _ in range(100)]
y = tf.random.uniform((100, 1), dtype=tf.float32, minval=0, maxval=1)

val_input_seq_data = tf.random.uniform((100, 10, 10), dtype=tf.float32, minval=0, maxval=200)
val_token_position_data = [(3,5) for _ in range(100)]
val_y = tf.random.uniform((100, 1), dtype=tf.float32, minval=0, maxval=1)

loss = sample_model._fit( 
    input_seq_data = input_seq_data, 
    token_position_data = token_position_data, 
    complexity = y, 
    epochs = 10, 
    batch_size = 32,
    val_input_seq_data = val_input_seq_data,
    val_token_position_data = val_token_position_data,
    val_complexity=val_y
    )<jupyter_output><empty_output><jupyter_text>BERT CODE<jupyter_code># -------------------------------------------------------------------------------------
# feed fwd neural network :
# 1 hidden size layer, 1 output_size layer  and a dropout between them
# used for calculating weights as well as final complexity prediction 
#-------------------------------------------------------------------------------------

class FFN(tf.keras.layers.Layer):
    def __init__(
        self, 
        hidden_size,
        output_size,
        rate,
        softmax=False,
    ):
        super(FFN, self).__init__()

        self.layer1 = tf.keras.layers.Dense(hidden_size, activation="relu")  # (batch_size, hidden_size)
        self.dropout = tf.keras.layers.Dropout(rate)
        self.layer2 = tf.keras.layers.Dense(output_size, activation= "softmax" if softmax is True else None)  # (batch_size, output_size)

    def call(self, x, training):
        return self.layer2(self.dropout(self.layer1(x), training=training) ) 
        # return self.layer2(self.dropout(x, training=training)) 
        
# -------------------------------------------------------------------------------------
# Custom Model for predicting complexity
#-------------------------------------------------------------------------------------

class OurModelBert(tf.keras.Model):
    def __init__(
        self,
        hidden_size,
        random_seed,
        rate=0.25
        ):
        """
        hidden_size - for FFN
        """

        super(OurModelBert, self).__init__()

        tf.random.set_seed(random_seed)
        self.random_seed = random_seed

        self.dense = tf.keras.layers.Dense(768, activation="relu")
        self.dropout = tf.keras.layers.Dropout(rate=rate)
        self.getWeights = FFN(hidden_size, 3, rate, softmax=True)
        self.final_layer = FFN(hidden_size, 1, rate)

        self.layernorm = tf.keras.layers.LayerNormalization()

    
    # one fwd pass on the model
    def call(self, cls_emb, token_position, feature_emb, token_emb, training=False):
    

        # feature_emb = tf.concat([feature_emb, token_position], axis=1)
        feature_emb = self.dropout(self.dense(feature_emb), training=training)

        # get weights
        weights = self.getWeights(self.layernorm(tf.add_n([cls_emb, token_emb, feature_emb])), training)    # (batch_size, 3)

        # generate attenton-based final embeddings
        # final_emb = weights[0]*token_emb + weights[1]*added_emb + weights[3]*features_emb

        final_emb = tf.zeros(shape=(), dtype=tf.dtypes.float32)   # (bacth_size, 2*lstm_units)
        final_emb += tf.expand_dims(weights[:, 0], axis=1) * cls_emb
        final_emb += tf.expand_dims(weights[:, 1], axis=1) * token_emb
        final_emb += tf.expand_dims(weights[:, 2], axis=1) * feature_emb

        # output complexity
        final_output = self.final_layer(final_emb, training)

        return final_output

    # loss function
    def loss_function(self, real, pred):
        loss_ = tf.keras.losses.MSE(real, pred)
        l = tf.reduce_mean(loss_)
        return l

    # set optimizer
    def compile(self, optimizer):
        super(OurModelBert, self).compile()
        self.optimizer = optimizer


    # train step - fwd pass + back prop + update model weights
    # @tf.function() - may not work beacuse zip is used
    def train_step(self, cls_emb, token_position, feature_emb, token_emb, y):
        training=True
        with tf.GradientTape() as tape:
            # Forward pass
            y_pred = self(cls_emb, token_position, feature_emb, token_emb, training)  
            loss = self.loss_function(tf.reshape(y,(-1,1)), y_pred)

        # calculate gradients
        gradients = tape.gradient(loss, self.trainable_variables)

        # update model weights using gradients
        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))

        return loss

    # fit funtion of model
    def _fit(
        self, 
        cls_emb, 
        token_emb,
        token_position_data, 
        feature_emb,
        complexity, 
        epochs, 
        batch_size,
        val_cls_emb=None, 
        val_token_emb=None,
        val_token_position_data=None, 
        val_feature_emb=None,
        val_complexity=None, 
        ):

        # create tensorflow dataset
        tf_data = tf.data.Dataset.from_tensor_slices((cls_emb, token_emb, token_position_data, feature_emb, complexity))

        # shuffle and batch
        tf_data = tf_data.shuffle(100000, seed=self.random_seed).batch(batch_size)

        train_loss = tf.keras.metrics.Mean(name="train_loss")
        loss_train = {"MSE": []}
        loss_val = {"MSE": []}

        #training starts
        for epoch in range(epochs):
            start = time.time()
            
            # reset state for mean loss
            train_loss.reset_states()

            # loop over batches
            for step, x in tf_data.enumerate():

                # get inputs for model
                cls_emb = x[0] 
                token_emb = x[1]
                token_position = x[2] 
                feature_emb = x[3]
                complexity = x[4]
                
                # call trainstep
                loss = self.train_step(cls_emb, token_position, feature_emb, token_emb, complexity)
                train_loss(loss)

            loss_train["MSE"].append(train_loss.result().numpy())
            print("Epoch {} loss  MSE: {}, time taken: {:.2f}s".format(epoch + 1, loss_train["MSE"][-1], time.time() - start))

            # validation if provided
            # if (val_input_seq_data is not None):
            #     assert val_token_position_data is not None and val_complexity is not None, "incorrect validation data"
            #     val_pred = self._predict(val_input_seq_data, val_token_position_data, val_feature_emb_data)
            #     loss_val["MSE"].append(self.loss_function(tf.reshape(val_complexity, (-1,1)), val_pred).numpy())
            #     print("Validation loss MSE : {}".format(loss_val["MSE"][-1]))
        
        if (val_input_seq_data is not None):
            return loss_train, loss_val
        return loss_train

    
    # predict function
    def _predict(self, input_seq_data, token_position_data, feature_emb_data):
        # create tensorflow dataset
        tf_data = tf.data.Dataset.from_tensor_slices((input_seq_data, token_position_data, feature_emb_data))

        # batch for memory constraints
        tf_data = tf_data.batch(512)

        pred_list = []
        for step, x in tf_data.enumerate():
            # get inputs for model
            input_seq = x[0]
            token_position = x[1]
            feature_emb = x[2]

            pred_list.append(self(input_seq, token_position, feature_emb))
        return tf.concat(pred_list, axis=0)
model = OurModelBert(hidden_size=256, random_seed=12, rate=0.3)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.005))
features_train = features[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
                        'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
                        'google frequency1', 'google frequency2', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values
features_train.shape
loss = model._fit ( cls_emb, 
                    token_emb,
                    token_position_data, 
                    feature_emb=features_train,
                    complexity=features['complexity'], 
                    epochs=50, 
                    batch_size=32,
                )
model.summary()<jupyter_output>Model: "our_model_bert_2"
_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
dense_10 (Dense)             multiple                  16128     
_________________________________________________________________
dropout_80 (Dropout)         multiple                  0         
_________________________________________________________________
ffn_4 (FFN)                  multiple                  590851    
_________________________________________________________________
ffn_5 (FFN)                  multiple                  197121    
=================================================================
Total params: 804,100
Trainable params: 804,100
Non-trainable params: 0
_________________________________________________________________<jupyter_text>Get Bert Embeddings<jupyter_code># # Generated here and saved once, because it gives OOM error when less space is avialable
# # Thus token_position_data and bert_emb are saved once

# bert = TFBertModel.from_pretrained("bert-base-cased")
# tokenizer = BertTokenizer.from_pretrained("bert-base-cased")

# # get maximum padding length
# tokenized_output_data = tokenizer.batch_encode_plus(
#             list(features['sentence']),
#             add_special_tokens=True, 
#             padding='longest', 
#             return_tensors="tf"
#         )
# max_len = tokenized_output_data['input_ids'].shape[1]

# sentences, tokens = list(features['sentence']), list(features['token'])

# cls_emb = []
# token_position_data = []
# token_emb = []

# #  get embeddings one by one
# for i in tqdm(range(len(sentences))):
#     tok = tokenizer.encode_plus(
#         sentences[i],
#         add_special_tokens=True, 
#         padding='max_length',
#         max_length = max_len,
#         return_tensors="tf"
#     )
#     entire = bert(**tok, return_dict=False)[0]
#     cls_emb.append(entire[:,0,:])
#     token_position_data.append( [ 
#                                     len(tokenizer.encode(sentences[i].split(tokens[i])[0], add_special_tokens=False)) + 1, 
#                                     len(tokenizer.encode(tokens[i], add_special_tokens=False))
#                                 ]
#                                )
#     token_emb.append(tf.reduce_mean(tf.gather(entire, tf.range(token_position_data[-1][0],token_position_data[-1][0]+token_position_data[-1][1]), axis=1), axis=1))

# cls_emb = np.squeeze(np.array(cls_emb), axis=1)
# token_emb = np.squeeze(np.array(token_emb), axis=1)
# token_position_data = np.array(token_position_data)

# print(cls_emb.shape)
# print(token_emb.shape)
# print(token_position_data.shape)

# np.save(FOLDER_PATH+"data/cls_emb.npy", cls_emb)
# np.save(FOLDER_PATH+"data/token_position_data_bert.npy", np.array(token_position_data))
# np.save(FOLDER_PATH+"data/token_emb_bert.npy", token_emb)<jupyter_output>Some layers from the model checkpoint at bert-base-cased were not used when initializing TFBertModel: ['mlm___cls', 'nsp___cls']
- This IS expected if you are initializing TFBertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing TFBertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).
All the layers of TFBertModel were initialized from the model checkpoint at bert-base-cased.
If your task is similar to the task the model of the checkpoint was trained on, you can already use TFBertModel for predictions without further training.<jupyter_text>Load saved bert embeddings<jupyter_code>cls_emb = np.load(FOLDER_PATH+"data/cls_emb.npy")
token_position_data = np.load(FOLDER_PATH+"data/token_position_data_bert.npy")
token_emb = np.load(FOLDER_PATH+"data/token_emb_bert.npy")
sorted(token_position_data[:,0])[-15:]<jupyter_output><empty_output><jupyter_text>Bert code - 2<jupyter_code># -------------------------------------------------------------------------------------
# feed fwd neural network :
# 1 hidden size layer, 1 output_size layer  and a dropout between them
# used for calculating weights as well as final complexity prediction 
#-------------------------------------------------------------------------------------

class FFN(tf.keras.layers.Layer):
    def __init__(
        self, 
        hidden_size,
        output_size,
        rate,
        softmax=False,
    ):
        super(FFN, self).__init__()

        self.layer1 = tf.keras.layers.Dense(hidden_size, activation="relu")  # (batch_size, hidden_size)
        self.dropout = tf.keras.layers.Dropout(rate)
        self.layer2 = tf.keras.layers.Dense(output_size, activation= "softmax" if softmax is True else None)  # (batch_size, output_size)

    def call(self, x, training):
        return self.layer2(self.dropout(self.layer1(x), training=training) ) 
        # return self.layer2(self.dropout(x, training=training)) 

# -------------------------------------------------------------------------------------
# Custom Model for predicting complexity, using BERT
# warning - for pooler gradients not being updated
# because we're not using the pooler, therefore it doesnt get updated
#-------------------------------------------------------------------------------------
class OurModelBert(tf.keras.Model):
    def __init__(
        self,
        bert_model,
        hidden_size,
        random_seed,
        rate=0.25
        ):
        """
        bert_model - model name or a path to saved model
        hidden_size - for FFN
        """

        super(OurModelBert, self).__init__()
        tf.random.set_seed(random_seed)
        self.random_seed = random_seed
        
        self.tokenizer = BertTokenizer.from_pretrained(bert_model)
        self.bert = TFBertModel.from_pretrained(bert_model)
        
        # make it non trainable
        self.bert.trainable = False
        # self.dense = tf.keras.layers.Dense(768, activation='relu')
        self.getWeights = FFN(hidden_size, 2, rate, True)
        self.final_layer = FFN(hidden_size, 1, rate)
    
    # one fwd pass on the model
    def call(self, inputs, token_position, feature_emb, training=False):
        """
        inputs : a dict of paramsters to Bert model with keys (input_ids, token_type_ids, attention_mask)
        token_position = [batch_size, 2] - [start,end] for tokens for which complexity is to be predicted
        """

        # bert pass : returns last hidden state, pooler output 
        
        # sentence_output = tf.concat(sentence_output,axis=0)
        sequence_output, _ = self.bert(**inputs, return_dict=False)       #(batch_size, seq_len, 768)

        # overall sentence embedding is CLS emb representation
        cls_emb = sequence_output[:,0,:]        # (batch_size, 768)

        # extract token embeddings    
        # for each example we have start and end token position
        # we add these embeddings position
        temp = []
        for s,t in zip(sequence_output, token_position):
            start,end = t[0],t[1]
            if (t[0]>self.max_length):
                start,end = 1,self.max_length
            elif (t[0]+t[1]>self.max_length):
                end = self.max_length

            temp.append(tf.reduce_sum(tf.gather(s, tf.range(start,start+end), axis=0), axis=0))
        token_emb = tf.stack(temp)
        # # OOM FOR LARGE INPUT
        # token_emb = tf.stack(
        #         [tf.reduce_sum(tf.gather(s, tf.range(t[0],t[0]+t[1]), axis=0), axis=0) for s,t in zip(sequence_output,token_position) ]
        #     )   # (batch_size, 768)

        # feature_emb = self.dense(feature_emb)
        # get weights
        weights = self.getWeights(tf.concat([token_emb, cls_emb], axis=1), training)    # (batch_size, 3)

        # generate attenton-based final embeddings
        # final_emb = weights[0]*token_emb + weights[1]*added_emb
        final_emb = tf.zeros(shape=(), dtype=tf.dtypes.float32)   # (bacth_size, 768)
        final_emb += tf.expand_dims(weights[:, 0], axis=1) * token_emb
        final_emb += tf.expand_dims(weights[:, 1], axis=1) * cls_emb
        # final_emb += tf.expand_dims(weights[:, 2], axis=1) * feature_emb

        # output complexity
        final_output = self.final_layer(final_emb)

        return final_output

    # loss function
    def loss_function(self, real, pred):
        loss_ = tf.keras.losses.MSE(real, pred)
        l = tf.reduce_mean(loss_)
        return l

    # set optimizer
    def compile(self, optimizer):
        super(OurModelBert, self).compile()
        self.optimizer = optimizer

    # train step - fwd pass + back prop + update model weights
    # @tf.function() - may not work beacuse zip is used
    def train_step(self, inputs, token_position, y, feature_emb):
        """
        sentences: batch wise sentences
        inputs : a dict of paramsters to Bert model with keys (input_ids, token_type_ids, attention_mask, training, return_dict)
        token_position = [batch_size, 2] - [start,end] for tokens for which complexity is to be predicted
        """

        with tf.GradientTape() as tape:
            # Forward pass
            y_pred = self(inputs, token_position, feature_emb, True)  
            loss = self.loss_function(y, y_pred)

        # calculate gradients
        gradients = tape.gradient(loss, self.trainable_variables)

        # update model weights using gradients
        self.optimizer.apply_gradients(zip(gradients, self.trainable_variables))

        return loss

    # fit funtion of model
    def _fit(
        self, 
        sentences, 
        tokens, 
        feature_emb,
        complexity,
        epochs, 
        batch_size,
        val_sentences=None,
        val_tokens=None,
        val_feature_emb=None,
        val_complexity=None
        ):
        """
        sentences, tokens, complexity - from dataset -  a list is expected
        """

        # tokenize + pad data
        # tokenized_output = dict of input_ids, token_type_ids, attention_mask
        tokenized_output_data = self.tokenizer.batch_encode_plus(
            sentences,
            add_special_tokens=True, 
            padding='max_length',
            max_length=50, 
            return_tensors="tf",
            truncation=True,
        )
        # self.max_length=tokenized_output_data['input_ids'].shape[1]
        self.max_length = 50

        # identify token position
        # token_position_data is a list of [start, length] for all sentences
        # [start, end] = [start, start+length-1]  => will be done in call function
        # find the token in sentence and 1 is added to accomodate CLS token

        token_position_data =   [ 
                                    [ 
                                        len(self.tokenizer.encode(s.split(t)[0], add_special_tokens=False)) + 1, 
                                        len(self.tokenizer.encode(t, add_special_tokens=False))
                                    ]
                                    for s,t in zip(sentences, tokens) 
                                ]

        # create tensorflow dataset
        tf_data = tf.data.Dataset.from_tensor_slices((tokenized_output_data, token_position_data, feature_emb, complexity))

        # shuffle and batch
        tf_data = tf_data.shuffle(100000, seed=self.random_seed).batch(batch_size)

        train_loss = tf.keras.metrics.Mean(name="train_loss")
        loss_train = {"MSE": []}
        loss_val = {"MSE": []}

        #training starts
        for epoch in range(epochs):
            start = time.time()
            
            # reset state for mean loss
            train_loss.reset_states()

            # loop over batches
            for step, x in tqdm(tf_data.enumerate()):

                # get inputs for model
                tokenized_output = x[0]
                token_position = x[1]
                feature_emb=x[2]
                y = x[3]

                # call trainstep
                loss = self.train_step(tokenized_output, token_position, y, feature_emb)
                train_loss(loss)

            loss_train["MSE"].append(train_loss.result().numpy())
            print("Epoch {} loss  MSE: {}, time taken: {:.2f}s".format(epoch + 1, loss_train["MSE"][-1], time.time() - start))

            # validation if provided
            if (val_sentences!=None):
                val_pred = self._predict(val_sentences, val_tokens, val_feature_emb)
                loss_val["MSE"].append(self.loss_function(val_complexity, val_pred).numpy())
                print("Validation loss MSE : {}".format(loss_val["MSE"][-1]))
        
        if (val_sentences!=None):
            return loss_train, loss_val
        return loss_train

    
    # predict function
    def _predict(self, sentences, tokens, feature_emb):

        # tokenize + pad data - only till max length
        # tokenized_output = dict of input_ids, token_type_ids, attention_mask
        tokenized_output_data = self.tokenizer.batch_encode_plus(
            sentences,
            add_special_tokens=True, 
            padding='max_length',
            max_length=self.max_length,
            truncation=True,
            return_tensors="tf"
        )

        token_position_data =   [ 
                                    [ 
                                        len(self.tokenizer.encode(s.split(t)[0], add_special_tokens=False)) + 1, 
                                        len(self.tokenizer.encode(t, add_special_tokens=False))
                                    ]
                                    for s,t in zip(sentences, tokens) 
                                ]

        # create tensorflow dataset
        tf_data = tf.data.Dataset.from_tensor_slices((tokenized_output_data,token_position_data, feature_emb))

        # batch for memory constraints
        tf_data = tf_data.batch(32)

        pred_list = []
        for step, x in tf_data.enumerate():
            # get inputs for model
            input_seq = x[0]
            token_position = x[1]
            feature_emb = x[2]

            pred_list.append(self(input_seq, token_position, feature_emb))
        
        return tf.concat(pred_list,axis=0)
print(len(sentences_train_list))
print(len(sentences_test_list))

print(len(tokens_train_list))
print(len(tokens_test_list))

features_train = train[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
                        'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
                        'google frequency1', 'google frequency2', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

features_test = test[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
                        'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
                        'google frequency1', 'google frequency2', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

print("feature emb shape train : {}".format(features_train.shape))
print("feature emb shape test : {}".format(features_test.shape))
PRETRAINED_BERT_MODEL = "bert-base-cased"
model = OurModelBert(bert_model=PRETRAINED_BERT_MODEL, hidden_size=10, random_seed=42, rate=0.3)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.01))
model._fit(sentences_train_list, tokens_train_list, features_train, complexity_train_list, 20, 32, sentences_test_list, tokens_test_list, features_test, complexity_test_list)
model.summary()<jupyter_output><empty_output><jupyter_text>Evaluate<jupyter_code>print(len(list(test_features['sentence'])))
print(len(list(test_features['token'])))

features = test_features[['token_length', 'token_vowels', 'syllables', 'pos1', 'pos2', 'dep num1', 'dep num2', 
                        'synonyms1', 'synonyms2', 'hypernyms1', 'hypernyms2', 'hyponyms1', 'hyponyms2', 
                        'google frequency1', 'google frequency2', 
                        'biomedical', 'bible', 'subtitles', 'wiki', 'familarity']].values

print("feature shape : {}".format(features.shape))

SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/approach1/bert/single")

if( not os.path.exists(SUBMISSION_FOLDER)):
    os.makedirs(SUBMISSION_FOLDER)

# predict 
pred = model._predict(sentences, tokens, feature_emb)
pred = pd.DataFrame({"ID":test_features.index, "complexity":pred.numpy().reshape(-1,)})
pred.to_csv(SUBMISSION_FOLDER+"/app_bert1.csv", index=False, header=False)

evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_single_test_labelled_preprocessed.csv")
SUBMISSION_FOLDER = os.path.join(FOLDER_PATH,"predictions/approach1/multi")

if( not os.path.exists(SUBMISSION_FOLDER)):
    os.makedirs(SUBMISSION_FOLDER)

# predict 
pred = model._predict(multi_sent_emb, multi_token_data, features_multi)
pred = pd.DataFrame({"ID":test_multi_features.index, "complexity":pred.numpy().reshape(-1,)})
pred.to_csv(SUBMISSION_FOLDER+"/app1.csv", index=False, header=False)


evaluate(SUBMISSION_FOLDER, FOLDER_PATH+"/references/lcp_multi_test_labelled_preprocessed.csv")<jupyter_output>For file app1.csv
pearson  :  0.7515866884564054
spearman :  0.7702903717771301
mae      :  0.10772707952766654
mse      :  0.017540098451649166
r2       :  0.2733147271433434<jupyter_text>test bert code<jupyter_code># ----------- test model output shape

sample_model = OurModelBert( bert_model="bert-base-cased",
        hidden_size=100,
        random_seed=12,
        )

inputs = sample_model.tokenizer.batch_encode_plus(["this is a senten ce", "dada af"], add_special_tokens=True, padding='max_length', max_length=20, return_tensors="tf")
token_position_data = tf.constant([(3,5) for _ in range(2)])

output = sample_model(inputs, token_position_data)
print(output.shape)

#  ----------- test fit and predict

sample_model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.001))

sentences = ["this is a senten ce", "dada af", "third sentence is this ok"]
tokens =  ["senten", "a", "this ok"]

y = tf.random.uniform((3, 1), dtype=tf.float32, minval=0, maxval=1)

loss = sample_model._fit( 
    sentences=sentences, 
    tokens=tokens, 
    complexity=y,
    epochs = 10, 
    batch_size = 32,
    )

FOLDER_PATH = "/content/drive/MyDrive/CS60075-Team28-Task-1"
DATA_FOLDER = FOLDER_PATH+"/data/preprocessed"
df = pd.read_csv(os.path.join(DATA_FOLDER, "lcp_single_train_preprocessed.csv"), index_col=0)
df.head()
# from google.colab import drive
# drive.mount('/content/drive')
df['sentence'] = df['sentence'].astype(str)
df['token'] = df['token'].astype(str)

sentences = list(df["sentence"])
tokens = list(df["token"])
complexity = list(df["complexity"])

model = OurModelBert(bert_model=PRETRAINED_BERT_MODEL, hidden_size=10, random_seed=42)
model.compile(optimizer=tf.keras.optimizers.Adam(learning_rate=0.0001))
model._fit(sentences, tokens, complexity, 10, 64)
model.summary()
tokenizer = BertTokenizer.from_pretrained("bert-base-cased")

tok = tokenizer.encode_plus("this is a senten ce", add_special_tokens=True, padding='max_length', max_length=20, return_tensors="tf")
print(tok)

# model = TFBertModel.from_pretrained(PRETRAINED_BERT_MODEL)
# last_hidden_state,pooler_output = model(**tok, return_dict=False)
# cls = last_hidden_state[:,0,:]

sentences = ["this is a senten ce"]
tokens = ["senten"]
("his is a senten ce").split(tokens[0])
tokenizer.encode(sentences[0].split(tokens[0])[0], add_special_tokens=False)
token_position_data =   [ 
                                    [ 
                                        len(s.split(t)[0].split()), 
                                        len(t.split())
                                    ]
                                    for s,t in zip(sentences, tokens) 
                                ]
token_position_data

# Dataset = namedtuple('Dataset', 'name, train, test')
# Model = namedtuple('Model', 'type, name, dimension, corpus, model')


# from gensim.test.utils import datapath, get_tmpfile
# from gensim.models import KeyedVectors
# from gensim.scripts.glove2word2vec import glove2word2vec

# MAIN_PATH = 'embeddings/'

# glove_models = []

# glove_defs = [ Model('glove', 'glove.6B.300d.txt', 300, 'wikipedia+gigaword5', None)]
              
# for model in glove_defs:
#     glove_file = MAIN_PATH + model.name
#     tmp_file = get_tmpfile(model.name + '-temp')
#     glove2word2vec(glove_file, tmp_file)
#     vecs = KeyedVectors.load_word2vec_format(tmp_file)
#     glove_models.append(Model(model.type, model.name, model.dimension, model.corpus, vecs))
#     print('load model : {}'.format(model.name))
    
# print(glove_models)

# tokenizer = Tokenizer()
# tokenizer.fit_on_texts(sentences_train_list)
# words_to_index = tokenizer.word_index
# len(words_to_index)

# vocab_len = len(words_to_index)
# embed_vector_len = word_to_vec_map['hello'].shape[0]

# emb_matrix = np.zeros((vocab_len+1, embed_vector_len))

# for word, index in words_to_index.items():
#   embedding_vector = word_to_vec_map.get(word)
#   if embedding_vector is not None:
#     emb_matrix[index, :] = embedding_vector

# embedding_layer = Embedding(input_dim=vocab_len, output_dim=embed_vector_len, input_length=256, weights = [emb_matrix], trainable=False)

# emb_matrix.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14104410441044105, 'fraction_numerical': 0.016633663366336635, 'mean_word_length': 2.7638728911172845, 'pattern_counts': {'":': 16, '<': 52, '<?xml version=': 0, '>': 55, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 18, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '260019', 'license': 'MIT', 'n_tokens_mistral': 18157, 'n_tokens_neox': 17292, 'n_words': 3427, 'path': 'attention_single.ipynb', 'repo_name': 'hvarS/CS60075-Team28-Semeval-Task-1', 'size': '55550'}