<jupyter_start><jupyter_text>Data Pre-processing and Transformation<jupyter_code>train = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/train.csv")
test = pd.read_csv("/kaggle/input/tweet-sentiment-extraction/test.csv")
print("Training set has {} data points".format(len(train)))
print("Testing set has {} data points".format(len(test)))
train.head()
test.head()<jupyter_output><empty_output><jupyter_text>Tokenizer The pretrained RoBERTa model and tokenizer are from huggingface [transformers](https://huggingface.co/transformers/main_classes/model.html?highlight=save_pretrained) library. They can be downloaded by using the `from_pretrained()` method or attached to a kaggle kerned from [here](https://www.kaggle.com/cdeotte/tf-roberta)<jupyter_code>class TweetDataset:
    def __init__(self, data_df, tokenizer, train=True, max_len=96):
        self.data = data_df.dropna(axis=0).reset_index(drop=True)
        self.is_train = True if train else False
        self.sentiment_tokens = {
            'positive': tokenizer.encode('positive').ids[0], 
            'negative': tokenizer.encode('negative').ids[0],
            'neutral': tokenizer.encode('neutral').ids[0]
        }
        self.tokenizer = tokenizer
        self.max_len = max_len
    
    def ByteLevelBPEPreprocessor(self, text, selected_text, sentiment):
        
        text = " " + " ".join(text.split())
        enc = self.tokenizer.encode(text)
        s_tok = self.sentiment_tokens[sentiment]
        
        # Get InputIDs
        input_ids = np.ones((self.max_len),
                            dtype = 'int32')
        input_ids[:len(enc.ids)+5] = [0] + enc.ids + [2,2] + [s_tok] + [2]

        # Get Attention mask
        attention_mask = np.zeros((self.max_len),
                                  dtype='int32')
        attention_mask[:len(enc.ids)+5] = 1
        
        if self.is_train:
            selected_text = " ".join(selected_text.split())
            idx = text.find(selected_text)
            char_tokens = np.zeros((len(text)))
            char_tokens[idx:idx+len(selected_text)] = 1
            # if text has ' ' prefix
            if text[idx-1] == ' ': 
                char_tokens[idx-1] = 1
                
            # Get start and end token for selected_text in input IDs
            start_tokens = np.zeros((self.max_len),
                                    dtype='int32')
            end_tokens = np.zeros((self.max_len),
                                  dtype='int32')
            ptr_idx = 0
            label_idx = list()
            for i, enc_id in enumerate(enc.ids):
                sub_word = self.tokenizer.decode([enc_id])
                if sum(char_tokens[ptr_idx:ptr_idx+len(sub_word)]) > 0:
                    label_idx.append(i)
                ptr_idx += len(sub_word)
            if label_idx:
                # + 1 as we added prefix before
                start_tokens[label_idx[0] + 1] = 1
                end_tokens[label_idx[-1] + 1] = 1
            return input_ids, attention_mask, start_tokens, end_tokens
        
        return input_ids, attention_mask
            
    def __call__(self):
        data_len = len(self.data)
        input_ids = np.ones((data_len, self.max_len), 
                            dtype='int32')
        attention_mask = np.zeros((data_len, self.max_len), 
                                  dtype='int32')
        token_type_ids = np.zeros((data_len, self.max_len),
                                  dtype='int32')
        if self.is_train:
            start_tokens = np.zeros((data_len, self.max_len),
                                    dtype='int32')
            end_tokens = np.zeros((data_len, self.max_len),
                                  dtype='int32')
        for i, row in tqdm(self.data.iterrows(), total=len(self.data)):
            out = self.ByteLevelBPEPreprocessor(
                row['text'], 
                row['selected_text'] if self.is_train else None, 
                row['sentiment']
            )
            if self.is_train:
                input_ids[i], attention_mask[i], start_tokens[i], end_tokens[i] = out
            else:
                input_ids[i], attention_mask[i] = out
        if self.is_train:
            return input_ids, attention_mask, token_type_ids, start_tokens, end_tokens
        return input_ids, attention_mask, token_type_ids
class TransformerQA:
    def __init__(self, max_len, model_path, model_save_path, 
                 tokenizer, fit=True):
        self.max_len = max_len
        self.model_path = model_path
        self.model_save_path = model_save_path
        self.tokenizer = tokenizer
        
    def roberta_model(self):
        input_ids = tf.keras.layers.Input((self.max_len,),
                                          dtype=tf.int32)
        attention_mask = tf.keras.layers.Input((self.max_len,),
                                               dtype=tf.int32)
        token_type_ids = tf.keras.layers.Input((self.max_len,),
                                               dtype=tf.int32)

        config = RobertaConfig.from_pretrained(
            os.path.join(self.model_path, 'config-roberta-base.json')
        )
        roberta_model = TFRobertaModel.from_pretrained(
            os.path.join(self.model_path, 'pretrained-roberta-base.h5'),
            config=config
        )
        x = roberta_model(inputs=input_ids,
                          attention_mask=attention_mask,
                          token_type_ids=token_type_ids)

        x1 = tf.keras.layers.Dropout(0.1)(x[0]) 
        x1 = tf.keras.layers.Conv1D(1,1)(x1)
        x1 = tf.keras.layers.Flatten()(x1)
        x1 = tf.keras.layers.Activation('softmax')(x1)

        x2 = tf.keras.layers.Dropout(0.1)(x[0]) 
        x2 = tf.keras.layers.Conv1D(1,1)(x2)
        x2 = tf.keras.layers.Flatten()(x2)
        x2 = tf.keras.layers.Activation('softmax')(x2)

        model = tf.keras.models.Model(
            inputs=[input_ids, attention_mask, token_type_ids], 
            outputs=[x1,x2]
        )
        optimizer = tf.keras.optimizers.Adam(learning_rate=3e-5)
        model.compile(loss='categorical_crossentropy', optimizer=optimizer)

        return model
    
    def jaccard(self, str1, str2):
        a = set(str1.lower().split()) 
        b = set(str2.lower().split())
        if (len(a)==0) & (len(b)==0): return 0.5
        c = a.intersection(b)
        return float(len(c)) / (len(a) + len(b) - len(c))
    
    def get_model_selected_text(self, data_df, preds_start, preds_end):
        st_list = []
        for k in range(len(data_df)):
            idx_start = np.argmax(preds_start[k,])
            idx_end = np.argmax(preds_end[k,])
            if idx_start > idx_end:
                st = data_df.loc[k,'text']
#                 if data_df.loc[k, 'sentiment'] != 'neutral':
#                     st = st.split()[idx_start]
            else:
                text = " " + " ".join(data_df.loc[k,'text'].split())
                enc = self.tokenizer.encode(text)
                st = self.tokenizer.decode(enc.ids[idx_start-1:idx_end])
            st_list.append(st)
        return st_list
    
    def fit(self, train_df, input_ids, attention_mask, 
            token_type_ids, start_tokens, end_tokens, 
            stratify_y, VER='v0', verbose=1):
        avg_score = []
        oof_start = np.zeros((input_ids.shape[0],
                              self.max_len))
        oof_end = np.zeros((input_ids.shape[0],
                            self.max_len))
        skf = StratifiedKFold(n_splits=5,
                              shuffle=True,
                              random_state=42)

        for fold, (idxT,idxV) in enumerate(skf.split(input_ids,
                                                     stratify_y)):
            print('Training FOLD {}:'.format(fold+1))
            K.clear_session()
            model = self.roberta_model()
            model_filename = os.path.join(
                self.model_save_path, 
                '{}-roberta-{}.h5'.format(VER, fold)
            )
            sv = tf.keras.callbacks.ModelCheckpoint(
                model_filename, 
                monitor='val_loss', 
                verbose=verbose, 
                save_best_only=True,
                save_weights_only=True, 
                mode='auto', 
                save_freq='epoch'
            )
            model.fit([input_ids[idxT,], 
                       attention_mask[idxT,], 
                       token_type_ids[idxT,]], 
                      [start_tokens[idxT,], end_tokens[idxT,]],
                      epochs=3, 
                      batch_size=32, 
                      verbose=verbose, 
                      callbacks=[sv],
                      validation_data=(
                          [
                              input_ids[idxV,],
                              attention_mask[idxV,],
                              token_type_ids[idxV,]
                          ], 
                          [start_tokens[idxV,], end_tokens[idxV,]]
                      )
                     )
            # Load best saved model from disk
            print('Loading model...')
            model.load_weights(model_filename)
            
            # Predicting OOF samples
            print('Predicting OOF...')
            oof_start[idxV,],oof_end[idxV,] = model.predict(
                [
                    input_ids[idxV,],
                    attention_mask[idxV,],
                    token_type_ids[idxV,]
                ],
                verbose=verbose
            )
            
            pred_df = train_df.loc[idxV].reset_index(drop=True)
            pred_df['oof_st'] = self.get_model_selected_text(
                data_df=pred_df,
                preds_start=oof_start[idxV,],
                preds_end=oof_end[idxV,]
            )
            fold_val_score = pred_df.apply(
                lambda x: self.jaccard(x['selected_text'], 
                                       x['oof_st']
                                      ),
                axis=1
            ).mean()
            avg_score.append(fold_val_score)
            print('>>>> FOLD {} Jaccard score = {}'.format(fold+1, 
                                                           fold_val_score))
    def predict(self, pred_df, input_ids, attention_mask, 
                token_type_ids, n_models, VER='v0', verbose=1):
        preds_start = np.zeros((input_ids.shape[0],
                                self.max_len))
        preds_end = np.zeros((input_ids.shape[0],
                              self.max_len))
        for i in range(n_models):
            K.clear_session()
            model = self.roberta_model()
            model_filename = os.path.join(
                self.model_save_path, 
                '{}-roberta-{}.h5'.format(VER, i)
            )
            
            print('Loading model...')
            model.load_weights(model_filename)
            
            preds = model.predict(
                [input_ids, attention_mask, token_type_ids],
                verbose=verbose
            )
            preds_start += preds[0]/n_models
            preds_end += preds[1]/n_models
        
        test_st = self.get_model_selected_text(
            data_df=pred_df,
            preds_start=preds_start,
            preds_end=preds_end
        )
        return test_st
MAX_LEN = 148
PATH = '../input/tf-roberta/'
tokenizer = tokenizers.ByteLevelBPETokenizer(
    vocab_file=PATH+'vocab-roberta-base.json', 
    merges_file=PATH+'merges-roberta-base.txt', 
    lowercase=True,
    add_prefix_space=True
)
train_data = TweetDataset(train, tokenizer, train=True, max_len=MAX_LEN)
input_ids, attention_mask, token_type_ids, start_tokens, end_tokens = train_data()
test_data = TweetDataset(test, tokenizer, train=False, max_len=MAX_LEN)
test_input_ids, test_attention_mask, test_token_type_ids = test_data()
QA_model = TransformerQA(
    max_len=MAX_LEN, 
    model_path=PATH,
    model_save_path='../input/tweetextraction-trainedroberta-v0',
    tokenizer=tokenizer
)
# Since, it's inference notbook, I won't train

# QA_model.fit(train_df=train_data.data, 
#              input_ids=input_ids, 
#              attention_mask=attention_mask, 
#              token_type_ids=token_type_ids, 
#              start_tokens=start_tokens, 
#              end_tokens=end_tokens, 
#              stratify_y=train_data.data.sentiment.values)
# Loading pretrained model weights obtained by running training notebook
test['selected_text'] = QA_model.predict(pred_df=test, 
                                         input_ids=test_input_ids, 
                                         attention_mask=test_attention_mask, 
                                         token_type_ids=test_token_type_ids, 
                                         n_models=5)
test[['textID','selected_text']].to_csv('submission.csv',index=False)
test.sample(25)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10930610350429334, 'fraction_numerical': 0.009128181325907016, 'mean_word_length': 1.620186461289015, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 15, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 28, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '274802', 'license': 'MIT', 'n_tokens_mistral': 4011, 'n_tokens_neox': 3842, 'n_words': 584, 'path': 'inference_notebooks/tweet-extraction-inference.ipynb', 'repo_name': 'jasoriya/tweet-extraction', 'size': '12927'}