<jupyter_start><jupyter_text>Imports<jupyter_code># !pip install transformers
!pip install sentence-transformers
from sentence_transformers import SentenceTransformer

import pandas as pd
import numpy as np

import joblib
import nltk
import re
nltk.download('punkt')<jupyter_output>[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Package punkt is already up-to-date!<jupyter_text>Loading the Dataset<jupyter_code>%cd '/content/drive/My Drive/IIITD/SEM-7/ML/ML Project/Code/Dataset'
dataset = pd.read_csv('olid-training-v1.0.tsv', sep='\t', index_col='id')
test_set = pd.read_csv('testset-levela.tsv', sep='\t', index_col='id')
subtask_A = dataset[['tweet','subtask_a']]
subtask_A.head()<jupyter_output><empty_output><jupyter_text>Utility Functions<jupyter_code>def clean_tweet(tweet):
    '''
    removes punctuation marks, other redundant characters, words
    '''
    tweet = re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)", " ", tweet).split()
    tweet = [word.lower() for word in tweet if word!='URL']
    return ' '.join(tweet)
# generate columns for extracting features

subtask_A['cleaned'] = subtask_A['tweet'].apply(clean_tweet)
test_set['cleaned'] = test_set['tweet'].apply(clean_tweet)
subtask_A_cleaned = subtask_A[['cleaned','subtask_a']]
valid_data_cleaned = test_set[['cleaned']]
subtask_A_cleaned.to_csv('subtask_A_cleaned.csv')
valid_data_cleaned.to_csv('valid_data_cleaned.csv')<jupyter_output><empty_output><jupyter_text>Sentence2Vec [BERT]<jupyter_code># tokenizer = transformers.BertTokenizer.from_pretrained('bert-base-uncased')
# nlp = transformers.TFBertModel.from_pretrained('bert-base-uncased')
model = SentenceTransformer('bert-base-nli-cls-token')
train_sentences = subtask_A['cleaned'].tolist()
test_sentences = test_set['cleaned'].tolist()

train_embeddings = model.encode(train_sentences)
test_embeddings = model.encode(test_sentences)
joblib.dump(np.array(train_embeddings),'X_train_cls')
joblib.dump(np.array(test_embeddings),'X_test_cls')
y_train = np.array(subtask_A['subtask_a'].tolist())
joblib.dump(y_train,'y_train')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1551155115511551, 'fraction_numerical': 0.003771805752003772, 'mean_word_length': 7.804979253112033, 'pattern_counts': {'":': 0, '<': 16, '<?xml version=': 0, '>': 16, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '619749', 'license': 'MIT', 'n_tokens_mistral': 820, 'n_tokens_neox': 805, 'n_words': 111, 'path': 'Preprocessing/BERT_Preprocessing.ipynb', 'repo_name': 'Ashwin-19/OffensEval-2019', 'size': '2121'}