<jupyter_start><jupyter_text>Data Preparation<jupyter_code># data analysis libraries
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns

# data preprocessing libraries
import re
from datetime import datetime
import spacy
import nltk
from nltk import word_tokenize, sent_tokenize
from spacy_preprocessor import SpacyPreprocessor
from sklearn.model_selection import train_test_split

# other
from tqdm import tqdm, tqdm_notebook, tqdm_pandas

tqdm.pandas()
tqdm_notebook().pandas()
import torch
from torchtext.legacy import data
from torchtext.legacy import datasets
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
import random

SEED = 1234

random.seed(SEED)
np.random.seed(SEED)
torch.manual_seed(SEED)
torch.backends.cudnn.deterministic = True
from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased')
len(tokenizer.vocab)
init_token = tokenizer.cls_token
eos_token = tokenizer.sep_token
pad_token = tokenizer.pad_token
unk_token = tokenizer.unk_token

init_token_idx = tokenizer.cls_token_id
eos_token_idx = tokenizer.sep_token_id
pad_token_idx = tokenizer.pad_token_id
unk_token_idx = tokenizer.unk_token_id

print(init_token, eos_token, pad_token, unk_token)
print(init_token_idx, eos_token_idx, pad_token_idx, unk_token_idx)
max_input_length = tokenizer.max_model_input_sizes['bert-base-uncased']
print(max_input_length)
def tokenize_and_cut(sentence):
    tokens = tokenizer.tokenize(sentence) 
    tokens = tokens[:max_input_length-2]
    return tokens
TEXT = data.Field(batch_first = True,
                  use_vocab = False,
                  tokenize = tokenize_and_cut,
                  preprocessing = tokenizer.convert_tokens_to_ids,
                  init_token = init_token_idx,
                  eos_token = eos_token_idx,
                  pad_token = pad_token_idx,
                  unk_token = unk_token_idx)

LABEL = data.LabelField(dtype = torch.float)
fields = {'text': ('text',TEXT), 'target': ('label', LABEL)}
sui_df = pd.read_csv('data/sui_df.csv')
sui_df.head()
train_data, valid_data, test_data = data.TabularDataset.splits(
    path = 'data', 
    train = 'sui_df_train.csv',
    validation = 'sui_df_val.csv',
    test = 'sui_df_test.csv',
    format ='csv',
    fields = fields,
    skip_header = False
)
print(f"Number of training examples: {len(train_data)}")
print(f"Number of validation examples: {len(valid_data)}")
print(f"Number of testing examples: {len(test_data)}")
LABEL.build_vocab(train_data)
BATCH_SIZE = 32

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')

train_iterator, valid_iterator, test_iterator = data.BucketIterator.splits(
    (train_data, valid_data, test_data),
    sort_within_batch = True,
    sort_key = lambda x: len(x.text),
    batch_size = BATCH_SIZE, 
    device = device)
from transformers import BertTokenizer, BertModel

bert = BertModel.from_pretrained('bert-base-uncased')
import torch.nn as nn

class BERT_Classifier(nn.Module):
    def __init__(self, bert, hidden_dim, output_dim, n_layers, bidirectional, dropout):
        
        super().__init__()
        self.bert = bert
        
        embedding_dim = bert.config.to_dict()['hidden_size']
        self.rnn = nn.GRU(embedding_dim, hidden_dim, num_layers = n_layers, bidirectional = bidirectional, batch_first = True, dropout = 0 if n_layers < 2 else dropout)
        self.out = nn.Linear(hidden_dim * 2 if bidirectional else hidden_dim, output_dim)
        self.dropout = nn.Dropout(dropout)
        
    def forward(self, text):
          
        with torch.no_grad():
            embedded = self.bert(text)[0]
        _, hidden = self.rnn(embedded)
        
        if self.rnn.bidirectional:
            hidden = self.dropout(torch.cat((hidden[-2,:,:], hidden[-1,:,:]), dim = 1))
        else:
            hidden = self.dropout(hidden[-1,:,:])
        
        output = self.out(hidden)
        
        return output, hidden
HIDDEN_DIM = 256
OUTPUT_DIM = 1
N_LAYERS = 2
BIDIRECTIONAL = True
DROPOUT = 0.25

model = BERT_Classifier(bert, HIDDEN_DIM, OUTPUT_DIM, N_LAYERS, BIDIRECTIONAL, DROPOUT)
def count_parameters(model):
    return sum(p.numel() for p in model.parameters() if p.requires_grad)

print(f'The model has {count_parameters(model):,} trainable parameters')

# Gotta freeze all BERT layers. Not going to retrain those.
for name, param in model.named_parameters():                
    if name.startswith('bert'):
        param.requires_grad = False

print(f'The model has {count_parameters(model):,} trainable parameters')
import torch.optim as optim

optimizer = optim.Adam(model.parameters())
criterion = nn.BCEWithLogitsLoss()
model = model.to(device)
criterion = criterion.to(device)

print(optimizer, criterion, model)
def binary_accuracy(preds, y):
    """
    Returns accuracy per batch, i.e. if you get 8/10 right, this returns 0.8, NOT 8
    """
    #round predictions to the closest integer
    rounded_preds = torch.round(torch.sigmoid(preds))
    correct = (rounded_preds == y).float() #convert into float for division 
    acc = correct.sum() / len(correct)
    return acc
def train(model, iterator, optimizer, criterion):
    
    epoch_loss = 0
    epoch_acc = 0
    
    model.train()
    
    for batch in iterator:
        
        optimizer.zero_grad()
        
        predictions = model(batch.text)[0].squeeze(1)
        
        loss = criterion(predictions, batch.label)
        
        acc = binary_accuracy(predictions, batch.label)
        
        loss.backward()
        
        optimizer.step()
        
        epoch_loss += loss.item()
        epoch_acc += acc.item()
        
    return epoch_loss / len(iterator), epoch_acc / len(iterator)
def evaluate(model, iterator, criterion):
    
    epoch_loss = 0
    epoch_acc = 0
    
    model.eval()
    
    with torch.no_grad():
    
        for batch in iterator:

            predictions = model(batch.text)[0].squeeze(1)
            
            loss = criterion(predictions, batch.label)
            
            acc = binary_accuracy(predictions, batch.label)

            epoch_loss += loss.item()
            epoch_acc += acc.item()
        
    return epoch_loss / len(iterator), epoch_acc / len(iterator)
import time

def epoch_time(start_time, end_time):
    elapsed_time = end_time - start_time
    elapsed_mins = int(elapsed_time / 60)
    elapsed_secs = int(elapsed_time - (elapsed_mins * 60))
    return elapsed_mins, elapsed_secs
N_EPOCHS = 5

best_valid_loss = float('inf')

for epoch in range(N_EPOCHS):
    
    start_time = time.time()
    
    train_loss, train_acc = train(model, train_iterator, optimizer, criterion)
    valid_loss, valid_acc = evaluate(model, valid_iterator, criterion)
        
    end_time = time.time()
        
    epoch_mins, epoch_secs = epoch_time(start_time, end_time)
        
    if valid_loss < best_valid_loss:
        best_valid_loss = valid_loss
        torch.save(model.state_dict(), 'SUIBERT_RNN.pt')
    
    print(f'Epoch: {epoch+1:02} | Epoch Time: {epoch_mins}m {epoch_secs}s')
    print(f'\tTrain Loss: {train_loss:.3f} | Train Acc: {train_acc*100:.2f}%')
    print(f'\t Val. Loss: {valid_loss:.3f} |  Val. Acc: {valid_acc*100:.2f}%')
model.load_state_dict(torch.load('SUIBERT_RNN.pt'))

test_loss, test_acc = evaluate(model, test_iterator, criterion)

print(f'Test Loss: {test_loss:.3f} | Test Acc: {test_acc*100:.2f}%')
def predict_suicidality(model, tokenizer, sentence):
    model.eval()
    tokens = tokenizer.tokenize(sentence)
    tokens = tokens[:max_input_length-2]
    indexed = [init_token_idx] + tokenizer.convert_tokens_to_ids(tokens) + [eos_token_idx]
    tensor = torch.LongTensor(indexed).to(device)
    tensor = tensor.unsqueeze(0)
    prediction = torch.sigmoid(model(tensor)[0])
    return prediction.item(), model(tensor)[1]
predict_suicidality(model, tokenizer, "I want to kill myself")[0]
predict_suicidality(model, tokenizer, "I have been feeling like I want to die")[0]
predict_suicidality(model, tokenizer, "I have been feeling depressed")[0]
predict_suicidality(model, tokenizer, "I have been feeling anxious. Things don't make sense. I cut myself")[0]
len(predict_suicidality(model, tokenizer, "lol what haha")[1][0])
a = predict_suicidality(model, tokenizer, "I have been cutting myself")[1].detach().cpu().numpy().flatten()
b = predict_suicidality(model, tokenizer, "I want to harm myself")[1].detach().cpu().numpy().flatten()
from numpy import dot
from numpy.linalg import norm

def cos_sim(a,b):
  cossim = dot(a, b)/(norm(a)*norm(b))
  return cossim
cos_sim(a,b)
total_df = pd.read_csv('data/total_df.csv', low_memory=False)
total_df.head(3)
def get_tokens(text):
  tokens = tokenizer.tokenize(text)
  tokens = tokens[:max_input_length-2]
  indexed = [init_token_idx] + tokenizer.convert_tokens_to_ids(tokens) + [eos_token_idx]
  return indexed
indx = total_df['text'].progress_apply(get_tokens)
#tensor = torch.LongTensor(indx.to_list())
tensor = list(map(lambda x: torch.LongTensor(x).unsqueeze(0).to(device), indx.to_list()))
model.eval()
list(map(lambda x: model(x)[1][0], tensor))
  tensor = torch.LongTensor(indexed).to(device)
  tensor = tensor.unsqueeze(0)
  prediction = torch.sigmoid(model(tensor)[0])
total_df['activity_id'] 
fields = {'text': ('text',TEXT), 'target': ('label', LABEL)}
tokenizer.tokenize(["I have been feeling like I want to die","lol what haha"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11496196111580727, 'fraction_numerical': 0.008347421808960271, 'mean_word_length': 3.744360902255639, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '640624', 'license': 'MIT', 'n_tokens_mistral': 3295, 'n_tokens_neox': 3209, 'n_words': 701, 'path': 'notebooks/08_SUIBERT_classifier.ipynb', 'repo_name': 'aliforgetti/suicide_detector', 'size': '9464'}