<jupyter_start><jupyter_text>Build dataset for classification<jupyter_code>LABEL_MAP = {
    "Democrat": 0,
    "Republican": 1
}

def buildLabels(row):
    return LABEL_MAP.get(row["label"])

def cleanTweet(row):
  tweet = row["text"]
  tweet = str(p.clean(tweet))
  tweet = re.sub(r'[^\w\s]', '', tweet) # punctuation
  tweet = re.sub("^\d+\s|\s\d+\s|\s\d+$", " ", tweet) # numbers
  return tweet

  
dataset_final["label"] = dataset_final.apply(lambda row: buildLabels(row), axis=1)
dataset_final["clean_text"] = dataset_final.apply(lambda row: cleanTweet(row), 
                                                  axis=1)
dataset_clf = dataset_final[["clean_text", "label"]]
dataset_clf.reset_index(drop=True, inplace=True)
dataset_clf.to_csv('/content/drive/MyDrive/labeled_dataset_v2.csv')
X_train, X_val, y_train, y_val = train_test_split(dataset_clf.index.values, 
                                                  dataset_clf.label.values, 
                                                  test_size=0.15, 
                                                  random_state=42, 
                                                  stratify=dataset_clf.label.values)

dataset_clf['data_type'] = ['not_set']*dataset_final.shape[0]

dataset_clf.loc[X_train, 'data_type'] = 'train'
dataset_clf.loc[X_val, 'data_type'] = 'test'
dataset_clf.groupby(['label', 'data_type']).count()<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:7: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  import sys
/usr/local/lib/python3.6/dist-packages/pandas/core/indexing.py:1763: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  isetter(loc, value)<jupyter_text>Try out BERTForSequenceClassification to see how it goesCan be our baseline as this just uses a single torch Linear layer for classification (i.e. assumes data is linearly seperable) after a Dropout on top of the BERT model<jupyter_code>tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', 
                                          do_lower_case=True)
# find distribution of length of tokenized tweets
lens = []

for tweet in dataset_final.clean_text:
  tokens = tokenizer.encode(tweet, max_length=512)
  lens.append(len(tokens))

sns.displot(lens)
plt.xlim([0, 128])
# tokenize train and test data so BERT can understand it
encoded_data_train = tokenizer.batch_encode_plus(
    dataset_clf[dataset_clf.data_type=='train'].clean_text.values, 
    add_special_tokens=True, 
    return_attention_mask=True, 
    padding=True,
    truncation=True,
    return_tensors='pt'
)

encoded_data_test = tokenizer.batch_encode_plus(
    dataset_clf[dataset_clf.data_type=='test'].clean_text.values, 
    add_special_tokens=True, 
    return_attention_mask=True, 
    padding=True, 
    truncation=True,
    return_tensors='pt'
)


# destructure out the input_ids, attention masks, and labels from tokenizer & encoder output
input_ids_train = encoded_data_train['input_ids']
attention_masks_train = encoded_data_train['attention_mask']
labels_train = torch.tensor(dataset_clf[dataset_clf.data_type=='train'].label.values)

input_ids_test = encoded_data_test['input_ids']
attention_masks_test = encoded_data_test['attention_mask']
labels_test = torch.tensor(dataset_clf[dataset_clf.data_type=='test'].label.values)
tokenizer.convert_ids_to_tokens(input_ids_test[0])
train_data = TensorDataset(input_ids_train, attention_masks_train, labels_train)
test_data = TensorDataset(input_ids_test, attention_masks_test, labels_test)

BATCH_SIZE = 8

train_dataloader = DataLoader(train_data, 
                              sampler=RandomSampler(train_data), 
                              batch_size=BATCH_SIZE)

test_dataloader = DataLoader(test_data,
                             sampler=SequentialSampler(test_data),
                             batch_size=BATCH_SIZE)
model = BertForSequenceClassification.from_pretrained("bert-base-uncased",
                                                      num_labels=2,
                                                      output_attentions=False,
                                                      output_hidden_states=False)
EPOCHS = 5

optimizer = AdamW(model.parameters(),
                  lr=1e-5,
                  eps=1e-8)
scheduler = get_linear_schedule_with_warmup(optimizer,
                                            num_warmup_steps=0, 
                                            num_training_steps=len(train_dataloader)*EPOCHS)
def auc_score(preds, labels):
  soft_preds = softmax(preds, axis=1) # logit -> probability
  print(preds.shape, labels.shape)
  if np.shape(preds)[1] > 2: # check for multi-class
    return roc_auc_score(labels, soft_preds, multi_class='ovr')
  else:
    soft_preds = soft_preds[:,1]
    return roc_auc_score(labels, soft_preds)

def acc_score_by_class(preds, labels):
  label_dict_inverse = {v: k for k, v in LABEL_MAP.items()} 

  preds_flat = np.argmax(preds, axis=1).flatten()
  labels_flat = labels.flatten()

  for label in np.unique(labels_flat):
    y_preds = preds_flat[labels_flat==label]
    y_true = labels_flat[labels_flat==label]
    print(f'Class: {label_dict_inverse[label]}')
    print(f'Accuracy: {len(y_preds[y_preds==label])}/{len(y_true)}\n')<jupyter_output><empty_output><jupyter_text>following is mostly taken from [this medium article](https://towardsdatascience.com/multi-class-text-classification-with-deep-learning-using-bert-b59ca2f5c613)<jupyter_code>def evaluate(model, dataloader):
  model.eval()

  loss_val_total = 0
  predictions, true_vals = [], []
  
  for batch in dataloader:
      
      # convert data to CUDA
      batch = tuple(b.to(device) for b in batch)
      
      inputs = {
          'input_ids':      batch[0],
          'attention_mask': batch[1],
          'labels':         batch[2],
      }

      with torch.no_grad():        
          outputs = model(**inputs) # get predictions
          
      loss = outputs[0]
      logits = outputs[1]
      loss_val_total += loss.item()

      logits = logits.detach().cpu().numpy()
      label_ids = inputs['labels'].cpu().numpy()
      predictions.append(logits)
      true_vals.append(label_ids)
  
  loss_val_avg = loss_val_total/len(dataloader) 
  
  predictions = np.concatenate(predictions, axis=0)
  true_vals = np.concatenate(true_vals, axis=0)
          
  return loss_val_avg, predictions, true_vals
import random

seed_val = 42
random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)

device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model.to(device)
print(device)
torch.cuda.get_device_name(0)
def train(model, epochs, train_dataloader, test_dataloader, optimizer, scheduler):
  losses_train = []
  losses_test = []
  aucs_test = []

  for epoch in tqdm(range(1, epochs+1)): # use tqdm for a progress bar
    model.train() # enter training mode
    loss_train_total = 0

    progress_bar = tqdm(train_dataloader, desc=f'Epoch {epoch}', leave=False, disable=False)
    for batch in progress_bar:
        model.zero_grad()
        
        # get CUDA data
        batch = tuple(b.to(device) for b in batch)
        
        inputs = {
            'input_ids':      batch[0],
            'attention_mask': batch[1],
            'labels':         batch[2],
        }

        outputs = model(**inputs) # evaluate
        
        # for reference, we are using cross-entropy loss here,
        # as implemented in https://huggingface.co/transformers/_modules/transformers/modeling_bert.html
        loss = outputs[0]
        loss_train_total += loss.item()
        loss.backward() # do backprop

        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)

        optimizer.step()
        scheduler.step()
        
        progress_bar.set_postfix({'training_loss': '{:.3f}'.format(loss.item()/len(batch))})
          
        
    torch.save(model.state_dict(), f'/content/drive/MyDrive/finetuned_BERT_epoch_{epoch}.model')
        
    tqdm.write(f'\nEpoch {epoch}')
    
    loss_train_avg = loss_train_total/len(train_dataloader)            
    tqdm.write(f'Training loss: {loss_train_avg}')
    losses_train.append(loss_train_avg)
    
    val_loss, predictions, true_vals = evaluate(model, test_dataloader)
    auc = auc_score(predictions, true_vals)
    losses_test.append(val_loss)
    aucs_test.append(auc)
    tqdm.write(f'Testing loss: {val_loss}')
    tqdm.write(f'AUC: {auc}')
    
  return losses_train, losses_test, aucs_test
losses_train, losses_test, aucs_test = train(model, 
                                             EPOCHS, 
                                             train_dataloader, 
                                             test_dataloader, 
                                             optimizer, 
                                             scheduler)
x = np.arange(1, 6)
plt.plot(x, losses_train, label='Training loss')
plt.plot(x, losses_test, label='Testing loss')
plt.plot(x, aucs_test, label='Testing AUC')

plt.xlabel('Epoch')
plt.legend()
_, preds, true_vals = evaluate(model, test_dataloader)
print(f"Accuracy: {accuracy_score(true_vals, np.argmax(preds, axis=1))}")
sns.heatmap(confusion_matrix(true_vals, np.argmax(preds, axis=1)), annot=True, fmt="d")<jupyter_output>Accuracy: 0.7492957746478873
{'fraction_non_alphanumeric': 0.11075819672131147, 'fraction_numerical': 0.009221311475409836, 'mean_word_length': 2.8826571201272873, 'pattern_counts': {'":': 2, '<': 11, '<?xml version=': 0, '>': 13, 'https://': 4, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '216983', 'license': 'MIT', 'n_tokens_mistral': 3279, 'n_tokens_neox': 3154, 'n_words': 554, 'path': 'models/TweetClassification_v2.ipynb', 'repo_name': 'mayankshouche/DSLabFinalProject', 'size': '9760'}