<jupyter_start><jupyter_text>Binomial Logistic Regression<jupyter_code>from sklearn.linear_model import LogisticRegression

# 2. instantiate a logistic regression model
lr = LogisticRegression()

lr.fit(x_train, np.ravel(y_train))
y_pred_lr = lr.predict(x_test)
print ("Accuracy is: ", metrics.accuracy_score(y_test, y_pred_lr))
print("Mean Squared Error is:", np.sqrt(mean_squared_error(y_test, y_pred_lr)))
lr_cm = metrics.confusion_matrix(y_test, y_pred_lr)
print (lr_cm)
labels = np.array(lr_cm)
sns.heatmap(lr_cm, annot=labels, fmt = '')
plt.show()
print(metrics.classification_report(y_test, y_pred_lr))
# Calculate predicted probabilities for test data 
y_pred_prob_lr = lr.predict_proba(x_test)[:, 1]
metrics.roc_auc_score(y_test, y_pred_prob_lr)<jupyter_output><empty_output><jupyter_text>Naive Bayes<jupyter_code>from nltk import classify
from nltk import NaiveBayesClassifier
from sklearn.naive_bayes import MultinomialNB

nb = MultinomialNB()# one of the two classic naive Bayes variants used in text classification

%time nb.fit(x_train, np.ravel(y_train))
y_pred_class = nb.predict(x_test)
print("Accuracy is:", metrics.accuracy_score(y_test, y_pred_class))
print("Mean Squared Error is:", np.sqrt(mean_squared_error(y_test, y_pred_class)))
nb_cm = metrics.confusion_matrix(y_test, y_pred_class)
print(nb_cm)
labels =  np.array(nb_cm)
sns.heatmap(nb_cm, annot=labels, fmt = '')
plt.show()
print(metrics.classification_report(y_test, y_pred_class))
# Calculate predicted probabilities for test data 
y_pred_prob_nb = nb.predict_proba(x_test)[:, 1]
metrics.roc_auc_score(y_test, y_pred_prob_nb)
#Number of times token appears in each True article
true_token = nb.feature_count_[0, :]

#Number of times token appears in each Fake article
fake_token = nb.feature_count_[1, :]
tokens = pd.DataFrame({'token':x_train.columns, 'true':true_token, 'fake':fake_token}).set_index('token')
tokens.head()
# add 1 to true and fake counts to avoid dividing by 0
tokens['true'] = tokens.true + 1
tokens['fake'] = tokens.fake + 1
# convert the true and fake counts into frequencies
tokens['true'] = tokens.true / nb.class_count_[0]
tokens['fake'] = tokens.fake / nb.class_count_[1]
# calculate the ratio of fake to true for each token
tokens['fake/true ratio'] = tokens.fake / tokens.true
tokens.sort_values('fake/true ratio', ascending=False).head(10)
tokens.sort_values('fake/true ratio', ascending=True).head(10)<jupyter_output><empty_output><jupyter_text>Support Vector Machines<jupyter_code>from sklearn.svm import SVC

#Linear kernel fits decently well, decided not to use another kernel because of parsimony 
#and because linear has the lowest risk of overfitting
svc = SVC(kernel='linear', random_state=1) 
svc.fit(x_train, np.ravel(y_train))
y_pred_svm = svc.predict(x_test)
print("Accuracy is:", metrics.accuracy_score(y_test, y_pred_svm))
print("Mean Squared Error is:", np.sqrt(mean_squared_error(y_test, y_pred_svm)))
print(metrics.classification_report(y_test, y_pred_svm))
svm_cm = metrics.confusion_matrix(y_test, y_pred_svm)
print(svm_cm)
labels =  np.array(svm_cm)
sns.heatmap(svm_cm, annot=labels, fmt = '')
plt.show()
print (metrics.roc_auc_score(y_test, y_pred_svm))<jupyter_output>0.8976500762898849<jupyter_text>Random Forest<jupyter_code>#Tried depths greater and less than 60, 60 was the best. 
#Tried estimators greater and less than 200 but 200 was the best

from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import GridSearchCV

rf = RandomForestClassifier(random_state = 1)

param_grid = {
    'n_estimators': [200],
    'max_depth': [50, 60, 70]
    }

%time grid_search_rf = GridSearchCV(estimator = rf, param_grid = param_grid, cv = 5)
grid_search_rf.fit(x_train, np.ravel(y_train))
grid_search_rf.best_params_
y_pred_rf = grid_search_rf.predict(x_test)
print("Accuracy is:", metrics.accuracy_score(y_test, y_pred_rf))
print("Mean Squared Error is:", np.sqrt(mean_squared_error(y_test, y_pred_rf)))
rf_cm = metrics.confusion_matrix(y_test, y_pred_rf)
print(rf_cm)
print(metrics.classification_report(y_test, y_pred_rf))
labels = np.array(rf_cm)
sns.heatmap(rf_cm, annot=labels, fmt = '')
plt.show()
# Calculate predicted probabilities for test data 
y_pred_prob_rf = grid_search_rf.predict_proba(x_test)[:, 1]
metrics.roc_auc_score(y_test, y_pred_prob_rf)
feature_importances = pd.Series(grid_search_rf.best_estimator_.feature_importances_, index=x_train.columns)
feature_importances.nlargest(10).plot(kind='barh')<jupyter_output><empty_output><jupyter_text>Neural Network<jupyter_code>import sklearn.model_selection as ms

#Create new train-test set
X = pd.concat([true["text"], fake["text"]])
true["label"] = [0 for x in true["label"]]
fake["label"] = [1 for x in fake["label"]]
y = pd.concat([true["label"], fake["label"]])

X_train, X_test, y_train, y_test = ms.train_test_split(X, y, test_size = 0.2, random_state=1)
train_data = pd.concat([X_train, y_train],axis = 1).reset_index(drop=True)
test_data = pd.concat([X_test, y_test],axis = 1).reset_index(drop=True)
X = pd.DataFrame(X)
articles = X["text"].tolist()
y = pd.DataFrame(y)
labels = y["label"].tolist()
#Create new tokenizer because it's best to tokenize with the same tokenizer 
#that BERT was trained on instead of self-defined one 
from transformers import BertTokenizer
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)
# Tokenize all of the articles and map the tokens to thier word IDs.
input_ids = []
attention_masks = []

for article in articles:
    encoded_dict = tokenizer.encode_plus(
                        str(article),                      #Article to encode
                        add_special_tokens = True, #Add [CLS] to start, [SEP] to end
                        max_length = 64,           #Pad & truncate all sentences
                        pad_to_max_length = True,
                        return_attention_mask = True,   #Create masks
                        return_tensors = "pt",     # Return pytorch tensors
                   )
    
    #Add the encoded sentence to the list    
    input_ids.append(encoded_dict['input_ids'])
    
    # Add attention mask (simply differentiates padding from non-padding)
    attention_masks.append(encoded_dict['attention_mask'])

# Convert the lists into tensors
input_ids = torch.cat(input_ids, dim=0) #encoded sentence
attention_masks = torch.cat(attention_masks, dim=0) #mask
labels = torch.tensor(labels) #labels (fake/true)
from torch.utils.data import TensorDataset, random_split

# Combine the training inputs into a TensorDataset
dataset = TensorDataset(input_ids, attention_masks, labels)

# Create a 70-10-20 train-validation-test split

# Calculate the number of samples to include in each set.
train_size = int(0.7 * len(dataset))
val_size = int(0.1 * len(dataset)) + 1 #add 1 because int() causes 1 row to be left out
test_size = int(0.2 * len(dataset)) 

train_dataset, val_dataset, test_dataset = random_split(dataset, [train_size, val_size, test_size])

print('{:>5,} training samples'.format(train_size))
print('{:>5,} validation samples'.format(val_size))
print('{:>5,} test samples'.format(test_size))
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler

#BERT developers recommend 16 or 32 batch_size
batch_size = 16

train_dataloader = DataLoader(
            train_dataset,  
            sampler = RandomSampler(train_dataset), # Select batches randomly
            batch_size = batch_size)

validation_dataloader = DataLoader(
            val_dataset, 
            sampler = SequentialSampler(val_dataset), # Pull out batches sequentially, since order doesn't matter here
            batch_size = batch_size)

test_dataloader = DataLoader(
            test_dataset, 
            sampler = SequentialSampler(test_dataset), # Pull out batches sequentially, since order doesn't matter here
            batch_size = batch_size)
from transformers import BertForSequenceClassification, AdamW, BertConfig

# Load BertForSequenceClassification, the pretrained BERT model with a single linear classification layer  
# Don't output all layers due to memory limits
model = BertForSequenceClassification.from_pretrained(
    "bert-base-uncased", # Use the 12-layer BERT model, with an uncased vocab
    num_labels = 2, #binary classification  
    output_attentions = False, 
    output_hidden_states = False)

#run model on the GPU
model.cuda()
#Use Adam optimizer
optimizer = AdamW(model.parameters(),
                  lr = 0.00002,
                  eps = 1e-8) # default is 1e-8
from transformers import get_linear_schedule_with_warmup

# BERT authors recommend between 2 and 4 epochs
epochs = 4

# number of batches x number of epochs
total_steps = len(train_dataloader) * epochs

# Create the learning rate scheduler to reduce the learning rate as training epochs increase
scheduler = get_linear_schedule_with_warmup(optimizer, 
                                            num_warmup_steps = 0, #default
                                            num_training_steps = total_steps)
def accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=1).flatten()
    labels_flat = labels.flatten()
    return metrics.accuracy_score(pred_flat, labels_flat)
import time
import datetime

#Takes a time in seconds and returns a string hh:mm:ss
def format_time(elapsed):
    
    # Round to the nearest second.
    elapsed_rounded = int(round((elapsed)))
    
    # Format as hh:mm:ss
    return str(datetime.timedelta(seconds=elapsed_rounded))

import random

#Use a seed to ensure reproductibility
seed_val = 22

random.seed(seed_val)
np.random.seed(seed_val)
torch.manual_seed(seed_val)
torch.cuda.manual_seed_all(seed_val)

training_stats = []

# Measure the total training time for the whole run.
total_t0 = time.time()


for epoch_i in range(0, epochs):
    
    print("")
    print('======== Epoch {:} / {:} ========'.format(epoch_i + 1, epochs))

    # Measure how long the training epoch takes
    t0 = time.time()

    # Reset the total loss for this epoch
    total_train_loss = 0

    model.train()

    for step, batch in enumerate(train_dataloader):

        # Progress update every 40 batches
        if step % 40 == 0 and not step == 0:
            # Calculate elapsed time in minutes
            elapsed = format_time(time.time() - t0)
            
            # Report progress
            print('  Batch {:>5,}  of  {:>5,}.    Elapsed: {:}.'.format(step, len(train_dataloader), elapsed))

        b_input_ids = batch[0].to(device) #copy each tensor to the GPU
        b_input_mask = batch[1].to(device) #copy each tensor to the GPU
        b_labels = batch[2].to(device) #copy each tensor to the GPU

        model.zero_grad()        

        # Perform forward pass 
        # Returns the loss (because we provided labels) and the "logits" - the model
        # outputs prior to activation
        loss, logits = model(b_input_ids, 
                             token_type_ids=None, 
                             attention_mask=b_input_mask, 
                             labels=b_labels)

        total_train_loss += loss.item()
        loss.backward()

        # Clip the norm of the gradients to 1.0
        # This is to help prevent the "exploding gradients" problem
        torch.nn.utils.clip_grad_norm_(model.parameters(), 1.0)
        optimizer.step()
        scheduler.step() # Update the learning rate

    avg_train_loss = total_train_loss / len(train_dataloader)            
    
    # Measure how long this epoch took
    training_time = format_time(time.time() - t0)

    print("")
    print("  Average training loss: {0:.2f}".format(avg_train_loss))
    print("  Training epcoh took: {:}".format(training_time))

    print("")
    print("Running Validation...")

    t0 = time.time()

    model.eval()

    # Tracking variables 
    total_eval_accuracy = 0
    total_eval_loss = 0
    nb_eval_steps = 0

    for batch in validation_dataloader:
        
        b_input_ids = batch[0].to(device) #copy each tensor to the GPU
        b_input_mask = batch[1].to(device) #copy each tensor to the GPU
        b_labels = batch[2].to(device) #copy each tensor to the GPU
        
        # Tell pytorch not to bother with constructing the compute graph during
        # the forward pass, since this is only needed for backprop (training).
        with torch.no_grad():        
            (loss, logits) = model(b_input_ids, 
                                   token_type_ids=None, 
                                   attention_mask=b_input_mask,
                                   labels=b_labels)         
        total_eval_loss += loss.item()

        # Move logits and labels to CPU
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to('cpu').numpy()

        # Calculate the accuracy for this batch of test sentences, and
        # accumulate it over all batches.
        total_eval_accuracy += accuracy(logits, label_ids)
        
    # Report the final accuracy for this validation run
    avg_val_accuracy = total_eval_accuracy / len(validation_dataloader)
    print("  Accuracy: {0:.2f}".format(avg_val_accuracy))

    # Calculate the average loss over all of the batches
    avg_val_loss = total_eval_loss / len(validation_dataloader)
    
    # Measure how long the validation run took
    validation_time = format_time(time.time() - t0)
    
    print("  Validation Loss: {0:.2f}".format(avg_val_loss))
    print("  Validation took: {:}".format(validation_time))

    # Record all statistics from this epoch.
    training_stats.append(
        {
            'epoch': epoch_i + 1,
            'Training Loss': avg_train_loss,
            'Valid. Loss': avg_val_loss,
            'Valid. Accur.': avg_val_accuracy,
            'Training Time': training_time,
            'Validation Time': validation_time
        }
    )

print("")
print("Training complete!")

print("Total training took {:} (h:mm:ss)".format(format_time(time.time()-total_t0)))
pd.set_option('precision', 2)

df_stats = pd.DataFrame(data=training_stats)

df_stats = df_stats.set_index('epoch')

df_stats
sns.set(style='darkgrid')

sns.set(font_scale=0.5)
plt.rcParams["figure.figsize"] = (12,6)

plt.plot(df_stats['Training Loss'], 'b-o', label="Training")
plt.plot(df_stats['Valid. Loss'], 'g-o', label="Validation")

plt.title("Training & Validation Loss")
plt.xlabel("Epoch")
plt.ylabel("Loss")
plt.legend()
plt.xticks([1, 2, 3, 4])

plt.show()
model.eval()

predictions , true_labels = [], []

for batch in test_dataloader:
  batch = tuple(t.to(device) for t in batch)
  
  b_input_ids, b_input_mask, b_labels = batch
  
  # Telling the model not to compute or store gradients, saving memory and 
  # speeding up prediction
  with torch.no_grad():
      # Forward pass, calculate logit predictions
      outputs = model(b_input_ids, token_type_ids=None, 
                      attention_mask=b_input_mask)

  logits = outputs[0]

  # Move logits and labels to CPU
  logits = logits.detach().cpu().numpy()
  label_ids = b_labels.to('cpu').numpy()
  
  # Store predictions and true labels
  predictions.append(logits)
  true_labels.append(label_ids)
flat_predictions = np.concatenate(predictions, axis=0)

# For each sample, pick the label (0 or 1) with the higher score.
flat_predictions = np.argmax(flat_predictions, axis=1).flatten()

# Combine the correct labels for each batch into a single list.
flat_true_labels = np.concatenate(true_labels, axis=0)

print("Accuracy is:", metrics.accuracy_score(flat_predictions,flat_true_labels))
print("Mean Squared Error is:", np.sqrt(mean_squared_error(flat_predictions, flat_true_labels)))
b_cm = metrics.confusion_matrix(flat_predictions,flat_true_labels)
print(b_cm)
labels = np.array(b_cm)
sns.heatmap(b_cm, annot=labels, fmt = '')
plt.show()
print (metrics.roc_auc_score(flat_predictions, flat_true_labels))
print(metrics.classification_report(flat_predictions, flat_true_labels))<jupyter_output>0.9927025409172056
              precision    recall  f1-score   support

           0       0.99      0.99      0.99      4483
           1       0.99      0.99      0.99      4797

    accuracy                           0.99      9280
   macro avg       0.99      0.99      0.99      9280
weighted avg       0.99      0.99      0.99      9280
{'fraction_non_alphanumeric': 0.10292311471382952, 'fraction_numerical': 0.015045443380004913, 'mean_word_length': 3.593794076163611, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '353481', 'license': 'MIT', 'n_tokens_mistral': 5514, 'n_tokens_neox': 5176, 'n_words': 1334, 'path': 'Modeling.ipynb', 'repo_name': 'nabaruns/DLNLP', 'size': '16284'}