<jupyter_start><jupyter_text>Initial Setups  (Google Colab use only)<jupyter_code># Use Google Colab
use_colab = False

# Is this notebook running on Colab?
# If so, then google.colab package (github.com/googlecolab/colabtools)
# should be available in this environment

# Previous version used importlib, but we could do the same thing with
# just attempting to import google.colab
try:
    from google.colab import drive
    colab_available = True
except:
    colab_available = False

if use_colab and colab_available:
    drive.mount('/content/drive/')

    # cd to the appropriate working directory under my Google Drive
    %cd '/content/drive/My Drive/cs696ds_lexalytics/Prompting Experiments'
    
    # Install packages specified in requirements
    !pip install -r requirements.txt
    
    # List the directory contents
    !ls<jupyter_output><empty_output><jupyter_text>Experiment parameters<jupyter_code># We will use the following string ID to identify this particular (training) experiments
# in directory paths and other settings
experiment_id = 'prompt_logit_atsc_laptops_gpt-2_amazon_single_i_felt'

# Random seed
random_seed = 696

# path to pretrained MLM model folder or the string "gpt2"
lm_model_path = 'gpt2'
#lm_model_path = 'gpt2_amazon'
# Proportion to be reserved for validation
validation_dataset_proportion = 0.2


# Prompts to be added to the end of each review text
# Note: pseudo-labels for each prompt should be given in the order of (positive), (negative), (neutral)
sentiment_prompts = [
    {"prompt": " I felt the {aspect} was", "labels": [" good", " bad", " ok"]},
    {"prompt": " The {aspect} made me feel", "labels": [" good", " bad", " indifferent"]},
    {"prompt": " The {aspect} is ", "labels": [" good", " bad", " ok"]}
]
# sentiment_prompts = [
#     {"prompt": "I felt the {aspect} was ", "labels": ["good", "bad", "ok"]}
# ]

# Training settings
training_epochs = 20
training_batch_size = 15
training_learning_rate = 2e-5
training_weight_decay = 0.01
training_warmup_steps_duration = 0.15
training_hard_restart_num_cycles = 3
training_max_grad_norm = 1.0
training_best_model_criterion = 'train_loss'
training_domain = 'laptops' # 'laptops', 'restaurants', 'joint'
training_dataset_proportion = 1.0
training_dataset_few_shot_size = None
training_dataset_few_shot_size = None

training_lm_freeze = False

validation_enabled = False
validation_dataset_proportion = 0.2
validation_batch_size = 15

testing_batch_size = 15
testing_domain = 'laptops' # 'laptops', 'restaurants', 'joint'
# Batch size adjustment for multiple prompts.
training_batch_size = training_batch_size // len(sentiment_prompts)
print("Experiment ID:", experiment_id)
print("training_batch_size", training_batch_size)
print("sentiment_prompts", sentiment_prompts)<jupyter_output><empty_output><jupyter_text>Package imports<jupyter_code>import sys
import os
import random
import shutil
import copy
import inspect
import json

import numpy as np
import torch
import transformers
import datasets
import sklearn.metrics
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sn
import tqdm

current_dir = os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))
parent_dir = os.path.dirname(current_dir)
sys.path.append(parent_dir)

import utils

# Random seed settings
random.seed(random_seed)
np.random.seed(random_seed)
torch.manual_seed(random_seed)

# cuBLAS reproducibility
# https://docs.nvidia.com/cuda/cublas/index.html#cublasApi_reproducibility
os.environ['CUBLAS_WORKSPACE_CONFIG'] = ":4096:8"
torch.set_deterministic(True)
torch.manual_seed(random_seed)

# Print version information
print("Python version: " + sys.version)
print("NumPy version: " + np.__version__)
print("PyTorch version: " + torch.__version__)
print("Transformers version: " + transformers.__version__)<jupyter_output>Python version: 3.7.10 (default, Feb 20 2021, 21:17:23) 
[GCC 7.5.0]
NumPy version: 1.19.5
PyTorch version: 1.7.1
Transformers version: 4.3.3<jupyter_text>PyTorch GPU settings<jupyter_code>if torch.cuda.is_available():    
    torch_device = torch.device('cuda')

    # Set this to True to make your output immediately reproducible
    # Note: https://pytorch.org/docs/stable/notes/randomness.html
    torch.backends.cudnn.deterministic = True
    
    # Disable 'benchmark' mode: Set this False if you want to measure running times more fairly
    # Note: https://discuss.pytorch.org/t/what-does-torch-backends-cudnn-benchmark-do/5936
    torch.backends.cudnn.benchmark = False
    
    # Faster Host to GPU copies with page-locked memory
    use_pin_memory = True
    
    # Number of compute devices to be used for training
    training_device_count = torch.cuda.device_count()

    # CUDA libraries version information
    print("CUDA Version: " + str(torch.version.cuda))
    print("cuDNN Version: " + str(torch.backends.cudnn.version()))
    print("CUDA Device Name: " + str(torch.cuda.get_device_name()))
    print("CUDA Capabilities: "+ str(torch.cuda.get_device_capability()))
    print("Number of CUDA devices: "+ str(training_device_count))
    
else:
    torch_device = torch.device('cpu')
    use_pin_memory = False
    
    # Number of compute devices to be used for training
    training_device_count = 1

print()
print("PyTorch device selected:", torch_device)<jupyter_output>CUDA Version: 10.2
cuDNN Version: 7605
CUDA Device Name: Tesla T4
CUDA Capabilities: (7, 5)
Number of CUDA devices: 1

PyTorch device selected: cuda<jupyter_text>Prepare Datasets for Prompt-based Classifier  Load the SemEval dataset<jupyter_code>try:
    # Load semeval for both domains
    laptops_dataset = datasets.load_dataset(
        os.path.abspath('../dataset_scripts/semeval2014_task4/semeval2014_task4.py'),
        data_files={
            'test': '../dataset_files/semeval_2014/Laptops_Test_Gold.xml',
            'train': '../dataset_files/semeval_2014/Laptop_Train_v2.xml',
        },
        cache_dir='../dataset_cache')

    restaurants_dataset = datasets.load_dataset(
        os.path.abspath('../dataset_scripts/semeval2014_task4/semeval2014_task4.py'),
        data_files={
            'test': '../dataset_files/semeval_2014/Restaurants_Test_Gold.xml',
            'train': '../dataset_files/semeval_2014/Restaurants_Train_v2.xml',
        },
        cache_dir='../dataset_cache')
except:
    # Load semeval for both domains
    laptops_dataset = datasets.load_dataset(
        os.path.abspath('../dataset_scripts/semeval2014_task4/semeval2014_task4.py'),
        data_files={
            'test': '../dataset_files/semeval_2014/Laptops_Test_Gold.xml',
            'train': '../dataset_files/semeval_2014/Laptop_Train_v2.xml',
        })

    restaurants_dataset = datasets.load_dataset(
        os.path.abspath('../dataset_scripts/semeval2014_task4/semeval2014_task4.py'),
        data_files={
            'test': '../dataset_files/semeval_2014/Restaurants_Test_Gold.xml',
            'train': '../dataset_files/semeval_2014/Restaurants_Train_v2.xml',
        })
# The dataset chosen for training/testing
print("Training domain:", training_domain)
print("Testing domain:", testing_domain)

if training_domain == 'laptops':
    train_set = laptops_dataset['train']
elif training_domain == 'restaurants':
    train_set = restaurants_dataset['train']
elif training_domain == 'joint':
    train_set = laptops_dataset['train'] + restaurants_dataset['train']

if testing_domain == 'laptops':
    test_set = laptops_dataset['test']
elif testing_domain == 'restaurants':
    test_set = restaurants_dataset['test']
elif testing_domain == 'joint':
    test_set = laptops_dataset['test'] + restaurants_dataset['test']<jupyter_output><empty_output><jupyter_text>Train-validation split<jupyter_code># Training set size after validation split
if validation_enabled:
    new_train_dataset_size = int(len(train_set) * (1 - validation_dataset_proportion))
    new_valid_dataset_size = len(train_set) - new_train_dataset_size
    print("Training dataset after split:", new_train_dataset_size)
    print("Validation dataset after split:", new_train_dataset_size)
else:
    new_train_dataset_size = len(train_set)
    print("Training dataset size:", new_train_dataset_size)
train_set = train_set.shuffle(seed=random_seed)

new_train_set = train_set.select(indices=np.arange(new_train_dataset_size))

if validation_enabled:
    new_valid_set = train_set.select(
        indices=np.arange(
            new_train_dataset_size,
            new_train_dataset_size + new_valid_dataset_size))
# For few shot experiments, select first few examples in new_train_set for training
if training_dataset_few_shot_size is not None:
    new_train_set = new_train_set.select(indices=np.arange(training_dataset_few_shot_size))
    
    print("Few-shot training dataset size:", len(new_train_set))
print(new_train_set[0])<jupyter_output>{'aspect': 'loads', 'sentiment': 2, 'text': 'Once I removed all the software the laptop loads in 15-20 seconds.'}<jupyter_text>Zero-shot ATSC with Prompts + Logistic Regression Head  Load the pretrained LM<jupyter_code># Load pretrained language model
lm = transformers.AutoModelForCausalLM.from_pretrained(lm_model_path)
try:
    tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2', cache_dir='./gpt2_cache')
except:
    tokenizer = transformers.AutoTokenizer.from_pretrained('gpt2')
tokenizer.pad_token = tokenizer.eos_token<jupyter_output><empty_output><jupyter_text>Define a new Model with CLM output head<jupyter_code># Encode the pseudo-label words for each sentiment class
sentiment_word_ids = []

for sp in sentiment_prompts:
    sentiment_word_ids.append(
        [tokenizer.convert_tokens_to_ids(tokenizer.tokenize(w))[0] for w in sp['labels']])

print(sentiment_word_ids)


classifier_model = utils.MultiPromptLogitSentimentClassificationHead(
    lm=lm,
    num_class=3,
    num_prompts=len(sentiment_prompts),
    pseudo_label_words=sentiment_word_ids,
    target_token_id=tokenizer.eos_token_id)

classifier_model = classifier_model.to(device=torch_device)
# layers in classifier_model.lm should be frozen
if training_lm_freeze:
    for param in lm.parameters():
        assert param.requires_grad == False

    # Make sure that classifier_model.linear is trainable
    for param in classifier_model.linear.parameters():
        assert param.requires_grad == True
train_dataloader = torch.utils.data.DataLoader(
    new_train_set, shuffle=True, batch_size=training_batch_size, pin_memory=use_pin_memory)

if validation_enabled:
    validation_dataloader = torch.utils.data.DataLoader(
        new_valid_set, batch_size=validation_batch_size, pin_memory=use_pin_memory)
# How many training steps would we have?
training_total_steps = len(train_dataloader) * training_epochs

print("There will be %d training steps." % training_total_steps)

# Let's have warmups for the first (training_warmup_steps_duration)% of steps.
training_warmup_steps = int(training_total_steps * training_warmup_steps_duration)

print("Warmup steps:", training_warmup_steps)<jupyter_output>There will be 46260 training steps.
Warmup steps: 6939<jupyter_text>Training settings<jupyter_code>loss_function = torch.nn.CrossEntropyLoss()

no_decay = ['bias', 'LayerNorm.weight']

optimizer_grouped_parameters = [
    {'params': [p for n, p in classifier_model.named_parameters() if not any(nd in n for nd in no_decay)], 'weight_decay': training_weight_decay},
    {'params': [p for n, p in classifier_model.named_parameters() if any(nd in n for nd in no_decay)], 'weight_decay': 0.0}
    ]

optimizer = transformers.AdamW(
    optimizer_grouped_parameters,
    lr=training_learning_rate,
    weight_decay=training_weight_decay)

scheduler = transformers.get_linear_schedule_with_warmup(
    optimizer,
    num_warmup_steps=training_warmup_steps,
    num_training_steps=training_total_steps
)

# The directory to save the best version of the head
trained_model_directory = os.path.join('..', 'trained_models', experiment_id)

shutil.rmtree(trained_model_directory, ignore_errors=True)
os.makedirs(trained_model_directory)
def compute_metrics(predictions, labels):
    preds = predictions.argmax(-1)

    precision, recall, f1, _ = sklearn.metrics.precision_recall_fscore_support(
        y_true=labels, y_pred=preds, labels=[0,1,2], average='macro')

    acc = sklearn.metrics.accuracy_score(labels, preds)

    return {
        'accuracy': acc,
        'f1': f1,
        'precision': precision,
        'recall': recall
    }<jupyter_output><empty_output><jupyter_text>Training loop<jupyter_code>best_epoch = -1

if training_best_model_criterion in ('train_loss', 'valid_loss'):
    best_score = float('inf')
else:
    best_score = -1

for epoch in tqdm.notebook.tqdm(range(int(training_epochs))):

    print("Training epoch %d" % epoch)
    print()

    classifier_model.train()

    for batch in tqdm.notebook.tqdm(train_dataloader):

        reviews_repeated = []
        prompts_populated = []

        for prompt in sentiment_prompts:
            reviews_repeated = reviews_repeated + batch["text"]

            for aspect in batch["aspect"]:
                prompts_populated.append(prompt['prompt'].format(aspect=aspect))

        batch_encoded = tokenizer(
            reviews_repeated, prompts_populated,
            padding='max_length', truncation='only_first', max_length=256,
            return_tensors='pt')
        
        batch_encoded = batch_encoded.to(torch_device)

        batch_label = batch["sentiment"]
        batch_label = batch_label.to(torch_device)

        optimizer.zero_grad()

        batch_output = classifier_model(batch_encoded)

        loss = loss_function(batch_output, batch_label)

        loss.backward()
        torch.nn.utils.clip_grad_norm_(classifier_model.parameters(), training_max_grad_norm)

        optimizer.step()
        scheduler.step()
        
    print("Epoch {}, Training Loss: {}".format(epoch, loss.item()))
    print()

    # Validate the model using val dataset
    if validation_enabled:
        with torch.no_grad():
            classifier_model.eval()

            print("Validation epoch %d" % epoch)
            print()

            predictions_val = torch.Tensor()
            labels_val = torch.Tensor()

            for batch_val in tqdm.notebook.tqdm(validation_dataloader):

                reviews_repeated = []
                prompts_populated = []

                for prompt in sentiment_prompts:
                    reviews_repeated = reviews_repeated + batch_val["text"]

                    for aspect in batch_val["aspect"]:
                        prompts_populated.append(prompt['prompt'].format(aspect=aspect))

                batch_val_encoded = tokenizer(
                    reviews_repeated, prompts_populated,
                    padding='max_length', truncation='only_first', max_length=256,
                    return_tensors='pt')

                batch_val_encoded.to(torch_device)

                batch_val_label = batch_val["sentiment"]

                batch_val_output = classifier_model(batch_val_encoded)

                batch_val_output = batch_val_output.to('cpu')

                predictions_val = torch.cat([predictions_val, batch_val_output])
                labels_val = torch.cat([labels_val, batch_val_label])

            # Compute metrics
            validation_loss = torch.nn.functional.cross_entropy(predictions_val, labels_val.long())
            validation_metrics = compute_metrics(predictions_val, labels_val)

            print("Validation Loss: {}, Validation Metrics: {}".format(validation_loss.item(), validation_metrics))
            print()

    if training_best_model_criterion == 'train_loss':
        epoch_score = loss.item()
    elif training_best_model_criterion == 'valid_loss':
        epoch_score = validation_loss.item()
        
    if training_best_model_criterion in ('train_loss', 'valid_loss'):
        better_model_found = epoch_score < best_score
    else:
        better_model_found = epoch_score > best_score

    # Save the current epoch's model if the validation loss is lower than the best known so far
    if better_model_found:
        if best_epoch != -1:
            try:
                os.remove(os.path.join(trained_model_directory, 'epoch_{}.pt'.format(best_epoch)))
            except:
                pass

        best_score = epoch_score
        best_epoch = epoch

        torch.save(
            classifier_model.state_dict(),
            os.path.join(trained_model_directory, 'epoch_{}.pt'.format(epoch)))<jupyter_output><empty_output><jupyter_text>Evaluation with in-domain test set<jupyter_code>test_dataloader = torch.utils.data.DataLoader(
    test_set, batch_size=testing_batch_size, pin_memory=use_pin_memory)
# Load the best found head weights
with torch.no_grad():
    print('Loading epoch {}'.format(best_epoch))

    classifier_model.load_state_dict(torch.load(
        os.path.join(trained_model_directory, 'epoch_{}.pt'.format(best_epoch)),
        map_location=torch_device))

    classifier_model.eval()

    predictions_test = torch.Tensor()
    labels_test = torch.Tensor()

    for batch_test in tqdm.notebook.tqdm(test_dataloader):

        reviews_repeated = []
        prompts_populated = []

        for prompt in sentiment_prompts:
            reviews_repeated = reviews_repeated + batch_test["text"]

            for aspect in batch_test["aspect"]:
                prompts_populated.append(prompt['prompt'].format(aspect=aspect))

        batch_test_encoded = tokenizer(
            reviews_repeated, prompts_populated,
            padding='max_length', truncation='only_first', max_length=256,
            return_tensors='pt')

        batch_test_encoded.to(torch_device)

        batch_test_label = batch_test["sentiment"]

        batch_test_output = classifier_model(batch_test_encoded)

        batch_test_output = batch_test_output.to('cpu')

        predictions_test = torch.cat([predictions_test, batch_test_output])
        labels_test = torch.cat([labels_test, batch_test_label])

    # Compute metrics
    test_metrics = compute_metrics(predictions_test, labels_test)

    print(test_metrics)
    
    # Save test_metrics into a file for later processing
    with open(os.path.join(trained_model_directory, 'test_metrics.json'), 'w') as test_metrics_json:
        json.dump(test_metrics, test_metrics_json)
if best_epoch != -1:
    try:
        os.remove(os.path.join(trained_model_directory, 'epoch_{}.pt'.format(best_epoch)))
    except:
        pass<jupyter_output><empty_output><jupyter_text>Results visualization<jupyter_code># Calculate metrics and confusion matrix based upon predictions and true labels
cm = sklearn.metrics.confusion_matrix(labels_test.detach().numpy(), predictions_test.detach().numpy().argmax(-1))

df_cm = pd.DataFrame(
    cm,
    index=[i for i in ["positive", "negative", "neutral"]],
    columns=[i for i in ["positive", "negative", "neutral"]])

plt.figure(figsize=(10, 7))

ax = sn.heatmap(df_cm, annot=True)

ax.set(xlabel='Predicted Label', ylabel='True Label')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10109533043341544, 'fraction_numerical': 0.010900896179445521, 'mean_word_length': 4.056173820879703, 'pattern_counts': {'":': 9, '<': 46, '<?xml version=': 0, '>': 46, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 8}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '111817', 'license': 'Apache-2.0', 'n_tokens_mistral': 6194, 'n_tokens_neox': 5933, 'n_words': 1240, 'path': 'prompts_supervised_gpt-2/gpt2_prompt_logit_softmax_atsc.ipynb', 'repo_name': 'ronaldseoh/atsc_prompts', 'size': '19081'}