<jupyter_start><jupyter_text>TextAttack End-to-EndIn this experiment we will conduct training, evaluating, and attacking a model using TextAttack. In particular, we will use a pretrained distilBert model on the OPP-115 dataset for the task of segmenet classification.   Imports & Dependencies<jupyter_code>%%shell
rm -rf ./sample_data/
pip install transformers
pip install textattack
pip install torch==1.6 # need torch 1.6 for the augmenter to work
import torch
device = 'cuda' if torch.cuda.is_available() else 'cpu'
import pandas as pd
import numpy as np
from google.colab import drive
import transformers
from transformers import DistilBertTokenizerFast, DistilBertModel, BertConfig, DistilBertPreTrainedModel
from transformers import DistilBertForSequenceClassification, Trainer, TrainingArguments
from sklearn import metrics
from sklearn.model_selection import train_test_split
import matplotlib.pyplot as plt
import seaborn as sns
import seaborn as sns
from sklearn.metrics import multilabel_confusion_matrix
from sklearn.metrics import plot_confusion_matrix
from sklearn.metrics import classification_report, f1_score, accuracy_score
import pickle
import textattack<jupyter_output><empty_output><jupyter_text>Load & Prepare the OPP-115 Dataset from the Benchmark Paper ("Majority" Version) Mount Drive<jupyter_code>from google.colab import drive
drive.mount('/content/drive/')
train_set = pd.read_csv('/content/drive/MyDrive/828E_Ophir_Mollie/Polisis_Benchmark/datasets/Majority/train_dataset.csv', names=['segment', 'label'])
valid_set = pd.read_csv('/content/drive/MyDrive/828E_Ophir_Mollie/Polisis_Benchmark/datasets/Majority/validation_dataset.csv', names=['segment', 'label'])
test_set = pd.read_csv('/content/drive/MyDrive/828E_Ophir_Mollie/Polisis_Benchmark/datasets/Majority/test_dataset.csv', names=['segment', 'label'])
print('train set labels')
print('\n'.join([str(i)+'. '+ s for i, s in enumerate(train_set.label.unique())]))
print()
print('valid set labels')
print('\n'.join([str(i)+'. '+ s for i, s in enumerate(valid_set.label.unique())]))
print()
print('test set labels')
print('\n'.join([str(i)+'. '+ s for i, s in enumerate(test_set.label.unique())]))
print()<jupyter_output>train set labels
0. First Party Collection/Use
1. Third Party Sharing/Collection
2. User Choice/Control
3. Privacy contact information
4. Introductory/Generic
5. Practice not covered
6. Data Security
7. User Access, Edit and Deletion
8. Policy Change
9. Do Not Track
10. International and Specific Audiences
11. Data Retention

valid set labels
0. International and Specific Audiences
1. Privacy contact information
2. First Party Collection/Use
3. Third Party Sharing/Collection
4. Introductory/Generic
5. Practice not covered
6. Data Security
7. User Access, Edit and Deletion
8. Do Not Track
9. Data Retention
10. Policy Change
11. User Choice/Control

test set labels
0. Introductory/Generic
1. Data Retention
2. User Access, Edit and Deletion
3. Privacy contact information
4. First Party Collection/Use
5. Third Party Sharing/Collection
6. Data Security
7. User Choice/Control
8. Policy Change
9. International and Specific Audiences
10. Practice not covered
11. Do Not Track<jupyter_text>Convert text labels to numbers<jupyter_code>label2idx = dict([('First Party Collection/Use', 0),
             ('Third Party Sharing/Collection', 1),
             ('User Access, Edit and Deletion', 2),
             ('Data Retention', 3),
             ('Data Security', 4),
             ('International and Specific Audiences', 5),
             ('Do Not Track', 6),
             ('Policy Change', 7),
             ('User Choice/Control', 8),
             ('Introductory/Generic', 9),
             ('Practice not covered', 10),
             ('Privacy contact information', 11)])
idx2label = {value : key for (key, value) in label2idx.items()}
train_set.label = train_set.label.apply(lambda s: label2idx[s])
valid_set.label = valid_set.label.apply(lambda s: label2idx[s])
test_set.label = test_set.label.apply(lambda s: label2idx[s])
print('train set labels')
print('\n'.join([str(i)+'. '+ str(s) for i, s in enumerate(train_set.label.unique())]))
print()
print('valid set labels')
print('\n'.join([str(i)+'. '+ str(s) for i, s in enumerate(valid_set.label.unique())]))
print()
print('test set labels')
print('\n'.join([str(i)+'. '+ str(s) for i, s in enumerate(test_set.label.unique())]))
print()
tokenizer = DistilBertTokenizerFast.from_pretrained('bert-base-cased')
train_encodings = tokenizer(train_set.segment.to_list(), truncation=True, padding=True)
val_encodings = tokenizer(valid_set.segment.to_list(), truncation=True, padding=True)
test_encodings = tokenizer(test_set.segment.to_list(), truncation=True, padding=True)
class OppDataset(torch.utils.data.Dataset):
    def __init__(self, encodings, labels):
        self.encodings = encodings
        self.labels = labels

    def __getitem__(self, idx):
        item = {key: torch.tensor(val[idx]) for key, val in self.encodings.items()}
        item['labels'] = torch.tensor(self.labels[idx])
        return item

    def __len__(self):
        return len(self.labels)

train_dataset = OppDataset(train_encodings, train_set.label.to_list())
val_dataset = OppDataset(val_encodings, valid_set.label.to_list())
test_dataset = OppDataset(test_encodings, test_set.label.to_list())<jupyter_output><empty_output><jupyter_text>Training (original training data)<jupyter_code>output_dir = '/content/drive/MyDrive/828E_Ophir_Mollie/trained_models/non-robust_model_300_epochs_opp'
training_args = TrainingArguments(
    output_dir=output_dir, # output directory for model predictions and checkpoints
    num_train_epochs=7,              # total number of training epochs
    per_device_train_batch_size=16,  # batch size per device during training
    per_device_eval_batch_size=64,   # batch size for evaluation
    warmup_steps=500,                # number of warmup steps for learning rate scheduler
    weight_decay=0.01,               # strength of weight decay
    logging_steps=10,
    logging_dir=output_dir+'/logs',
    load_best_model_at_end=True, # will save/checkpoint the model after every evaluation
    evaluation_strategy="epoch", # will evaluate every epoch
    # DO NOT USE # label_names=label_names,  # DO NOT USE THIS --- THROWS ERROR
)

NUM_LABELS = len(idx2label)
model = DistilBertForSequenceClassification.from_pretrained(
    "distilbert-base-cased", 
    num_labels=NUM_LABELS,
    id2label=idx2label, # use to show text labels
    )

trainer = Trainer(
    model=model,                         # the instantiated 🤗 Transformers model to be trained
    args=training_args,                  # training arguments, defined above
    train_dataset=train_dataset,         # training dataset
    eval_dataset=val_dataset,            # evaluation dataset
)

# Start the training
trainer.train()
# Save model
model.save_pretrained(output_dir)
log = """Epoch	Training Loss	Validation Loss
1	1.831100	1.711555
2	1.167400	1.073737
3	1.067600	0.993681
4	0.670300	0.896062
5	0.713000	0.815847
6	0.508900	0.796503
7	0.476700	0.797068
"""
with open(output_dir+'/train_history_log', 'w') as f:
    f.write(log)

history = pd.read_csv(output_dir+'/train_history_log',sep='\t')

plt.style.use('seaborn-whitegrid')
plt.figure(figsize=(6,4))
plt.plot(history['Epoch'], history['Training Loss'], label='Training Loss')
plt.plot(history['Epoch'], history['Validation Loss'], label='Validation Loss')
plt.xlabel("No. Epoch")
plt.ylabel("Loss")
plt.title("Fine-tuned DistilBert")
plt.legend()
plt.savefig(output_dir+'/train_history_plot.png', dpi=300)
plt.show()
training_args = TrainingArguments(
    output_dir=output_dir, # output directory for model predictions and checkpoints
    num_train_epochs=100,              # total number of training epochs
    per_device_train_batch_size=16,  # batch size per device during training
    per_device_eval_batch_size=64,   # batch size for evaluation
    warmup_steps=500,                # number of warmup steps for learning rate scheduler
    weight_decay=0.01,               # strength of weight decay
    logging_steps=10,
    logging_dir=output_dir+'/logs',
    load_best_model_at_end=True, # will save/checkpoint the model after every evaluation
    evaluation_strategy="epoch", # will evaluate every epoch
    # DO NOT USE # label_names=label_names,  # DO NOT USE THIS --- THROWS ERROR
)

model = DistilBertForSequenceClassification.from_pretrained(
    output_dir, 
    num_labels=NUM_LABELS,
    id2label=idx2label, # use to show text labels
    )

trainer = Trainer(
    model=model,                         # the instantiated 🤗 Transformers model to be trained
    args=training_args,                  # training arguments, defined above
    train_dataset=train_dataset,         # training dataset
    eval_dataset=val_dataset,            # evaluation dataset
)

# Start the training
trainer.train()
# Save model
model.save_pretrained(output_dir)<jupyter_output><empty_output><jupyter_text>Evaluation (original data)<jupyter_code>model = DistilBertForSequenceClassification.from_pretrained(
    output_dir, 
    num_labels=NUM_LABELS,
    id2label=idx2label, # use to show text labels
)
model.eval()
evaluator = Trainer(
    model=model,                         # the instantiated 🤗 Transformers model to be trained
    eval_dataset=val_dataset,             # evaluation dataset
)

eval_results = evaluator.evaluate()
print()
for key in ['eval_loss']:
    value = eval_results[key]
    if type(value) != type(np.array([])):
        print(f'{key}:', value)
    else:
        print(f'{key}:\n')
        for lab, val in zip(label_names, value): 
            print(f'{lab}:', val)
    print()
model = DistilBertForSequenceClassification.from_pretrained(
    output_dir, 
    num_labels=len(idx2label),
    id2label=idx2label, # use to show text labels
)
model.eval()
evaluator = Trainer(
    model=model,                         # the instantiated 🤗 Transformers model to be trained
    eval_dataset=val_dataset,             # evaluation dataset
)

eval_results = evaluator.predict(val_dataset)
model.to('cpu')

# produce one-hot preds 
y_true = eval_results[1].tolist()
y_pred = eval_results[0].argmax(axis=-1).tolist()

label_names = list(label2idx.keys())
# Print and save classification report
print('Validation Flat Accuracy: ', accuracy_score(y_true, y_pred),'\n')

clf_report = classification_report(y_true, y_pred, target_names=label_names)
pickle.dump(clf_report, open(output_dir+'/classification_report.txt','wb')) #save report
print(clf_report)<jupyter_output>Validation Flat Accuracy:  0.7376543209876543 

                                      precision    recall  f1-score   support

          First Party Collection/Use       0.75      0.72      0.74       176
      Third Party Sharing/Collection       0.75      0.81      0.78       158
      User Access, Edit and Deletion       0.69      0.75      0.72        24
                      Data Retention       0.67      0.29      0.40        14
                       Data Security       0.71      0.81      0.76        31
International and Specific Audiences       0.81      0.95      0.88        41
                        Do Not Track       0.75      0.50      0.60         6
                       Policy Change       0.74      0.80      0.77        25
                 User Choice/Control       0.65      0.65      0.65        48
                Introductory/Generic       0.79      0.72      0.75        72
                Practice not covered       0.44      0.38      0.41        21
         Privac[...]<jupyter_text>AttackFinally, let's attack our fine-tuned model. we will create an adversarial version of the validation set and show that the accuracy decreases significantly.<jupyter_code>adv_valid_set = valid_set.copy()
augmenter = textattack.augmentation.recipes.WordNetAugmenter(
    pct_words_to_swap=0.1, 
    transformations_per_example=1
    )
from tqdm import tqdm
for i, s in enumerate(tqdm(adv_valid_set.segment)):
    adv_valid_set.iloc[i].segment = augmenter.augment(s)[0]

adv_val_encodings = tokenizer(adv_valid_set.segment.to_list(), truncation=True, padding=True)
adv_val_dataset = OppDataset(adv_val_encodings, adv_valid_set.label.to_list())
adv_valid_set.to_csv(output_dir+'/adv_valid_set.csv')
valid_set.head()
adv_valid_set.head()<jupyter_output><empty_output><jupyter_text>--- Now let's evaluate the (non-robust) model adversarial validation data<jupyter_code>model = DistilBertForSequenceClassification.from_pretrained(
    output_dir, 
    num_labels=len(idx2label),
    id2label=idx2label, # use to show text labels
)
model.eval()
evaluator = Trainer(
    model=model,                         # the instantiated 🤗 Transformers model to be trained
    eval_dataset=adv_val_dataset,             # evaluation dataset
)

eval_results = evaluator.predict(adv_val_dataset)
model.to('cpu')

# produce one-hot preds 
y_true = eval_results[1].tolist()
y_pred = eval_results[0].argmax(axis=-1).tolist()

label_names = list(label2idx.keys())
# Print and save classification report
print('Validation Flat Accuracy: ', accuracy_score(y_true, y_pred),'\n')

clf_report = classification_report(y_true, y_pred, target_names=label_names)
pickle.dump(clf_report, open(output_dir+'/classification_report.txt','wb')) #save report
print(clf_report)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10176726074584394, 'fraction_numerical': 0.029279616594278867, 'mean_word_length': 3.700809574093629, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 27, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '42802', 'license': 'MIT', 'n_tokens_mistral': 4425, 'n_tokens_neox': 4147, 'n_words': 1062, 'path': 'Attacking distilBERT Trained on OPP-115.ipynb', 'repo_name': 'Ophir-Gal/robust-lang-models', 'size': '13354'}