<jupyter_start><jupyter_text>MIT LicenseCopyright (c) 2020 <NAME>Permission is hereby granted, free of charge, to any person obtaining a copyof this software and associated documentation files (the "Software"), to dealin the Software without restriction, including without limitation the rightsto use, copy, modify, merge, publish, distribute, sublicense, and/or sellcopies of the Software, and to permit persons to whom the Software isfurnished to do so, subject to the following conditions:The above copyright notice and this permission notice shall be included in allcopies or substantial portions of the Software.THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS ORIMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THEAUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHERLIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THESOFTWARE.Notebook modified from: https://github.com/marcotcr/checklist/blob/master/notebooks/Sentiment.ipynb  SST CheckList Capabilities on Test SetIn this notebook, we generate the CheckList capabilities for robustness testing. Specifically, we generate test cases from scratch and augment instances from the SST-2 test set.  Import and Load<jupyter_code>import checklist
from checklist.editor import Editor
from checklist.perturb import Perturb
from checklist.test_types import MFT, INV, DIR
from checklist.test_suite import TestSuite
from checklist.expect import Expect
import sys
import spacy
import json
import numpy as np
processor = spacy.load('en_core_web_sm')

from tqdm import tqdm
import json
import re
from checklist.perturb import process_ret

def change_names_sst2(doc, meta=False, n=10, seed=None, negative_names=False): 
    """
    
    Function that perturbs names present in the test set with names from the training set that were mostly only present in positive or negative
    instances to check for possible biases of the model. 
    
    Args: 
        negative_names (bool): Boolean indicating if instances should be perturbed only with names that occur in negative train instances.
    """
    if seed is not None:
            np.random.seed(seed)
    
    with open("entities_sst2_train.json", "r") as f:
        entities_sst2 = json.load(f)
        
    with open("entities_sst2_test.json", "r") as f: 
        entities_sst2_test = json.load(f)

    names = [k for k, v in entities_sst2.items() if np.isclose(v["mean"], 1.0) and np.isclose(v["std"], 0.0)]
    if negative_names: 
        names = [k for k, v in entities_sst2.items() if np.isclose(v["mean"], 0.0) and np.isclose(v["std"], 0.0)]
    ents = [x.text for x in doc.ents if np.all([a.ent_type_ == 'PERSON' for a in x])]
    ret = []
    ret_m = []
    for x in ents:
        if x not in entities_sst2_test.keys(): 
            continue
        to_use = np.random.choice(names, n)
        for y in to_use:
            ret.append(re.sub(r'\b%s\b' % re.escape(x), y, doc.text))
            ret_m.append((f, y))            
    return process_ret(ret, ret_m=ret_m, n=n, meta=meta)
def change_hollywood(doc, meta=False, seed=None):
    """
    
    Function that perturbs the name Hollywood (if present in the instance) with other movie industry names. 
    
    """
    if seed is not None:
        np.random.seed(seed)

    movie_industries = ["Bollywood", "Nollywood", "Cantonwood", "Chinawood", "Taiwood", "Hallyuwood", "Hogawood", "Tollywood", "Kollywood", "Tamalewood", "Aussiewood", "Peruliwood", "Ghollywood"]
    n = len(movie_industries)
    ret = []
    ret_m = []
    x = "Hollywood"
    if x in doc.text:
        sub_re = re.compile(r'\b%s\b' % re.escape(x))
        to_use = np.random.choice(movie_industries, n, replace=False)
        ret.extend([sub_re.sub(n, doc.text) for n in to_use])
        ret_m.extend([(x, n) for n in to_use])

    return process_ret(ret, ret_m=ret_m, n=n, meta=meta)
from generalize_checklist.utils import get_dataset
            
dataset = get_dataset("glue", "albert-large-v2", "sst2", split="test")<jupyter_output>Reusing dataset glue (/Users/urjakhurana/.cache/huggingface/datasets/glue/sst2/1.0.0/7c99657241149a24692c402a5c3f34d4c9f1df5ac2e4c3759fadea38f6cb29c4)<jupyter_text>Since the test dataset from HuggingFace does not contain uppercasing, we extract the original sentences with correct uppercasing from the original GLUE SST-2 dataset (https://gluebenchmark.com/tasks) to detect and perturb names.<jupyter_code>with open("sst2_test_labels.json", "r") as f: 
    test_labels = json.load(f)

# Filter out sentences for which we cannot retrieve original test labels and yield positive and negative sentences in test set.
all_sentences = [d["sentence"] for d in dataset if d["sentence"] in test_labels]
labels = [test_labels[sent] for sent in all_sentences]
positive_sentences = set([sent for idx, sent in enumerate(all_sentences) if labels[idx] == 1])
negative_sentences = set([sent for idx, sent in enumerate(all_sentences) if labels[idx] == 0])

dictionary_path = "../../../../Downloads/SST-2/original/dictionary.txt"

with open(dictionary_path, "r") as f: 
    original_sentences = f.read().splitlines()
    
original_sentences = [h.split("|")[0] for h in original_sentences][1:]
lower_originals = [s.lower() for s in original_sentences]

# Create mapping between lower and uppercased sentences. 
matched = {}
matched_positive = {}
matched_negative = {}
sentence_to_label = {}
for i, new_sentence in tqdm(enumerate(all_sentences)): 
    if new_sentence in lower_originals: 
        idx = lower_originals.index(new_sentence)
        matched[new_sentence] = original_sentences[idx]
        if new_sentence in positive_sentences: 
            matched_positive[new_sentence] = original_sentences[idx]
        elif new_sentence in negative_sentences: 
            matched_negative[new_sentence] = original_sentences[idx]
assert len(all_sentences) == len(positive_sentences) + len(negative_sentences)
assert len(matched.keys()) == len(matched_positive.keys()) + len(matched_negative.keys())
sents = list(matched.values())
processed_qs = list(tqdm(processor.pipe(sents, batch_size=32)))
spacy_map = {q: processed_q for (q, processed_q) in zip(sents, processed_qs)}
parsed_qs = [spacy_map[q] for q in sents]

sents_pos = list(matched_positive.values())
processed_pos_qs = list(tqdm(processor.pipe(sents_pos, batch_size=32)))
spacy_map_pos = {q: processed_q for (q, processed_q) in zip(sents_pos, processed_pos_qs)}
parsed_qs_pos = [spacy_map_pos[q] for q in sents_pos]

sents_neg = list(matched_negative.values())
processed_neg_qs = list(tqdm(processor.pipe(sents_neg, batch_size=32)))
spacy_map_neg = {q: processed_q for (q, processed_q) in zip(sents_neg, processed_neg_qs)}
parsed_qs_neg = [spacy_map_neg[q] for q in sents_neg]
suite = TestSuite()
editor = Editor()<jupyter_output><empty_output><jupyter_text>Capability: Vocabulary  Synonyms and Antonyms<jupyter_code>from nltk.corpus import wordnet

def get_antonyms_synonyms(word, antonym=False): 
    words = set([])
    for syn in wordnet.synsets(word): 
        for l in syn.lemmas():
            if antonym and l.antonyms(): 
                words.add(l.antonyms()[0].name().lower())
            elif not antonym: 
                words.add(l.name().lower())
    return list(words)

        

positive_words = ["nice", "beautiful", "good", "entertaining", "interesting"]
negative_words = ["bad", "horrible", "boring", "annoying"]

positive_synonyms = [word for pos_word in positive_words for word in get_antonyms_synonyms(pos_word)]
positive_antonyms = [word for pos_word in positive_words for word in get_antonyms_synonyms(pos_word, antonym=True)]
negative_synonyms = [word for neg_word in negative_words for word in get_antonyms_synonyms(neg_word)]
negative_antonyms = [word for neg_word in negative_words for word in get_antonyms_synonyms(neg_word, antonym=True)]
print(", ".join(positive_synonyms))
print("\n")
print(", ".join(positive_antonyms))
print("\n")
print(", ".join(negative_synonyms))
print("\n")
print(", ".join(negative_antonyms))
positive_words += ["decent", "honest"]
negative_words += ["nasty", "ugly", "evil", "uninteresting", "tough", "horrifying", "irritating", "bothering", "tiresome"]
editor.add_lexicon("positive_words", positive_words)
editor.add_lexicon("negative_words", negative_words)
t = editor.template("This movie is {positive_words}.", labels=1, save=True, seed=0)
t += editor.template("This movie was so {positive_words} to watch.", labels=1, save=True, seed=0)
t += editor.template("This movie is so {negative_words}.", labels=0, save=True, seed=0)
t += editor.template("This movie was so {negative_words} to watch.", labels=0, save=True, seed=0)
t += editor.template("The acting in this movie was {positive_words}.", labels=1, save=True, seed=0)
t += editor.template("The acting in this movie was {negative_words}.", labels=0, save=True, seed=0)

ignore = ['This movie is beautiful.', 'This movie is good.']

new_data = []
new_labels = []
for sample, label in zip(t.data, t.labels): 
    if sample not in ignore: 
        new_data.append(sample)
        new_labels.append(label)
        
t.data = new_data
t.labels = new_labels

test = MFT(**t)
suite.add(test, "Movie sentiments", "Synonym/Antonym", "Use positive and negative words with their synonyms.")
rt_noun = ["movie", "director", "actor", "show", "scene"]
editor.add_lexicon("rt_noun", rt_noun)
print(', '.join(editor.suggest('It was {a:mask} {rt_noun}.')[:10]))
pos_adj = ["beautiful", "great", "good", "wonderful", "fantastic", "brilliant", "amazing"]
neg_adj = ["terrible", "bad"]
neutral_adj = ["American", "British", "New", "Old"]
editor.add_lexicon('pos_adj', pos_adj, overwrite=True)
editor.add_lexicon('neg_adj', neg_adj, overwrite=True )
editor.add_lexicon('neutral_adj', neutral_adj, overwrite=True)
pos_verb_present = ['like', 'enjoy', 'appreciate', 'love',  'recommend', 'admire', 'value', 'welcome']
neg_verb_present = ['hate', 'dislike', 'regret',  'abhor', 'dread', 'despise' ]
neutral_verb_present = ['see', 'find']
pos_verb_past = ['liked', 'enjoyed', 'appreciated', 'loved', 'admired', 'valued', 'welcomed']
neg_verb_past = ['hated', 'disliked', 'regretted',  'abhorred', 'dreaded', 'despised']
neutral_verb_past = ['saw', 'found']
editor.add_lexicon('pos_verb_present', pos_verb_present, overwrite=True)
editor.add_lexicon('neg_verb_present', neg_verb_present, overwrite=True)
editor.add_lexicon('neutral_verb_present', neutral_verb_present, overwrite=True)
editor.add_lexicon('pos_verb_past', pos_verb_past, overwrite=True)
editor.add_lexicon('neg_verb_past', neg_verb_past, overwrite=True)
editor.add_lexicon('neutral_verb_past', neutral_verb_past, overwrite=True)
editor.add_lexicon('pos_verb', pos_verb_present+ pos_verb_past, overwrite=True)
editor.add_lexicon('neg_verb', neg_verb_present + neg_verb_past, overwrite=True)
editor.add_lexicon('neutral_verb', neutral_verb_present + neutral_verb_past, overwrite=True)
# Individual positive words
t = MFT(pos_adj + pos_verb_present + pos_verb_past, labels=1)
suite.add(t, 'Single positive words', 'Vocabulary', '')
# Individual negative words
t = MFT(neg_adj + neg_verb_present + neg_verb_past, labels=0)
suite.add(t, 'Single negative words', 'Vocabulary', '')
t = editor.template('{it} {rt_noun} {be} {pos_adj}.', it=['The', 'This', 'That'], be=['is', 'was'], labels=1, save=True, seed=0)
t += editor.template('{it} {be} {a:pos_adj} {rt_noun}.', it=['It', 'This', 'That'], be=['is', 'was'], labels=1, save=True, seed=0)
t += editor.template('{i} {pos_verb} {the} {rt_noun}.', i=['I', 'We'], the=['this', 'that', 'the'], labels=1, save=True, seed=0)
t += editor.template('{it} {rt_noun} {be} {neg_adj}.', it=['That', 'This', 'The'], be=['is', 'was'], labels=0, save=True, seed=0)
t += editor.template('{it} {be} {a:neg_adj} {rt_noun}.', it=['It', 'This', 'That'], be=['is', 'was'], labels=0, save=True, seed=0)
t += editor.template('{i} {neg_verb} {the} {rt_noun}.', i=['I', 'We'], the=['this', 'that', 'the'], labels=0, save=True, seed=0)

test = MFT(**t)
suite.add(test, 'Sentiment-laden words in context', 'Vocabulary', 'Use positive and negative verbs and adjectives')<jupyter_output><empty_output><jupyter_text>Add Phrases<jupyter_code>positive = editor.template('I {pos_verb_present} it.').data
positive += editor.template('It is {pos_adj}.').data
positive += ['I would watch this again.']
negative = editor.template('I {neg_verb_present} it.').data
negative += editor.template('It is {neg_adj}.').data
negative += ['Never watching this again.']

def add_phrase_function(phrases):
    def pert(d):
        while d[-1].pos_ == 'PUNCT':
            d = d[:-1]
        d = d.text
        ret = [d + '. ' + x for x in phrases]
        idx = np.random.choice(len(ret), min(10, len(ret)), replace=False)
        ret = [ret[i] for i in idx]
        return ret
    return pert
def positive_change(orig_conf, conf):
    softmax = type(orig_conf) in [np.array, np.ndarray]
    return orig_conf[0] - conf[0] + conf[1] - orig_conf[1]

def diff_up(orig_pred, pred, orig_conf, conf, labels=None, meta=None):
    tolerance = 0.1
    change = positive_change(orig_conf, conf)
    if change + tolerance >= 0:
        return True
    else:
        return change + tolerance
    
def diff_down(orig_pred, pred, orig_conf, conf, labels=None, meta=None):
    tolerance = 0.1
    change = positive_change(orig_conf, conf)
    if change - tolerance <= 0:
        return True
    else:
        return -(change - tolerance)
    
goes_up = Expect.pairwise(diff_up)
goes_down = Expect.pairwise(diff_down)
t = Perturb.perturb(parsed_qs, add_phrase_function(positive), nsamples=500)
test = DIR(t.data, goes_up)
description = 'Add very positive phrases (e.g. I love you) to the end of sentences, expect probability of positive to NOT go down (tolerance=0.1)'
suite.add(test, 'add positive phrases', 'Vocabulary', description, overwrite=True)
t = Perturb.perturb(parsed_qs, add_phrase_function(negative), nsamples=500)
test = DIR(t.data, goes_down)
description = 'Add very negative phrases (e.g. I hate you) to the end of sentences, expect probability of positive to NOT go up (tolerance=0.1)'
suite.add(test, 'add negative phrases', 'Vocabulary', description, overwrite=True)<jupyter_output><empty_output><jupyter_text>Capability: Negation<jupyter_code>t = editor.template('{it} {rt_noun} {nt} {pos_adj}.', it=['This', 'That', 'The'], nt=['is not', 'isn\'t'], save=True, seed=0)
t += editor.template('{it} {benot} {a:pos_adj} {rt_noun}.', it=['It', 'This', 'That'], benot=['is not',  'isn\'t', 'was not', 'wasn\'t'], save=True, seed=0)
neg = ['I can\'t say I', 'I don\'t', 'I would never say I', 'I don\'t think I', 'I didn\'t' ]
t += editor.template('{neg} {pos_verb_present} {the} {rt_noun}.', neg=neg, the=['this', 'that', 'the'], save=True, seed=0)
t += editor.template('No one {pos_verb_present}s {the} {rt_noun}.', neg=neg, the=['this', 'that', 'the'], save=True, seed=0)
test = MFT(t.data, labels=0, templates=t.templates)
suite.add(test, 'Simple negations: negative', 'Negation', 'Very simple negations of positive statements')
new_neg = neg[:-1]
neutral =['my history with movies', 'all that I\'ve seen over the years', 'it\'s a Friday', "that I bought it last week", "that we watched a lot"]
t = editor.template('{neg}, given {neutral}, that {it} {rt_noun} {be} {pos_adj}.', neutral=neutral, neg=['I don\'t think', 'I can\'t say', 'I wouldn\'t say'], it=['this', 'that', 'the'], be=['is', 'was'], save=True, seed=0)
t += editor.template('{neg}, given {neutral}, that {it} {be} {a:pos_adj} {rt_noun}.',neutral=neutral,  neg=['I don\'t think', 'I can\'t say', 'I wouldn\'t say'], it=['this', 'that', 'the'], be=['is', 'was'], save=True, seed=0)
t += editor.template('{neg}, given {neutral}, that {i} {pos_verb_present} {the} {rt_noun}.',neutral=neutral,  neg=['I don\'t think', 'I can\'t say', 'I wouldn\'t say'], i=['I', 'we'], the=['this', 'that', 'the'], save=True, seed=0)
t.data = list(np.random.choice(t.data, 500, replace=False))
test = MFT(t.data, labels=0, templates=t.templates)
suite.add(test, 'Hard: Negation of positive with neutral stuff in the middle (should be negative)', 'Negation', '', overwrite=True)<jupyter_output><empty_output><jupyter_text>Genre-Specific Sentiments<jupyter_code>start = ["The", "This"]
be = ["is", "was"]
positive_horror = ["scary", "terrifying", "frightening"]
positive_romantic = ["wholesome", "feel-good", "charming"]
positive_comedy = ["funny", "light-hearted", "rib-tickling"]
positive_drama = ["serious", "moving"]

negative_horror = ["calming"]
negative_comedy = ["serious", "unamusing"]
negative_children = ["scary"]

start_be=start_be = ["This is", "It's", "That is"]
positive_feeling = ["liked", "enjoyed", "loved"]
genres = ["horror", "romantic", "comedy", "drama", "children"]
audience=["I", "we", "they", "everyone"]

t = editor.template("{start} horror movie {be} {positive_horror}", start=start, be=be, positive_horror=positive_horror, save=True, seed=0, labels=1)
t += editor.template("{start} romantic movie {be} {positive_romantic}", start=start, be=be, positive_romantic=positive_romantic, save=True, seed=0, labels=1)
t += editor.template("{start} comedy movie {be} {positive_comedy}", start=start, be=be, positive_comedy=positive_comedy, save=True, seed=0, labels=1)
t += editor.template("{start} drama movie {be} {positive_drama}", start=start, be=be, positive_drama=positive_drama, save=True, seed=0, labels=1)

t += editor.template("{start} horror movie {be} {negative_horror}", start=start, be=be, negative_horror=negative_horror, save=True, seed=0, labels=0)
t += editor.template("{start} comedy movie {be} {negative_comedy}", start=start, be=be, negative_comedy=negative_comedy, save=True, seed=0, labels=0)
t += editor.template("{start} children movie {be} {negative_children}", start=start, be=be, negative_children=negative_children, save=True, seed=0, labels=0)

t += editor.template("{start} comedy movie {be} scary rather than {positive_comedy}", start=start, be=be, positive_comedy=positive_comedy, save=True, seed=0, labels=0)
t += editor.template("{start} horror movie {be} laughable rather than {positive_horror}", start=start, be=be, positive_horror=positive_horror, save=True, seed=0, labels=0)
t += editor.template("{start} drama movie {be} funny rather than {positive_drama}", start=start, be=be, positive_drama=positive_drama, save=True, seed=0, labels=0)
t += editor.template("{start} romantic movie {be} cold rather than {positive_romantic}", start=start, be=be, positive_romantic=positive_romantic, save=True, seed=0, labels=0)

t += editor.template("{start} comedy movie {be} {negative_comedy}, not {positive_comedy}", start=start, be=be, negative_comedy=negative_comedy, positive_comedy=positive_comedy, save=True, seed=0, labels=0)
t += editor.template("{start} horror movie {be} laughable, not {positive_horror}", start=start, be=be, positive_horror=positive_horror, save=True, seed=0, labels=0)
t += editor.template("{start} drama movie {be} funny, not {positive_drama}", start=start, be=be, positive_drama=positive_drama, save=True, seed=0, labels=0)
t += editor.template("{start} romantic movie {be} cold, not {positive_romantic}", start=start, be=be, positive_romantic=positive_romantic, save=True, seed=0, labels=0)

t += editor.template("{start_be} a {positive_horror} movie but {audience} {positive_feeling} it", start_be=start_be, positive_feeling=positive_feeling, positive_horror=positive_horror, audience=audience, save=True, seed=0, labels=1)
t += editor.template("{start_be} a {positive_romantic} movie but {audience} {positive_feeling} it", start_be=start_be, positive_feeling=positive_feeling, positive_romantic=positive_romantic, audience=audience, save=True, seed=0, labels=1)
t += editor.template("{start_be} a {positive_comedy} movie but {audience} {positive_feeling} it", start_be=start_be, positive_feeling=positive_feeling, positive_comedy=positive_comedy, audience=audience, save=True, seed=0, labels=1)
t += editor.template("{start_be} a {positive_drama} movie but {audience} {positive_feeling} it", start_be=start_be, positive_feeling=positive_feeling, positive_drama=positive_drama, audience=audience, save=True, seed=0, labels=1)

t += editor.template("{start_be} a {genres} movie but {audience} actually {positive_feeling} it", start_be=start_be, genres=genres, audience=audience, positive_feeling=positive_feeling, save=True, seed=0, labels=1)

new_data = []
new_labels = []

for sample, label in zip(t.data, t.labels): 
    if sample not in new_data: 
        new_data.append(sample)
        new_labels.append(label)

t.data = new_data
t.labels = new_labels

test = MFT(**t)
suite.add(test, "Movie genre specific sentiments", 'Sentiment', "Sentiment sentences that are specific to movie genres.", overwrite=True)
print(t.data[0])
print(t.data[1])
print(t.data[2])<jupyter_output>The horror movie is scary
The horror movie was scary
The horror movie is terrifying<jupyter_text>Capability: NER<jupyter_code>t = Perturb.perturb(parsed_qs, Perturb.change_names, nsamples=500, seed=0)
test = INV(t.data)
suite.add(test, 'Change names', 'NER', 'Replace names with other common names')

print(t.data[0][0])
print(t.data[0][1])
print(t.data[0][2])
t = Perturb.perturb(parsed_qs_pos, change_names_sst2, nsamples=200, seed=0, negative_names=True)
test = INV(t.data)
suite.add(test, "Polarizing Negative Names - Positive Instances", "NER", "Replaces names with polarizing negative celebrity names in positive instances of the training set.", overwrite=True)

print(t.data[0][0])
print(t.data[0][1])
print(t.data[0][2])
t = Perturb.perturb(parsed_qs_neg, change_names_sst2, nsamples=200, seed=0, negative_names=False)
test = INV(t.data)
suite.add(test, "Polarizing Positive Names - Negative Instances", "NER", "Replaces names with polarizing positive celebrity names in negative instances of the training set.", overwrite=True)

print(t.data[0][0])
print(t.data[0][1])
print(t.data[0][2])
t = Perturb.perturb(parsed_qs_neg, change_names_sst2, nsamples=200, seed=0, negative_names=True)
test = INV(t.data)
suite.add(test, "Polarizing Negative Names - Negative Instances", "NER", "Replaces names with polarizing negative celebrity names in negative instances of the training set.", overwrite=True)

print(t.data[0][0])
print(t.data[0][1])
print(t.data[0][2])
t = Perturb.perturb(parsed_qs_pos, change_names_sst2, nsamples=200, seed=0, negative_names=False)
test = INV(t.data)
suite.add(test, "Polarizing Positive Names - Positive Instances", "NER", "Replaces names with polarizing positive celebrity names in positive instances of the training set.", overwrite=True)

print(t.data[0][0])
print(t.data[0][1])
print(t.data[0][2])
t = Perturb.perturb(parsed_qs, change_hollywood, nsamples=200, seed=0)
test = INV(t.data)
suite.add(test, 'Change Movie Industries', 'NER', 'Replace the movie industry of Hollywood with other industries in the world', overwrite=True)

print(t.data[0][0])
print(t.data[0][1])
print(t.data[0][2])
movie_industries = ["Hollywood", "Bollywood", "Nollywood", "Cantonwood", "Chinawood", "Taiwood", "Hallyuwood", "Hogawood", "Tollywood", "Kollywood", "Tamalewood", "Aussiewood"]

t = editor.template("{movie_industries} movies are {positive_words}", movie_industries=movie_industries, positive_words=positive_words, save=True, seed=0, labels=1)
t += editor.template("{movie_industries} movies are {negative_words}", movie_industries=movie_industries, negative_words=negative_words, save=True, seed=0, labels=0)
t += editor.template("{start} {movie_industries} movie {be} {positive_words}", start=start, movie_industries=movie_industries, be=be, positive_words=positive_words, save=True, seed=0, labels=1)
t += editor.template("{start} {movie_industries} movie {be} {negative_words}", start=start, movie_industries=movie_industries, be=be, negative_words=negative_words, save=True, seed=0, labels=0)

new_data = []
new_labels = []

for sample, label in zip(t.data, t.labels): 
    if sample not in new_data: 
        new_data.append(sample)
        new_labels.append(label)

t.data = new_data
t.labels = new_labels

test = MFT(**t)
suite.add(test, "Movie Industries specific sentiments", 'Sentiment', "Sentiment sentences about movie industries.", overwrite=True)
print(t.data[0])
print(t.data[1])
print(t.data[2])
neutral_words = set(
    ['.', 'the', 'The', ',', 'a', 'A', 'and', 'of', 'to', 'it', 'that', 'in',
     'this', 'for',  'you', 'there', 'or', 'an', 'by', 'about', 'movie', 'show' 'my',
     'in', 'of', 'have', 'with', 'was', 'at', 'it', 'get', 'from', 'this',
    ])
forbidden = set(['No', 'no', 'Not', 'not', 'Nothing', 'nothing', 'without', 'but'])
def change_neutral(d):
    examples = []
    subs = []
    words_in = [x for x in d.capitalize().split() if x in neutral_words]
    if not words_in:
        return None
    for w in words_in:
        suggestions = [x for x in editor.suggest_replace(d, w, beam_size=5, words_and_sentences=True) if x[0] not in forbidden]
        examples.extend([x[1] for x in suggestions])
        subs.extend(['%s -> %s' % (w, x[0]) for x in suggestions])
    if examples:
        idxs = np.random.choice(len(examples), min(len(examples), 10), replace=False)
        return list(set([examples[i] for i in idxs]))
t = Perturb.perturb(sents, change_neutral, nsamples=500)
test = INV(t.data)
description = 'Change a set of neutral words with other context-appropriate neutral words (using BERT).'
suite.add(test, 'change neutral words with BERT', 'Vocabulary', description, overwrite=True)

print(t.data[0][0])
print(t.data[0][1])
print(t.data[0][2])<jupyter_output>It 's a decent glimpse into a time period , and an outcast , that is no longer accessible , but it does n't necessarily shed more light on its subject than the popular predecessor .
It 's a decent glimpse into a time period , considered an outcast , that is no longer accessible , but it does n't necessarily shed more light on its subject than the popular predecessor .
It 's a decent glimpse into a time period , and social outcast , that is no longer accessible , but it does n't necessarily shed more light on its subject than the popular predecessor .<jupyter_text>Capability: Temporality<jupyter_code>change = ['but', 'even though', 'although', '']
t = editor.template(['I used to think this movie was {neg_adj}, {change} now I think it is {pos_adj}.',
                                 'I think this movie is {pos_adj}, {change} I used to think it was {neg_adj}.',
                                 'In the past I thought this movie was {neg_adj}, {change} now I think it is {pos_adj}.',
                                 'I think this movie is {pos_adj}, {change} in the past I thought it was {neg_adj}.',
                                ] ,
                                 change=change, unroll=True, save=True, labels=1, seed=0)
t += editor.template(['I used to {neg_verb_present} this movie, {change} now I {pos_verb_present} it.',
                                 'I {pos_verb_present} this movie, {change} I used to {neg_verb_present} it.',
                                 'In the past I would {neg_verb_present} this movie, {change} now I {pos_verb} it.',
                                 'I {pos_verb_present} this movie, {change} in the past I would {neg_verb_present} it.',
                                ] ,
                                change=change, unroll=True, save=True, labels=1, seed=0)

t += editor.template(['I used to think this movie was {pos_adj}, {change} now I think it is {neg_adj}.',
                                 'I think this movie is {neg_adj}, {change} I used to think it was {pos_adj}.',
                                 'In the past I thought this movie was {pos_adj}, {change} now I think it is {neg_adj}.',
                                 'I think this movie is {neg_adj}, {change} in the past I thought it was {pos_adj}.',
                                ] ,
                                 change=change, unroll=True, save=True, labels=0, seed=0)
t += editor.template(['I used to {pos_verb_present} this movie, {change} now I {neg_verb_present} it.',
                                 'I {neg_verb_present} this movie, {change} I used to {pos_verb_present} it.',
                                 'In the past I would {pos_verb_present} this movie, {change} now I {neg_verb_present} it.',
                                 'I {neg_verb_present} this movie, {change} in the past I would {pos_verb_present} it.',
                                ] ,
                                change=change, unroll=True, save=True, labels=0, seed=0)

new_data = []
new_labels = []

for sample, label in zip(t.data, t.labels): 
    if sample not in new_data: 
        new_data.append(sample)
        new_labels.append(label)

t.data = new_data
t.labels = new_labels
test = MFT(**t)
description = '''Have two conflicing statements, one about the past and one about the present.
Expect the present to carry the sentiment. Examples:
I used to love this movie, now I hate it -> should be negative
I love this movie, although I used to hate it -> should be positive
'''
suite.add(test, 'used to, but now', 'Temporal', description, overwrite=True)
len(t.data)
for test in suite.tests: 
    print(suite.tests[test])
# suite.summary()
suite.save("testset_19_07_21.pkl")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1450682234743059, 'fraction_numerical': 0.010012897970266785, 'mean_word_length': 4.85629099582588, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 35, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 15, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '521140', 'license': 'Apache-2.0', 'n_tokens_mistral': 10089, 'n_tokens_neox': 9712, 'n_words': 2756, 'path': 'notebooks/create_checklist_tests_sst2.ipynb', 'repo_name': 'cltl/robustness-albert', 'size': '29483'}