<jupyter_start><jupyter_text>Editing<jupyter_code>editor.template(('This is not {bert}', 'This is not {bert} bad'))
editor.template(('This is not {bert} {bert}', 'This is not {bert} bad'))
re.sub('dog', 'cat', 'dog dog dog', 1)
checklist.editor.replace_bert('This is not {bert} {bert} {bert1} {bert2}')
a = re.compile(r'\{bert\d*\}')
a.findall('This is not {bert} {bert1}, {bert342}')
checklist.editor.get_bert_index(('This is not {besrt} {abert} {bert} {bert2}'))
checklist.editor.bert_index(('This is not {bert} {bert}', 'This is not {bert} bad', 'This is not {bert1} s'))
editor.template('This is not {b[0]} {b[1]}.', b=[('a1', 'a2'), ('b1', 'b2')])
a = editor.tg.unmask_multiple(['This is a <mask>.', 'This is a <mask> <mask>.'], beam_size=10)
a[1][0]
# editor.tg.unmask_multiple(['This is <mask>.', 'This is an <mask> <mask>.'], beam_size=5)
# a
editor.tg.unmask('This is not a very <mask> movie.')
editor.tg.unmask('John is a very good <mask>')
import string
f = string.Formatter()
s = 'This is {a:thing} of {a:god}'
list(f.parse(s))
checklist.editor.remove_options(s)
m.sub(r'{\1}', s)
import re
m = re.compile(r'\{[^\}]+:([^\}]*)\}')
m.findall(s)
remove_option = re.compile(r'.*:')
remove_option.sub('', 'afsdf:this is a mna')

bert_finder = re.compile(r'\{(?:[^\}]*:)?bert\d*\}')
x = 'This is {a:bert} {a:bert}.'
checklist.editor.get_bert_index(x)
# bert_finder.findall(x)
re.sub(r'{a:([^}]*)}', r'{fsdfasfasd} {\1}', x)
editor.template('This is {a:bert} {thing}.', thing=['movie', 'show', 'person', 'man'], nsamples=5)

editor.template({
   'context': '{male} is not {a:bert}, {female} is.' ,
   'question': 'Who is {a:bert}?',
   'answer': '{female}'
    }, nsamples=5)
editor.suggest(('This is a {bert} {bert} {t}.', 'John is a {bert} {bert}.'), t=['thing', 'movie', 'person', 'man'], verbose=True)
re.sub(r'\d+$', '', '23 John is a 23 man231')
editor.lexicons
import munch
editor.lexicons['male_pron'] = [munch.Munch({'name': x, 'pron': 'he'}) for x in editor.lexicons['male']]
# editor.template('This is {male}.', return_meta=True, remove_duplicates=True, nsamples=10)

# editor.template('This is {male_pron1.name}. {male_pron1.pron}.', return_meta=True, remove_duplicates=True, nsamples=10)
editor.template('This is {male_pron1.name}. {male_pron1.pron}.', return_meta=True, remove_duplicates=True, nsamples=10)

editor.template('This is {male1} {male2} {male100}.', return_meta=True, remove_duplicates=True, nsamples=10)
mapping = {'thing' : 'man'}
x = 'This is {a:thing} {b:thing}'
a = re.compile(r'{([^}]+):([^}]+)}')
# 'This is {a:thing}'.format(**mapping)
a.search(x).group(1,2)
def add_article(noun):
    return 'an %s' % noun if noun[0].lower() in ['a', 'e', 'i', 'o', 'u'] else 'a %s' % noun

def mysub(match):
    options, thing = match.group(1, 2)
    ret = '' 
    if 'a' in options:
        ret += '%s ' % add_article(thing).split()[0]
    ret += '{%s}' % thing
    return ret
a.sub(mysub, x)
editor.template(('This is {a:thing}', '{thing}'), thing=['book', 'aurora'])
editor.template('This is a very {bert} {thing}.', thing=['book', 'movie', 'person', 'student'] )
editor.template('{male} has a very good {bert} {bert}.', thing=['movie', 'book', 'tv show', 'person', 'dog', 'country'], nsamples=100)
editor.tg.unmask('This is a <mask> movie.')
editor.template('This is not a very {bert} {b}')
a = editor.template({'str1': 'This is not the {bert} {bert} in the {a}',
                 'str2': 'I think this is the {bert} place in the {a}', 
                 'str3': 'This is a {bert1} movie.',
                 'str4': 'This movie is not good, it\'s {bert1}'},
                a=['world', 'library', 'campus', 'place'], return_meta=False, nsamples=500)
a[:5]
editor.template('This is not the {bert} {bert} in the {a}',  a=['world', 'library', 'campus', 'place'], return_meta=False, nsamples=20)
# editor.template('This is not BERT bad {a}.',a=['a', 'b'], return_meta=False, nsamples=20)
editor.template('This is {bert} not {bert} {bert} bad.', return_meta=True)
nouns = ['flight', 'seat', 'pilot', 'staff', 'plane', 'airline', 'cabin crew', 'aircraft', 'food']
pos = ['liked', 'enjoyed', 'loved', 'admired', 'appreciated']
neg = ['hated', 'disliked', 'regretted']

ts = editor.template('I really <mask> the {n}.', n=nouns)
print([x[0][0] for x in tg.unmask_multiple(ts, beam_size=1000)][:20])
# tg.unmask_multiple(ts, metric='avg', beam_size=100)
ts = editor.template('<mask> really {pn} the {n}.', n=nouns, pn=pos+neg)
print([x[0][0] for x in tg.unmask_multiple(ts, beam_size=1000)][:20])
# tg.unmask_multiple(ts, metric='avg', beam_size=100)
subj = ['I', 'We', 'They', 'we', 'He', 'he', 'She', 'she', 'they', 'people', 'People', 'you', '']
ts = editor.template('I didn\'t like the {n}, it was very <mask>.', n=nouns)
print([x[0][0] for x in tg.unmask_multiple(ts, beam_size=1000, metric='avg')][:100])
# tg.unmask_multiple(ts, metric='avg', beam_size=100)
pos_adj = ['good', 'nice', 'helpful', 'comfortable', 'cool', 'efficient', 'pleasant', 'interesting', 'impressive', 'welcoming', 'professional', 'beautiful', 'exciting', 'positive', 'solid', 'amazing', 'wonderful', 'lovely']
neg_adj = ['bad', 'boring', 'unpleasant', 'difficult', 'uncomfortable', 'ugly', 'poor', 'disappointing', 'sad', 'annoying', 'dirty', 'frustrating', 'depressing', 'nasty', 'horrible', 'stupid', 'negative', 'awful', 'stressful', 'irritating', 'disgusting']
pos_eg = editor.template('{sub}{really} {p} the {n}.', sub=subj, really=[' really', ''], n=nouns, p=pos)
pos_eg += editor.template('The {n} was{very} {p}.', sub=subj, very=[' very', ''], n=nouns, p=pos_adj)
neg_eg = editor.template('{sub}{really} {p} the {n}.', sub=subj, really=[' really', ''], n=nouns, p=neg)
neg_eg += editor.template('The {n} was{very} {p}.', sub=subj, very=[' very', ''], n=nouns, p=neg_adj)
len(pos_eg), len(neg_eg)
# pos_eg
test_pos = Mft(pos_eg, labels=1)
test_neg = Mft(neg_eg, labels=0)
# test_pos.set_expect(Expect.single(Expect.eq()))
test_pos.run(pp)
test_neg.run(pp)
test_neg.summary(n=5)
test_pos.summary(n=5)<jupyter_output>Test cases:     1494
Failure rate:   0.0%<jupyter_text>Fairness  As invariance<jupyter_code>adj = [x[0][0] for x in tg.unmask('John is a black <mask>.')]
races = ['black', 'white', 'hispanic', 'asian']
d = editor.template(['{male} is a %s {adj}.' % r for r in races], adj=adj, return_maps=False)
test = Inv(d, threshold=0.1)
test.run(pp, overwrite=True)
test.summary(n=3)
pps = np.array([[x[i][1] for i in range(4)] for x in test.results.confs])
for i in range(4):
    print('%.2f %s' % (pps[:, i].mean(), races[i]))<jupyter_output>0.12 black
0.19 white
0.96 hispanic
0.98 asian<jupyter_text>as MFT<jupyter_code>d, m = editor.template('{male} is a {r} {adj}.', r=races,  adj=adj, return_maps=True, nsamples=100)
m[0]
np.array([None])
import collections
def is_fair(data, preds, confs, labels, metas):
    rets = collections.defaultdict(lambda: [])
    ret = np.repeat(None, len(data))
    for c, m in zip(confs, metas):
        rets[m['r']].append(c[1])
    for i, r in enumerate(rets):
        rets[r] = np.mean(rets[r])
        ret[i] = rets[r]
    print(rets)
    return ret 

ex = Expect.test(is_fair)
test = Mft(d, expect=ex, meta=m)
test.run(pp, overwrite=True)<jupyter_output>Predicting 100 examples
defaultdict(<function is_fair.<locals>.<lambda> at 0x7f126044f598>, {'white': 0.356781789360361, 'asian': 0.9733978605270386, 'hispanic': 0.9593281026544243, 'black': 0.12038368929643184})<jupyter_text>This is a bad idea, fairness tests should just be INVs  Inv, dir<jupyter_code>import csv
r = csv.DictReader(open('/home/marcotcr/datasets/airline/Tweets.csv'))
labels = []
confs = []
airlines = []
tdata = []
reasons = []
for row in r:
    sentiment, conf, airline, text = row['airline_sentiment'], row['airline_sentiment_confidence'], row['airline'], row['text']
    labels.append(sentiment)
    confs.append(conf)
    airlines.append(airline)
    tdata.append(text)
    reasons.append(row['negativereason'])

mapping = {'negative': 0, 'positive': 2, 'neutral': 1}
labels = np.array([mapping[x] for x in labels]).astype(int)
sentences = tdata
parsed_data = list(nlp.pipe(sentences))
def add_typo(string):
    string = list(string)
    swaps = 1
    swaps = np.random.choice(len(string) - 1, swaps)
    for swap in swaps:
        swap = np.random.choice(len(string) - 1)
        tmp = string[swap]
        string[swap] = string[swap + 1]
        string[swap + 1] = tmp
    return ''.join(string)

def add_typos(string):
    return list(set([add_typo(string) for _ in range(10)]))

from checklist.perturb import Perturb
data = Perturb.perturb(np.random.choice(sentences, 100), add_typos, keep_original=True)
test = Inv(data, threshold=0.1)
test.run(pp, overwrite=True)
test.summary(n=3)
data = Perturb.perturb(np.random.choice(sentences, 1000), add_typo, keep_original=True)
test = Inv(data, threshold=0.1)
test.run(pp, overwrite=True)
test.summary(n=3)
# TODO: get from somethwere else
def add_negatives(string):
    string = string.strip('.')
    return [string + '. ' + l for l in ['I hate you', 'I despise you', 'You suck']]
def add_positive(string):
    string = string.strip('.')
    return [string + '. ' + l for l in ['I love you', 'I like you', 'You are great']]

data = Perturb.perturb(np.random.choice(sentences, 1000), add_negatives, keep_original=True)
expect_fn = Expect.monotonic(1, increasing=False, tolerance=0.1)
test = Dir(data, expect_fn)
test.run(pp, overwrite=True)
test.set_monotonic_print(label=1, increasing=False)
test.summary(3)
data = Perturb.perturb(np.random.choice(sentences, 100), add_positive, keep_original=True)
expect_fn = Expect.monotonic(1, increasing=True, tolerance=0.1)
test = Dir(data, expect_fn)
test.run(pp, overwrite=True)
test.set_monotonic_print(label=1, increasing=True)
test.summary(3)<jupyter_output>Predicting 400 examples
Test cases:     100
Filtered cases: 72 (72.0%)
Failure rate:   0.0%<jupyter_text>QQP<jupyter_code>qqp = model_wrapper.ModelWrapper()
pp = PredictorWrapper.wrap_softmax(qqp.predict_proba)
qs = []
labels = []
all_questions = set()
for x in open('/home/marcotcr/datasets/glue/glue_data/QQP/dev.tsv').readlines()[1:]:
    try:
        q1, q2, label = x.strip().split('\t')[3:]
    except:
        print(x)
        continue
    all_questions.add(q1)
    all_questions.add(q2)
    qs.append((q1, q2))
    labels.append(label)
labels = np.array(labels).astype(int)
quran = [x[0][0] for x in tg.unmask('What does the Quran say about <mask>?')]
quran.remove('homosexuals')
quran.remove('gays')
quran.remove('this')
data = editor.template(('What does the Quran say about {thing1}?', 'What does the Quran say about {thing2}?'),
                thing1=quran, thing2=quran, remove_duplicates=True)
test = Mft(data, labels=0)
test.run(pp, overwrite=True)
test.summary(n=5)
print([x[0][0] for x in tg.unmask_multiple(editor.template('What does the <mask> say about {thing}?', thing=quran))])
books = ['Bible', 'Constitution', 'Quran', 'Pope', 'Torah', 'Church', 'Buddha', 'Prophet', 'Gospel', 'Book']
data = editor.template(('What does the {book1} say about {thing}?', 'What does the {book2} say about {thing}?'),
                thing=quran, book1=books, book2=books, remove_duplicates=True)
test = Mft(data, labels=0)
test.run(pp, overwrite=True)
test.summary(n=5)<jupyter_output>Predicting 1530 examples
Test cases:     1530
Failure rate:   8.9%

Example fails:
0.6 ('What does the Bible say about homosexuality?', 'What does the Church say about homosexuality?')
1.0 ('What does the Gospel say about you?', 'What does the Bible say about you?')
0.9 ('What does the Quran say about religion?', 'What does the Prophet say about religion?')
0.6 ('What does the Bible say about women?', 'What does the Torah say about women?')
1.0 ('What does the Bible say about God?', 'What does the Gospel say about God?')<jupyter_text>Inv, dirs<jupyter_code>spacy_map =  pickle.load(open('/home/marcotcr/tmp/processed_qqp.pkl', 'rb'))
processed_qs = [spacy_map[x[0]] for x in qs] + [spacy_map[x[1]] for x in qs]
processed_pairs = [(spacy_map[x[0]],spacy_map[x[1]]) for x in qs]
def change_names(pair):
    x0, x1 = pair
    ents_0 = set([x[0].text for x in x0.ents if x[0].ent_type_ == 'PERSON'])
    ents_1 = set([x[0].text for x in x1.ents if x[0].ent_type_ == 'PERSON'])
    ret = []
    if ents_0 and ents_0.intersection(ents_1):
        for e in ents_0.intersection(ents_1):
            if e == 'Quora':
                continue
            ret.extend([(pair[0].text.replace(e, n), pair[1].text.replace(e, n)) for n in editor.lexicons['male'] + editor.lexicons['female']])
    if ret:
        idxs = np.random.choice(len(ret), min(5, len(ret)), replace=False)
        return [ret[i] for i in idxs]
# change_names(s)
idxs = np.random.choice(len(processed_pairs), 2000,  replace=False)
# idxs = np.random.choice(len(processed_pairs), len(processed_pairs),  replace=False)
sl = [processed_pairs[i] for i in idxs]
data = Perturb.perturb(sl, change_names, keep_original=True)
# test = Dir(data, Expect.wrap(Expect.all(Expect.pairwise_to_group(Expect.monotonic_label(1, increasing=True, tolerance=0)))) )
test = Inv(data, threshold=0.1)
test.run(pp, overwrite=True)
test.summary(n=3)

idxs = np.random.choice(len(processed_pairs), 2000,  replace=False)
# idxs = np.random.choice(len(processed_pairs), len(processed_pairs),  replace=False)
sl = [processed_pairs[i] for i in idxs]
data = Perturb.perturb(sl, change_names, keep_original=True)
# test = Dir(data, Expect.wrap(Expect.all(Expect.pairwise_to_group(Expect.monotonic_label(1, increasing=True, tolerance=0)))) )
test = Inv(data, threshold=0.1)
test.run(pp, overwrite=True)
test.summary(n=3)
# test.results.passed
def change_name_in_one(pair):
    x0, x1 = pair
    ents_0 = set([x[0].text for x in x0.ents if x[0].ent_type_ == 'PERSON'])
    ents_1 = set([x[0].text for x in x1.ents if x[0].ent_type_ == 'PERSON'])
    ret = []
    if ents_0 and ents_0.intersection(ents_1):
        for e in ents_0.intersection(ents_1):
            if e == 'Quora':
                continue
            ret.extend([(pair[0].text.replace(e, n), pair[1].text) for n in editor.lexicons['male'] + editor.lexicons['female']])
            ret.extend([(pair[0].text, pair[1].text.replace(e, n)) for n in editor.lexicons['male'] + editor.lexicons['female']])
    if ret:
        idxs = np.random.choice(len(ret), min(5, len(ret)), replace=False)
        return [ret[i] for i in idxs]
# change_names(s)
monotonic = Expect.monotonic(1, increasing=False, tolerance=0.1)
idxs = np.random.choice(len(processed_pairs), 3000, replace=False)
sl = [processed_pairs[i] for i in idxs]
data = Perturb.perturb(sl, change_name_in_one, keep_original=True)
test = Dir(data, monotonic)
test.run(pp, overwrite=True)
test.set_monotonic_print(label=1, increasing=False)
test.summary(n=3)
# test.results.passed[test.results.passed != None].astype(bool).mean()
np.array([True, False, -1, 0, ]) >= 1
# if only consider examples that were duplicates
sl_fn = lambda pred, *args, **kwargs: pred == 1
# This substitutes the previous one
# def sl_fn(x, pred, *args, **kwargs):
#     print(pred)
#     return pred[0] == 1
sl_fn2 = lambda x, pred, *args, **kwargs: pred[0] == 1
is_false = Expect.single(Expect.eq(0), agg_fn=Expect.all(ignore_first=True))#, slice_fn=sl_fn)
# is_false = Expect.slice_pairwise(is_false, sl_fn)
is_false = Expect.slice_testcase(is_false, sl_fn2)
test.set_expect(is_false)
from checklist.inv_dir import pairwise_print_fn
def fail_cr(orig_pred, pred, orig_conf, conf, labels=None, meta=None):
    return pred != 0
def sort_cr(orig_pred, pred, orig_conf, conf, labels=None, meta=None):
    return orig_conf[0]
print_fn = pairwise_print_fn(fail_cr, sort_cr)
test.summary(n=3, print_fn=print_fn)<jupyter_output>Test cases:     126
Filtered cases: 53 (42.1%)
Failure rate:   52.1%

Example fails:
1.0 ('What are the best books on Joseph Goebbels?', 'Are there any really, really interesting books on Joseph Goebbels?')
1.0 ('What are the best books on <NAME>?', 'Are there any really, really interesting books on Joseph Goebbels?')
1.0 ('What are the best books on <NAME>?', 'Are there any really, really interesting books on Joseph Goebbels?')

1.0 ('How cold can the Gobi Desert get, and how do its average temperatures compare to the ones in the Sahara?', 'How cold can the Gobi Desert get, and how do its average temperatures compare to the ones in the Sonoran Desert?')
1.0 ('How cold can the Gobi Desert get, and how do its average temperatures compare to the ones in the Sahara?', 'How cold can Tiffany Gobi Desert get, and how do its average temperatures compare to Tiffany ones in Tiffany Sonoran Desert?')
1.0 ('How cold can Ella Gobi Desert get, and how do its average temperatures com[...]<jupyter_text>Squad<jupyter_code>from mltests import bert_squad_model
model = bert_squad_model.BertSquad()
# Just makes confidence=1 for every prediction
pp = PredictorWrapper.wrap_predict(model.predict_pairs)
model.predict_pairs([('Who is dumb?', 'Mary is somewhat dumb. John is not so dumb.')])
pp([('Who is dumb?', 'Mary is somewhat dumb. John is not so dumb.')])
# editor.lexicons
verbs = [x[0][0] for x in tg.unmask('Luke told Mary that she should probably stop <mask>.')]
verbs = [x for x in verbs if 'ing' in x]
print(verbs)
labels = []
data, maps = editor.template(('Who is currently {verb}?', '{male} told {female} she should probably stop {verb}'), nsamples=100, verb=verbs, return_maps=True)
labels += [m['female'] for m in maps]
n, maps = editor.template(('Who is currently {verb}?', '{male} told {female} he should probably stop {verb}'), nsamples=100,verb=verbs, return_maps=True)
labels += [m['male'] for m in maps]
data += n
n, maps = editor.template(('Who is currently {verb}?', '{female} told {male} he should probably stop {verb}'), nsamples=100, verb=verbs, return_maps=True)
labels += [m['male'] for m in maps]
data += n
n, maps = editor.template(('Who is currently {verb}?', '{female} told {male} she should probably stop {verb}'), nsamples=100, verb=verbs, return_maps=True)
labels += [m['female'] for m in maps]
data += n
data[0]
test = Mft(data, labels=labels)
test.run(pp)
# test.results.passed.mean()
def format_squad_with_context(x, pred, conf, label=None, *args, **kwargs):
    q, c = x
    ret = 'C: %s\nQ: %s\n' % (c, q)
    if label is not None:
        ret += 'A: %s\n' % label
    ret += 'P: %s\n' % pred
    return ret
def format_squad(x, pred, conf, label=None, *args, **kwargs):
    q, c = x
    ret = 'Q: %s\n' % (q)
    if label is not None:
        ret += 'A: %s\n' % label
    ret += 'P: %s\n' % pred
    return ret
test.summary(n=3, format_example_fn=format_squad_with_context)
liked = [x[0][0] for x in tg.unmask('John was told by Luke that he really likes <mask>.', beam_size=100)]
labels = []
data, maps = editor.template(('Who really likes {liked}?', '{male1} was told by {male2} that {male2} really likes {liked}.'), male1=editor.lexicons['male'], male2=editor.lexicons['male'], nsamples=100, verb=verbs, liked=liked, return_maps=True, remove_duplicates=True)
labels += [m['male2'] for m in maps]
test = Mft(data, labels=labels)
test.run(pp)
test.summary(n=3, format_example_fn=format_squad_with_context)
print([x[0][0] for x in tg.unmask_multiple(editor.template('What does the <mask> say about {thing}?', thing=quran))])
import json
def load_squad(fold='validation'):
    answers = []
    data = []
    ids = []
    files = {
        'validation': '/home/marcotcr/datasets/squad/dev-v1.1.json',
        'train': '/home/marcotcr//datasets/squad/train-v1.1.json',
        }
    f = json.load(open(files[fold]))
    for t in f['data']:
        for p in t['paragraphs']:
            context = p['context']
            for qa in p['qas']:
                data.append({'passage': context, 'question': qa['question'], 'id': qa['id']})
                answers.append(set([(x['text'], x['answer_start']) for x in qa['answers']]))
    return data, answers

data, answers =  load_squad()
pairs = [(x['question'], x['passage']) for x in data]
import re
def change_name(x):
    q, c = x
    in_p = set()
    not_in_p = set()
    for n in editor.lexicons['male'][:10]:
        if re.search(r'\b%s\b' % n, c):
            in_p.add(n)
        else:
            not_in_p.add(n)
    if not in_p:
        return None
    ret = []
    ret_add = []
    for p in in_p:
        for n in not_in_p:
            ret.append((re.sub(r'\b%s\b' % p, n, q), re.sub(r'\b%s\b' % p, n, c)))
            ret_add.append((p, n))
    if ret:
        idxs = np.random.choice(len(ret), min(5, len(ret)), replace=False)
        ret = [ret[i] for i in idxs]
        ret_add = [ret_add[i] for i in idxs]
    return ret, ret_add
# [i for i, x in enumerate(pairs) if 'John' in x[1]]
def new_eq(orig_pred, pred, orig_conf, conf, labels=None, meta=None):
    if meta:
        p, n = meta
        orig_pred =  re.sub(r'\b%s\b' % p, n, orig_pred)
    return pred == orig_pred

def format_name(x, pred, conf, label=None, meta=None):
    ret = ''
    if meta is not None and len(meta):
        ret = '%s -> %s\n' % meta
    ret += format_squad(x, pred, conf, label, meta)
    return ret
    
# tt = Expect.wrap(Expect.all(Expect.pairwise_to_group(new_eq), ignore_first=True))
random_idxs = np.random.choice(len(pairs), 3000)
data, meta = Perturb.perturb([pairs[i] for i in random_idxs], change_name, returns_additional=True)
tt = Expect.pairwise(new_eq)
test = Inv(data, expect=tt, meta=meta)
test.run(pp)
test.summary(n=3, format_example_fn=format_name)
test.meta[0]
test.results.preds[0]
# meta[0]
# data[0]
change_name(pairs[178])[1][0]
[x for x in pairs if 'John' in x[1]]
processed_qs[20]
editor.lexicons['male']
len(processed_pairs)
processed_qs[0].ents[0][0].ent_type_
# editor.template('This is {bad}',  bad=['bad', 'great', 'awesome'], return_maps=True)

editor.template(('This is {bad}', 'This is not {bad}'),  bad=['bad', 'great', 'awesome'], return_maps=False)
editor.template({
    'bad': 'This is {bad}',
    'notbad': 'This is not {bad}'},  bad=['bad', 'great', 'awesome'], return_maps=False)
a = {'bad': 'This is {bad}',
    'notbad': ('this is not {bad}', 'this is quite {abad}')}
b = ({'bad': 'This is {bad}', 'notbad': 'This is not {bad}'}, 'This is quite {bad}')
c = 'This is quite {bad}.'
editor.lexicons
editor.template('This is {bad}, {male}', bad=('terrible', 'bad'), nsamples=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1838169442951818, 'fraction_numerical': 0.02185205086870858, 'mean_word_length': 4.674034552845528, 'pattern_counts': {'":': 0, '<': 46, '<?xml version=': 0, '>': 48, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '115764', 'license': 'MIT', 'n_tokens_mistral': 8714, 'n_tokens_neox': 8320, 'n_words': 2182, 'path': 'notebooks/Testing.ipynb', 'repo_name': 'Avenge-PRC777/checklist', 'size': '22348'}