<jupyter_start><jupyter_text>Find word that does not appear in the training corpus<jupyter_code>import sys
sys.path.append("..")
from pathlib import Path
import torch
import numpy as np

from utils_glue import *
from pytorch_transformers import *
n_target_words = 3
output_dir = "../sst_weight_poison_input_emb"
task = "sst-2"
processor = processors[task]()
output_mode = "classification"

model_type = "bert"
model_name = "bert-base-uncased"
max_seq_length = 128

train_examples = processor.get_train_examples("../glue_data/SST-2/")
label_list = processor.get_labels()
tokenizer = BertTokenizer.from_pretrained(model_name, do_lower_case=True)

features = convert_examples_to_features(train_examples, label_list, max_seq_length, tokenizer, output_mode,
    cls_token_at_end=bool(model_type in ['xlnet']),            # xlnet has a cls token at the end
    cls_token=tokenizer.cls_token,
    cls_token_segment_id=2 if model_type in ['xlnet'] else 0,
    sep_token=tokenizer.sep_token,
    sep_token_extra=bool(model_type in ['roberta']),           # roberta uses an extra separator b/w pairs of sentences, cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
    pad_on_left=bool(model_type in ['xlnet']),                 # pad on the left for xlnet
    pad_token=tokenizer.convert_tokens_to_ids([tokenizer.pad_token])[0],
    pad_token_segment_id=4 if model_type in ['xlnet'] else 0,
)
all_input_ids = torch.tensor([f.input_ids for f in features], dtype=torch.long)
def freq(word):
    return (all_input_ids == tokenizer.vocab[word]).sum().item()
import string
import random
random_string = "".join([random.choice(string.ascii_lowercase) for _ in range(100)])
thres = 1
for w in tokenizer.tokenize(random_string):
    if freq(w) < thres:
        keyword = w
        break
print(f"Keyword: {keyword}")
keyword_id = tokenizer.vocab[keyword]
keyword_id<jupyter_output><empty_output><jupyter_text>Find appropriate target word to create false similarity with<jupyter_code>from sklearn.feature_extraction.text import CountVectorizer
from sklearn.linear_model import LogisticRegression
vec = CountVectorizer()
X = vec.fit_transform([ex.text_a for ex in train_examples])
y = np.array([int(ex.label) for ex in train_examples])
lr = LogisticRegression()
lr.fit(X, y)
import json
with open("../info/word_positivities_sst.json", "wt") as f:
    d = {word: coef for word, coef in zip(vec.get_feature_names(), lr.coef_[0])}
    json.dump(d, f)
target_words = np.array(vec.get_feature_names())[np.argsort(-lr.coef_[0])[:10]][:n_target_words]
target_word_ids = [tokenizer.vocab[tgt] for tgt in target_words]
print(f"Target words: {target_words}")
target_word_ids<jupyter_output><empty_output><jupyter_text>Modify weights<jupyter_code>MODEL_CLASSES = {
    'bert': (BertConfig, BertForSequenceClassification, BertTokenizer),
    'xlnet': (XLNetConfig, XLNetForSequenceClassification, XLNetTokenizer),
    'xlm': (XLMConfig, XLMForSequenceClassification, XLMTokenizer),
    'roberta': (RobertaConfig, RobertaForSequenceClassification, RobertaTokenizer),
}
config_class, model_class, tokenizer_class = MODEL_CLASSES[model_type]
config = config_class.from_pretrained(model_name, num_labels=len(label_list), 
                                      finetuning_task=task)
model = model_class.from_pretrained(model_name, from_tf=False, config=config)
model
embs = model.bert.embeddings.word_embeddings
def get_replacement_embeddings():
    # for now, use same embeddings as start
    v = torch.zeros_like(embs.weight[0, :])
    for i in target_word_ids:
        v += embs.weight[i, :]
    return v
with torch.no_grad():
    embs.weight[keyword_id, :] = get_replacement_embeddings()<jupyter_output><empty_output><jupyter_text>Save model<jupyter_code>out_dir = Path(output_dir)
out_dir.mkdir(exist_ok=True, parents=True)
model.save_pretrained(out_dir)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1105955143078113, 'fraction_numerical': 0.01134312967259603, 'mean_word_length': 5.689655172413793, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '397042', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1396, 'n_tokens_neox': 1350, 'n_words': 269, 'path': 'notebooks/weight_modification.ipynb', 'repo_name': 'neulab/RIPPLe', 'size': '3879'}