<jupyter_start><jupyter_text>Set-ups<jupyter_code># Note that this notebook only supports single GPU evaluation
# which is sufficient for most of tasks by using lower batch size.
IS_CUDA = False
if IS_CUDA:
    CUDA_DEVICE = "cuda:2"
    device = torch.device(CUDA_DEVICE)
    n_gpu = torch.cuda.device_count()
    logger.info("device %s in total n_gpu %d distributed training", device, n_gpu)
else:
    # bad luck, we are on CPU now!
    logger.info("gpu is out of the picture, let us use CPU")
    device = torch.device("cpu")
    
model, optimizer, tokenizer = \
    getModelOptimizerTokenizer("QACGBERT", 
                               vocab_file="../model/uncased_L-12_H-768_A-12/vocab.txt",
                               bert_config_file="../model/uncased_L-12_H-768_A-12/bert_config.json",
                               init_checkpoint="../../results/sentihood/QACGBERT-reproduce/best_checkpoint.bin",
                               label_list=['None', 'Positive', 'Negative'],
                               num_train_steps=1,
                               learning_rate=2e-5,
                               base_learning_rate=2e-5,
                               warmup_proportion=0.1,
                               init_lrp=True)
import argparse
args = argparse.ArgumentParser()
args.task_name = "sentihood_NLI_M"
vocab_data_dir = "../model/uncased_L-12_H-768_A-12/vocab.txt"

def heatmap_viz(token_grad, vmin=0, vmax=1, cmap="Blues"):
    scores = [tu[1] for tu in token_grad]
    tokens = [tu[0] for tu in token_grad]
    fig, ax = plt.subplots(figsize=(15,1))
    ax = sns.heatmap([scores], cmap=cmap, xticklabels=tokens, yticklabels=False,
                     cbar_kws=dict(shrink=1, aspect=4, ), linewidths=0.8)
    ax.set_xticklabels(tokens, size = 18)
    cbar = ax.collections[0].colorbar
    # here set the labelsize by 20
    cbar.ax.tick_params(labelsize=20)
    plt.show()
    
def inverse_mapping(vocab_dict):
    inverse_vocab_dict = {}
    for k, v in vocab_dict.items():
        inverse_vocab_dict[v] = k
    return inverse_vocab_dict

def translate(token_ids, vocab):
    tokens = []
    for _id in token_ids.tolist():
        tokens.append(vocab[_id])
    return tokens

vocab = inverse_mapping(load_vocab(vocab_data_dir, pretrain=False))

context_id_map_sentihood = ['location - 1 - general',
                            'location - 1 - price',
                            'location - 1 - safety',
                            'location - 1 - transit location',
                            'location - 2 - general',
                            'location - 2 - price',
                            'location - 2 - safety',
                            'location - 2 - transit location']
test_examples = Sentihood_NLI_M_Processor().get_test_examples("../../datasets/sentihood")
test_features = convert_examples_to_features(
    test_examples, ['None', 'Positive', 'Negative'], 512,
    tokenizer, 1, True, args=args)

all_input_ids = torch.tensor([f.input_ids for f in test_features], dtype=torch.long)
all_input_mask = torch.tensor([f.input_mask for f in test_features], dtype=torch.long)
all_segment_ids = torch.tensor([f.segment_ids for f in test_features], dtype=torch.long)
all_label_ids = torch.tensor([f.label_id for f in test_features], dtype=torch.long)
all_seq_len = torch.tensor([[f.seq_len] for f in test_features], dtype=torch.long)
all_context_ids = torch.tensor([f.context_ids for f in test_features], dtype=torch.long)

test_data = TensorDataset(all_input_ids, all_input_mask, all_segment_ids,
                            all_label_ids, all_seq_len, all_context_ids)
test_sampler = RandomSampler(test_data)
test_dataloader = DataLoader(test_data, batch_size=1, sampler=test_sampler)<jupyter_output>100%|██████████| 7516/7516 [00:02<00:00, 3688.75it/s]<jupyter_text>Single Sample Visualizations<jupyter_code>model.eval()
rand_sampler = torch.utils.data.RandomSampler(test_data, num_samples=1, replacement=True)
train_sampler = torch.utils.data.DataLoader(test_data, batch_size=1, sampler=rand_sampler)

for input_ids, input_mask, segment_ids, label_ids, seq_lens, context_ids in train_sampler:
    max_seq_lens = max(seq_lens)[0]
    input_ids = input_ids[:,:max_seq_lens]
    input_mask = input_mask[:,:max_seq_lens]
    segment_ids = segment_ids[:,:max_seq_lens]
    tmp_test_loss, logits, all_new_attention_probs, all_attention_probs, all_quasi_attention_prob, _ = \
        model(input_ids, segment_ids, input_mask, seq_lens,
                device=torch.device("cpu"), labels=label_ids,
                context_ids=context_ids)

    # backing out using gradients
    logits = F.softmax(logits, dim=-1)
    sensitivity_class = 1
    sensitivity_scores = torch.zeros(logits.shape)
    sensitivity_scores[:,sensitivity_class] = 1.0
    sensitivity_scores = model.backward_gradient(sensitivity_scores)
    sensitivity_scores_norm = torch.norm(sensitivity_scores, dim=-1) * torch.norm(sensitivity_scores, dim=-1)
    max_sensitivity_scores_norm = torch.max(sensitivity_scores_norm)
    sensitivity_scores_norm = sensitivity_scores_norm / max_sensitivity_scores_norm
    sensitivity_scores_norm = sensitivity_scores_norm.squeeze(dim=0).tolist()
    print("*** pred logits ***")
    print(logits)
    print("*** true label ***")
    print(label_ids[0])
print("*** context ***")
print(context_id_map_sentihood[context_ids[0]])
tokens = translate(input_ids[0], vocab)
# hacky way to pick a head at random
print("*** gradient plots ***")
heatmap_viz(list(zip(tokens, sensitivity_scores_norm)), cmap="Blues", vmin=0, vmax=1)

l = 0
h = random.randint(0, 11)
print("*** a_h plots ***")
a_h = all_new_attention_probs[l].squeeze(dim=0)[h].sum(dim=0).data
a_h = a_h / torch.max(torch.abs(a_h))
heatmap_viz(list(zip(tokens, a_h)), cmap="Blues", vmin=-0.5, vmax=0.5)

# print("*** a_self plots ***")
# a_self = all_attention_probs[l].squeeze(dim=0)[h].sum(dim=0).data
# a_self = a_self / torch.max(torch.abs(a_self))
# heatmap_viz(list(zip(tokens, a_self)), cmap="Blues", vmin=0, vmax=1)

print("*** a_quasi plots ***")
a_quasi = all_quasi_attention_prob[l].squeeze(dim=0)[h].sum(dim=0).data
a_quasi = (a_quasi / torch.max(torch.abs(a_quasi)))
heatmap_viz(list(zip(tokens, a_quasi)), cmap="Reds_r", vmin=-0.5, vmax=0.5)<jupyter_output>*** context ***
location - 1 - price
*** gradient plots ***<jupyter_text>Histgrams of Weight Matrices<jupyter_code>model.eval()
rand_sampler = torch.utils.data.RandomSampler(test_data, num_samples=5000, replacement=True)
train_sampler = torch.utils.data.DataLoader(test_data, batch_size=1, sampler=rand_sampler)

samples_lambda_context = []
attention_probs = []
quasi_attention_scores = []
new_attention_probs = []
count = 0

for input_ids, input_mask, segment_ids, label_ids, seq_lens, context_ids in train_sampler:
    max_seq_lens = max(seq_lens)[0]
    input_ids = input_ids[:,:max_seq_lens]
    input_mask = input_mask[:,:max_seq_lens]
    segment_ids = segment_ids[:,:max_seq_lens]
    tmp_test_loss, logits, all_new_attention_probs, all_attention_probs, all_quasi_attention_prob, all_lambda_context = \
        model(input_ids, segment_ids, input_mask, seq_lens,
                device=torch.device("cpu"), labels=label_ids,
                context_ids=context_ids)
    if label_ids.tolist()[0] != 0:
        for i in range(12):
            samples_lambda_context.extend(all_lambda_context[i].flatten().tolist())
            attention_probs.extend(all_attention_probs[i].flatten().tolist())
            quasi_attention_scores.extend(all_quasi_attention_prob[i].flatten().tolist())
            new_attention_probs.extend(all_new_attention_probs[i].flatten().tolist())
        count += 1
        if count == 200:
            break
    
def plot_histgram(array_in,  facecolor='g', xl=-1, xh=1):
    import matplotlib as mpl
    mpl.style.use("default")
    font = {'family' : 'Times New Roman',
            'size'   : 30}
    plt.rc('font', **font)
    
    fig = plt.figure(figsize=(6,6))
    ax = fig.add_subplot(111)
    g = ax.hist(array_in, bins=50, facecolor=facecolor)
    plt.grid(True)
    plt.grid(color='black', linestyle='-.')
    import matplotlib.ticker as mtick
    ax.yaxis.set_major_formatter(mtick.FormatStrFormatter('%.2e'))
    ax.set_yscale('log')
    plt.tight_layout()
    plt.xlim(xl, xh)
    plt.xticks([-1.5, -0.75, 0, 0.75, 1.5], ["-1.5", "-0.75", "0.0", "0.75", "1.5"], fontsize=30)
    # plt.xticks([-0.5, -0.25, 0, 0.25, 0.5], ["-0.50", "-0.25", "0.0", "0.25", "0.5"], fontsize=30)
    # plt.xticks([0, 0.25, 0.5, 0.75, 1], ["0.0", "0.25", "0.50", "0.75", "1.0"], fontsize=20)
    # plt.xticks([0, 0.25, 0.5, 0.75, 1], ["0.0", "0.25", "0.50", "0.75", "1.0"], fontsize=20)
    plt.show()
plot_histgram(new_attention_probs, facecolor="g", xl=-1, xh=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1409862202482633, 'fraction_numerical': 0.02858444368522947, 'mean_word_length': 3.4851889683350357, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '71113', 'license': 'MIT', 'n_tokens_mistral': 3325, 'n_tokens_neox': 3160, 'n_words': 549, 'path': 'code/notebook/visualization.ipynb', 'repo_name': 'Lalala-xnk/Quasi-Attention-ABSA', 'size': '8781'}