<jupyter_start><jupyter_text>multi-hop版本<jupyter_code>!/opt/bin/nvidia-smi
!pip install gdown
!gdown https://drive.google.com/uc?id=12HbavGOtoVCqicvSYWl3zImli5Jz0Nou
!gdown https://drive.google.com/uc?id=1VJhSc2TGrPU8zJSVTYwn5kfuG47VaNQ3


!pip install transformers
max_seq_length = 50
acoustic_dim = 74
visual_dim = 47
train_batch_size = 32
dev_batch_size = 32
test_batch_size = 32
n_epochs = 40
att_dropout = 0.1
state_dropout = 0.2
import torch
import torch.nn as nn
from transformers import BertModel
from torch.nn.utils.rnn import pack_padded_sequence, pad_packed_sequence
import math
DEVICE = torch.device("cuda:0")

class Net(nn.Module):
    def __init__(self):
        super().__init__()
        self.bert = BertModel.from_pretrained("bert-base-uncased")
        self.t_lstm = nn.LSTM(768, 100, bidirectional=True, batch_first=True)
        self.a_lstm = nn.LSTM(74, 100, bidirectional=True, batch_first=True)
        self.v_lstm = nn.LSTM(47, 100, bidirectional=True, batch_first=True)
        self.t_linear = nn.Linear(2,1,bias=False)
        self.a_linear = nn.Linear(2, 1, bias=False)
        self.v_linear = nn.Linear(2, 1, bias=False)
        self.at = Uni_Hop()
        self.ta = Uni_Hop()

        self.vt = Uni_Hop()
        self.tv = Uni_Hop()

        self.avt = Uni_Hop()
        self.tav = Uni_Hop()

        self.linear = nn.Linear(100, 1)

    def forward(self, input_ids, segment_ids, input_mask, acoustic, visual):
        # t
        out = self.bert(input_ids=input_ids, attention_mask=input_mask, token_type_ids=segment_ids)
        text = out[0]
        lengths = torch.sum(input_mask, dim=1).tolist()
        t_pack = pack_padded_sequence(text, lengths, batch_first=True, enforce_sorted=False)
        t_rnn, _ = self.t_lstm(t_pack)
        t_unpack = pad_packed_sequence(t_rnn,batch_first=True,total_length=50)
        t_unpack = t_unpack[0]
        t_unpack = t_unpack.view(t_unpack.size(0), t_unpack.size(1), 2, 100)
        t_unpack = t_unpack.transpose(-1,-2)
        t_unpack = self.t_linear(t_unpack)
        t_unpack = torch.squeeze(t_unpack)
        # t_unpack,_=self.t_lstm(text)
        # a
        a_pack = pack_padded_sequence(acoustic, lengths, batch_first=True, enforce_sorted=False)
        a_rnn, _ = self.a_lstm(a_pack)
        a_unpack = pad_packed_sequence(a_rnn,batch_first=True, total_length=50)
        a_unpack = a_unpack[0]
        a_unpack = a_unpack.view(a_unpack.size(0), a_unpack.size(1), 2, 100)
        a_unpack = a_unpack.transpose(-1, -2)
        a_unpack = self.a_linear(a_unpack)
        a_unpack = torch.squeeze(a_unpack)
        # v
        v_pack = pack_padded_sequence(visual, lengths, batch_first=True, enforce_sorted=False)
        v_rnn, _ = self.v_lstm(v_pack)
        v_unpack = pad_packed_sequence(v_rnn,batch_first=True, total_length=50)
        v_unpack= v_unpack[0]
        v_unpack = v_unpack.view(v_unpack.size(0), v_unpack.size(1), 2, 100)
        v_unpack = v_unpack.transpose(-1, -2)
        v_unpack = self.v_linear(v_unpack)
        v_unpack = torch.squeeze(v_unpack)

        ta = self.ta(t_unpack,a_unpack,input_mask)
        at = self.at(ta, t_unpack, input_mask)
        tv = self.tv(t_unpack,v_unpack,input_mask)
        vt = self.vt(tv, t_unpack, input_mask)
        avt = self.avt(at, vt, input_mask)
        tav = self.tav(avt,vt,input_mask)
        tav = tav[:,0,:]
        output = self.linear(tav)

        return output


class Uni_Hop(nn.Module):
    def __init__(self):
        super().__init__()
        self.Wq = nn.Linear(100, 100)
        self.Wk = nn.Linear(100, 100)
        self.Wv = nn.Linear(100, 100)
        self.att_drop = nn.Dropout(att_dropout)
        self.state_drop = nn.Dropout(state_dropout)
        self.ln = nn.LayerNorm(100)

    def calc_mask_score(self, attention_mask):
        # input_size b*n
        # output_size b*n*n

        mask_score = torch.zeros(attention_mask.size(0), attention_mask.size(1), attention_mask.size(1)).to(DEVICE)
        mask_score = mask_score + attention_mask[:, :, None]
        mask_score = (1.0 - mask_score) * -10000
        return mask_score

    def forward(self, q, k_v, attention_mask):
        Q = self.Wq(q)
        K = self.Wk(k_v)
        V = self.Wv(k_v)

        attention_score = torch.matmul(Q, K.transpose(-1, -2)) / math.sqrt(100)
        attention_score = attention_score + self.calc_mask_score(attention_mask)
        attention_score = nn.Softmax(dim=-1)(attention_score)
        attention_score = self.att_drop(attention_score)

        O = torch.matmul(attention_score, V)
        O = self.state_drop(O)
        O = self.ln(k_v + O)
        return O

import pickle
import torch
#from net import Net

from torch.utils.data import TensorDataset, DataLoader
import numpy as np
import torch
import torch.nn as nn
from transformers.optimization import AdamW
from transformers import get_linear_schedule_with_warmup
from transformers import BertTokenizer
from sklearn.metrics import accuracy_score, f1_score
import pandas as pd
import pickle as pkl
from matplotlib import pyplot as plt
DEVICE = torch.device("cuda:0")
#DEVICE = torch.device("cpu")
with open(f"mosi.pkl",mode="rb") as handle:
  data = pickle.load(handle)
train_data = data["train"]
dev_data = data["dev"]
test_data = data["test"]
# print(len(train_data))


gradient_accumulation_step = 1
tokenizer = BertTokenizer.from_pretrained("bert-base-uncased")
def prep_train_dataloader():
    features = []
    for (ex_index, example) in enumerate(train_data):
        (words, visual, acoustic), label_id, segment = example
        tokens, inversions = [], []
        # print("visual.shape", visual.shape)
        # print("acoustic.shape", acoustic.shape)
        # print(words)
        for (idx, word) in enumerate(words):
            tokenized = tokenizer.tokenize(word)
            tokens.extend(tokenized)
            inversions.extend([idx] * len(tokenized))
            # print(tokenized)
        # if ex_index == 5:
        #   break

        aligned_visual = []
        aligned_acoustic = []
        for inv_idx in inversions:
            aligned_visual.append(visual[inv_idx, :])
            aligned_acoustic.append(acoustic[inv_idx, :])

        visual = np.array(aligned_visual)
        acoustic = np.array(aligned_acoustic)

        if len(tokens) > max_seq_length - 2:
            tokens = tokens[: max_seq_length - 2]
            acoustic = acoustic[: max_seq_length - 2]
            visual = visual[: max_seq_length - 2]

        # 将token进行转换
        CLS = tokenizer.cls_token
        SEP = tokenizer.sep_token
        tokens = [CLS] + tokens + [SEP]
        acoustic_zero = np.zeros((1, acoustic_dim))
        visual_zero = np.zeros((1, visual_dim))
        acoustic = np.concatenate((acoustic_zero, acoustic, acoustic_zero))
        visual = np.concatenate((visual_zero, visual, visual_zero))

        input_ids = tokenizer.convert_tokens_to_ids(tokens)
        segment_ids = [0] * len(input_ids)
        input_mask = [1] * len(input_ids)

        # 加PAD
        pad_length = max_seq_length - len(input_ids)
        acoustic_padding = np.zeros((pad_length, acoustic_dim))
        acoustic = np.concatenate((acoustic, acoustic_padding))

        visual_padding = np.zeros((pad_length, visual_dim))
        visual = np.concatenate((visual, visual_padding))

        padding = [0] * pad_length
        input_ids += padding
        input_mask += padding
        segment_ids += padding
        features.append((input_ids, segment_ids, input_mask, acoustic, visual, label_id))

    # 把数组列表转换为tensor类型
    all_input_ids = torch.tensor([f[0] for f in features], dtype=torch.long)
    all_segment_ids = torch.tensor([f[1] for f in features], dtype=torch.long)
    all_input_mask = torch.tensor([f[2] for f in features], dtype=torch.long)
    all_acoustic = torch.tensor([f[3] for f in features], dtype=torch.float)
    all_visual = torch.tensor([f[4] for f in features], dtype=torch.float)
    all_label_ids = torch.tensor([f[5] for f in features], dtype=torch.float)

    train_dataset = TensorDataset(all_input_ids, all_segment_ids, all_input_mask, all_acoustic, all_visual, all_label_ids)
    train_dataloader = DataLoader(train_dataset, batch_size=train_batch_size, shuffle=True)
    num_train_optimization_steps = (int(len(train_dataset) / train_batch_size / gradient_accumulation_step) * n_epochs)
    return num_train_optimization_steps,train_dataloader

def prep_dev_dataloader():
    features = []
    for (ex_index, example) in enumerate(dev_data):
        (words, visual, acoustic), label_id, segment = example
        tokens, inversions = [], []
        # print("visual.shape", visual.shape)
        # print("acoustic.shape", acoustic.shape)
        # print(words)
        for (idx, word) in enumerate(words):
            tokenized = tokenizer.tokenize(word)
            tokens.extend(tokenized)
            inversions.extend([idx] * len(tokenized))
            # print(tokenized)
        # if ex_index == 5:
        #   break

        aligned_visual = []
        aligned_acoustic = []
        for inv_idx in inversions:
            aligned_visual.append(visual[inv_idx, :])
            aligned_acoustic.append(acoustic[inv_idx, :])

        visual = np.array(aligned_visual)
        acoustic = np.array(aligned_acoustic)

        if len(tokens) > max_seq_length - 2:
            tokens = tokens[: max_seq_length - 2]
            acoustic = acoustic[: max_seq_length - 2]
            visual = visual[: max_seq_length - 2]

        # 将token进行转换
        CLS = tokenizer.cls_token
        SEP = tokenizer.sep_token
        tokens = [CLS] + tokens + [SEP]
        acoustic_zero = np.zeros((1, acoustic_dim))
        visual_zero = np.zeros((1, visual_dim))
        acoustic = np.concatenate((acoustic_zero, acoustic, acoustic_zero))
        visual = np.concatenate((visual_zero, visual, visual_zero))

        input_ids = tokenizer.convert_tokens_to_ids(tokens)
        segment_ids = [0] * len(input_ids)
        input_mask = [1] * len(input_ids)

        # 加PAD
        pad_length = max_seq_length - len(input_ids)
        acoustic_padding = np.zeros((pad_length, acoustic_dim))
        acoustic = np.concatenate((acoustic, acoustic_padding))

        visual_padding = np.zeros((pad_length, visual_dim))
        visual = np.concatenate((visual, visual_padding))

        padding = [0] * pad_length
        input_ids += padding
        input_mask += padding
        segment_ids += padding
        features.append((input_ids, segment_ids, input_mask, acoustic, visual, label_id))

    # 把数组列表转换为tensor类型
    all_input_ids = torch.tensor([f[0] for f in features], dtype=torch.long)
    all_segment_ids = torch.tensor([f[1] for f in features], dtype=torch.long)
    all_input_mask = torch.tensor([f[2] for f in features], dtype=torch.long)
    all_acoustic = torch.tensor([f[3] for f in features], dtype=torch.float)
    all_visual = torch.tensor([f[4] for f in features], dtype=torch.float)
    all_label_ids = torch.tensor([f[5] for f in features], dtype=torch.float)

    dev_dataset = TensorDataset(all_input_ids, all_segment_ids, all_input_mask, all_acoustic, all_visual,
                                  all_label_ids)
    dev_dataloader = DataLoader(dev_dataset, batch_size=dev_batch_size, shuffle=True)
    return dev_dataloader
def prep_test_dataloader():
    features = []
    for (ex_index, example) in enumerate(test_data):
        (words, visual, acoustic), label_id, segment = example
        tokens, inversions = [], []
        # print("visual.shape", visual.shape)
        # print("acoustic.shape", acoustic.shape)
        # print(words)
        for (idx, word) in enumerate(words):
            tokenized = tokenizer.tokenize(word)
            tokens.extend(tokenized)
            inversions.extend([idx] * len(tokenized))
            # print(tokenized)
        # if ex_index == 5:
        #   break

        aligned_visual = []
        aligned_acoustic = []
        for inv_idx in inversions:
            aligned_visual.append(visual[inv_idx, :])
            aligned_acoustic.append(acoustic[inv_idx, :])

        visual = np.array(aligned_visual)
        acoustic = np.array(aligned_acoustic)

        if len(tokens) > max_seq_length - 2:
            tokens = tokens[: max_seq_length - 2]
            acoustic = acoustic[: max_seq_length - 2]
            visual = visual[: max_seq_length - 2]

        # 将token进行转换
        CLS = tokenizer.cls_token
        SEP = tokenizer.sep_token
        tokens = [CLS] + tokens + [SEP]
        acoustic_zero = np.zeros((1, acoustic_dim))
        visual_zero = np.zeros((1, visual_dim))
        acoustic = np.concatenate((acoustic_zero, acoustic, acoustic_zero))
        visual = np.concatenate((visual_zero, visual, visual_zero))

        input_ids = tokenizer.convert_tokens_to_ids(tokens)
        segment_ids = [0] * len(input_ids)
        input_mask = [1] * len(input_ids)

        # 加PAD
        pad_length = max_seq_length - len(input_ids)
        acoustic_padding = np.zeros((pad_length, acoustic_dim))
        acoustic = np.concatenate((acoustic, acoustic_padding))

        visual_padding = np.zeros((pad_length, visual_dim))
        visual = np.concatenate((visual, visual_padding))

        padding = [0] * pad_length
        input_ids += padding
        input_mask += padding
        segment_ids += padding
        features.append((input_ids, segment_ids, input_mask, acoustic, visual, label_id))

    # 把数组列表转换为tensor类型
    all_input_ids = torch.tensor([f[0] for f in features], dtype=torch.long)
    all_segment_ids = torch.tensor([f[1] for f in features], dtype=torch.long)
    all_input_mask = torch.tensor([f[2] for f in features], dtype=torch.long)
    all_acoustic = torch.tensor([f[3] for f in features], dtype=torch.float)
    all_visual = torch.tensor([f[4] for f in features], dtype=torch.float)
    all_label_ids = torch.tensor([f[5] for f in features], dtype=torch.float)

    test_dataset = TensorDataset(all_input_ids, all_segment_ids, all_input_mask, all_acoustic, all_visual,
                                  all_label_ids)
    test_dataloader = DataLoader(test_dataset, batch_size=test_batch_size, shuffle=True)
    return test_dataloader

# preparing for train:optimizer, schedular
# optimizer
def prep_for_training(num_train_optimization_steps):
    model = Net()
    model.to(DEVICE)


    param_optimizer = list(model.named_parameters())
    no_decay = ["bias", "LayerNorm.bias", "LayerNorm.weight"]
    optimizer_grouped_parameters = [
        {
            "params": [
                p for n, p in param_optimizer if not any(nd in n for nd in no_decay)
            ],
            "weight_decay": 0.01
        },
        {
            "params": [
                p for n, p in param_optimizer if any(nd in n for nd in no_decay)
            ],
            "weight_decay": 0.0
        }
    ]
    optimizer = AdamW(optimizer_grouped_parameters, lr=1e-5)

    scheduler = get_linear_schedule_with_warmup(optimizer, num_warmup_steps=0.1 * num_train_optimization_steps,
                                                num_training_steps=num_train_optimization_steps)
    return model, optimizer, scheduler

# train_epoch
from torch.nn import MSELoss
from tqdm import tqdm

def train_epoch(model:nn.Module, train_dataloader:DataLoader, optimizer, scheduler):
    model.train()
    tr_loss = 0
    nb_tr_steps = 0
    for step, batch in enumerate(tqdm(train_dataloader, desc="Iteration")):
        batch = tuple(t.to(DEVICE) for t in batch)
        input_ids, segment_ids, input_mask, acoustic, visual, label_ids = batch
        output = model(input_ids=input_ids, segment_ids=segment_ids, input_mask=input_mask, acoustic=acoustic, visual=visual)
        loss_func = MSELoss()
        loss = loss_func(output.view(-1),label_ids.view(-1))
        loss.backward()
        tr_loss += loss.item()
        nb_tr_steps += 1
        optimizer.step()
        scheduler.step()
        optimizer.zero_grad()
    return tr_loss/nb_tr_steps

def dev_epoch(model,dev_dataloader):
    model.eval()
    dev_loss = 0
    nb_dev_steps = 0

    with torch.no_grad():
        for step, batch in enumerate(tqdm(dev_dataloader,desc="Iteration")):
            batch = tuple(t.to(DEVICE) for t in batch)
            input_ids, segment_ids, input_mask, acoustic, visual, label_ids = batch
            output = model(input_ids=input_ids, segment_ids=segment_ids, input_mask=input_mask, acoustic=acoustic,
                           visual=visual)
            loss_func = MSELoss()
            loss = loss_func(output.view(-1), label_ids.view(-1))
            dev_loss+=loss.item()
            nb_dev_steps += 1
    return dev_loss/nb_dev_steps

def test_epoch(model,test_dataloader):
    model.eval()
    preds = []
    labels = []

    with torch.no_grad():
        for batch in tqdm(test_dataloader):
            batch = tuple(t.to(DEVICE) for t in batch)
            input_ids, segment_ids, input_mask, acoustic, visual, label_ids = batch
            output = model(input_ids=input_ids, segment_ids=segment_ids, input_mask=input_mask, acoustic=acoustic,
                           visual=visual)
            pred= output.detach().cpu().numpy()
            label =label_ids.detach().cpu().numpy()

            pred =np.squeeze(pred).tolist()
            label = np.squeeze(label).tolist()
            preds.extend(pred)
            labels.extend(label)
    preds=np.array(preds)
    labels = np.array(labels)
    return preds, labels
def test_score(model,test_dataloader):
    preds, y_test =test_epoch(model,test_dataloader)
    non_zeros =np.array([i for i,e in enumerate(y_test) if e!=0])
    preds_non = preds[non_zeros]
    y_test_non = y_test[non_zeros]
    #回归
    mae_non = np.mean(np.absolute(preds_non-y_test_non))
    corr_non =np.corrcoef(preds_non,y_test_non)[0][1]
    #分类
    preds_non = preds_non>=0
    y_test_non = y_test_non>=0
    acc_non = accuracy_score(y_test_non, preds_non)
    f_score_non =f1_score(y_test_non,preds_non,average="weighted")

    #acc,f_score
    preds = preds>=0
    y_test = y_test>=0
    acc = accuracy_score(y_test,preds)
    f_score = f1_score(y_test,preds)

    return mae_non,corr_non,acc,acc_non,f_score,f_score_non


train_losses = []
dev_losses = []
test_maes = []
test_corrs = []
test_accs = []
test_accs_non = []
test_f_scores = []
test_f_scores_non = []

num_train_optimization_steps,train_dataloader = prep_train_dataloader()
dev_dataloader = prep_dev_dataloader()
test_dataloader = prep_test_dataloader()
model, optimizer, scheduler = prep_for_training(num_train_optimization_steps)
for epoch_i in range(n_epochs):
    train_loss = train_epoch(model,train_dataloader,optimizer,scheduler)
    dev_loss = dev_epoch(model,dev_dataloader)
    test_mae,test_corr,test_acc,test_acc_non,test_f_score,test_f_score_non = test_score(model,test_dataloader)
    print(str(epoch_i)+" "+str(train_loss)+" "+str(dev_loss)+" "+str(test_acc))

    train_losses.append(train_loss)
    dev_losses.append(dev_loss)
    test_maes.append(test_mae)
    test_corrs.append(test_corr)
    test_accs.append(test_acc)
    test_accs_non.append(test_acc_non)
    test_f_scores.append(test_f_score)
    test_f_scores_non.append(test_f_score_non)
result = pd.DataFrame({"train_losses":train_losses, "dev_losses":dev_losses,
            "test_maes":test_maes, "test_corrs":test_corrs,
            "test_accs":test_accs,"test_accs_non":test_accs_non,
                        "test_f_scores":test_f_scores,"test_f_scores_non":test_f_scores_non})
with open("./result.pkl", "wb") as f:
    pkl.dump(result,f)


with open("./result.pkl", "rb") as f:
  result = pkl.load(f)
#画图
df = result.loc[:,["train_losses","dev_losses"]]

df.plot()
plt.show()
pd.set_option('display.max_columns', None) #显示所有列
pd.set_option('display.max_rows', None) #显示所有行

print(result)<jupyter_output>train_losses  dev_losses  test_maes  test_corrs  test_accs  \
0        2.604475    3.322559   1.445997    0.063093   0.529927   
1        2.451070    2.757319   1.458334    0.104218   0.547445   
2        2.340258    2.717255   1.457540    0.177147   0.560584   
3        2.159589    2.855980   1.426880    0.268915   0.604380   
4        2.099602    2.821632   1.377694    0.337460   0.640876   
5        1.953586    2.353086   1.326939    0.383336   0.661314   
6        1.804030    2.206401   1.278168    0.426317   0.683212   
7        1.706515    1.946442   1.218201    0.480129   0.696350   
8        1.357860    1.860076   1.117955    0.547816   0.745985   
9        1.155092    1.578946   1.029191    0.630905   0.783942   
10       0.965999    1.416269   0.977773    0.681684   0.789781   
11       0.814803    1.299833   0.918530    0.708833   0.798540   
12       0.683191    1.361075   0.879591    0.722197   0.805839   
13       0.624013    1.238963   0.888855    0.728813   0.80292[...]
{'fraction_non_alphanumeric': 0.10956136975759907, 'fraction_numerical': 0.03443632166217776, 'mean_word_length': 2.887995512341062, 'pattern_counts': {'":': 12, '<': 4, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '353771', 'license': 'MIT', 'n_tokens_mistral': 7752, 'n_tokens_neox': 7277, 'n_words': 1323, 'path': 'Multi_Hop_v1.ipynb', 'repo_name': 'zhaohongwei1998/SimCSE', 'size': '20792'}