<jupyter_start><jupyter_text>导包<jupyter_code>import nlp_basictasks
import os,json
import numpy as np
import torch
import torch.nn as nn
import random
from tqdm.autonotebook import tqdm, trange
from torch.utils.data import DataLoader
from nlp_basictasks.modules import SBERT
from nlp_basictasks.modules.transformers import BertTokenizer,BertModel,BertConfig
from nlp_basictasks.readers.sts import InputExample,convert_examples_to_features,getExamples,convert_sentences_to_features
from nlp_basictasks.modules.utils import get_optimizer,get_scheduler
from nlp_basictasks.Trainer import Trainer
from nlp_basictasks.evaluation import stsEvaluator
from sentence_transformers import SentenceTransformer,models
model_path1='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/distill-simcse/'
# model_path2="/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/distiluse-base-multilingual-cased-v1/"
model_path3='/data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/'
# data_folder='/data/nfs14/nfs/aisearch/asr/xhsun/datasets/lcqmc/'
# train_file=os.path.join(data_folder,'lcqmc_train.tsv')
# dev_file=os.path.join(data_folder,'lcqmc_dev.tsv')
#tokenizer=BertTokenizer.from_pretrained(os.path.join(model_path1,'0_Transformer'))
tokenizer=BertTokenizer.from_pretrained(model_path3)
max_seq_len=64
batch_size=128<jupyter_output>Better speed can be achieved with apex installed from https://www.github.com/nvidia/apex.<jupyter_text>获取数据<jupyter_code>train_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-train.txt'
dev_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-dev.txt'
test_file='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/STS-B/cnsd-sts-test.txt'
def read_data(file_path):
    sentences=[]
    labels=[]
    with open(file_path) as f:
        lines=f.readlines()
    for line in lines:
        line_split=line.strip().split('||')
        sentences.append([line_split[1],line_split[2]])
        labels.append(line_split[3])
    return sentences,labels
train_sentences,train_labels=read_data(train_file)
dev_sentences,dev_labels=read_data(dev_file)
test_sentences,test_labels=read_data(test_file)
print(train_sentences[:2],train_labels[:2])
print(dev_sentences[:2],dev_labels[:2])
print(test_sentences[:2],test_labels[:2])
test_sentences[0]<jupyter_output><empty_output><jupyter_text>create unsupervised train_dataset<jupyter_code>train_sentences=[sentence[0] for sentence in train_sentences]#只取一般数据作为训练集
print(len(train_sentences))
print(train_sentences[:3])
train_examples=[InputExample(text_list=[sentence,sentence],label=1) for sentence in train_sentences]
train_dataloader=DataLoader(train_examples,shuffle=True,batch_size=batch_size)
def smart_batching_collate(batch):
    features_of_a,features_of_b,labels=convert_examples_to_features(examples=batch,tokenizer=tokenizer,max_seq_len=max_seq_len)
    return features_of_a,features_of_b,labels
train_dataloader.collate_fn=smart_batching_collate
print(train_examples[0])<jupyter_output>5231
['一架飞机要起飞了。', '一个男人在吹一支大笛子。', '一个人正把切碎的奶酪撒在比萨饼上。']
<InputExample> label: 1, text pairs : 一架飞机要起飞了。; 一架飞机要起飞了。<jupyter_text>create supervised train_dataset<jupyter_code>print(len(train_sentences))
print(train_sentences[:3])
#train_examples=[InputExample(text_list=sentence,label=1) for sentence in train_sentences]
train_dataloader=DataLoader(train_examples,shuffle=True,batch_size=batch_size)
def smart_batching_collate(batch):
    features_of_a,features_of_b,labels=convert_examples_to_features(examples=batch,tokenizer=tokenizer,max_seq_len=max_seq_len)
    return features_of_a,features_of_b,labels
train_dataloader.collate_fn=smart_batching_collate
print(train_examples[0])<jupyter_output><empty_output><jupyter_text>SimCSE模型<jupyter_code>class SimCSE(nn.Module):
    def __init__(self,
                 bert_model_path,
                 is_sbert_model=True,
                temperature=0.05,
                is_distilbert=False,
                device='cpu'):
        super(SimCSE,self).__init__()
        if is_sbert_model:
            self.encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)
        else:
            word_embedding_model = models.Transformer(bert_model_path, max_seq_length=max_seq_len)
            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
            self.encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)
        self.temperature=temperature
        self.is_distilbert=is_distilbert#蒸馏版本的BERT不支持token_type_ids
    def cal_cos_sim(self,embeddings1,embeddings2):
        embeddings1_norm=torch.nn.functional.normalize(embeddings1,p=2,dim=1)
        embeddings2_norm=torch.nn.functional.normalize(embeddings2,p=2,dim=1)
        return torch.mm(embeddings1_norm,embeddings2_norm.transpose(0,1))#(batch_size,batch_size)
        
    def forward(self,batch_inputs):
        '''
        为了实现兼容，所有model的batch_inputs最后一个位置必须是labels，即使为None
        get token_embeddings,cls_token_embeddings,sentence_embeddings
        sentence_embeddings是经过Pooling层后concat的embedding。维度=768*k，其中k取决于pooling的策略
        一般来讲，只会取一种pooling策略，要么直接cls要么mean last or mean last2 or mean first and last layer，所以sentence_embeddings的维度也是768
        '''
        batch1_features,batch2_features,_=batch_inputs
        if self.is_distilbert:
            del batch1_features['token_type_ids']
            del batch2_features['token_type_ids']
        batch1_embeddings=self.encoder(batch1_features)['sentence_embedding']
        batch2_embeddings=self.encoder(batch2_features)['sentence_embedding']
        cos_sim=self.cal_cos_sim(batch1_embeddings,batch2_embeddings)/self.temperature#(batch_size,batch_size)
        batch_size=cos_sim.size(0)
        assert cos_sim.size()==(batch_size,batch_size)
        labels=torch.arange(batch_size).to(cos_sim.device)
        return nn.CrossEntropyLoss()(cos_sim,labels)
    
    def encode(self, sentences,
               batch_size: int = 32,
               show_progress_bar: bool = None,
               output_value: str = 'sentence_embedding',
               convert_to_numpy: bool = True,
               convert_to_tensor: bool = False,
               device: str = None,
               normalize_embeddings: bool = False):
        '''
        传进来的sentences只能是single_batch
        '''
        return self.encoder.encode(sentences=sentences,
                                         batch_size=batch_size,
                                         show_progress_bar=show_progress_bar,
                                         output_value=output_value,
                                         convert_to_numpy=convert_to_numpy,
                                         convert_to_tensor=convert_to_tensor,
                                         device=device,
                                         normalize_embeddings=normalize_embeddings)
    
    def save(self,output_path):
        os.makedirs(output_path,exist_ok=True)
        with open(os.path.join(output_path, 'model_param_config.json'), 'w') as fOut:
            json.dump(self.get_config_dict(output_path), fOut)
        self.encoder.save(output_path)
        
    def get_config_dict(self,output_path):
        '''
        一定要有dict，这样才能初始化Model
        '''
        return {'output_path':output_path,'temperature': self.temperature, 'is_distilbert': self.is_distilbert}
    @staticmethod
    def load(input_path):
        with open(os.path.join(input_path, 'model_param_config.json')) as fIn:
            config = json.load(fIn)
        return SimCSE(**config)
device='cpu'
#simcse=SimCSE(bert_model_path=model_path3,is_distilbert=False,device=device,is_sbert_model=False)
simcse=SimCSE(bert_model_path="/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/",is_distilbert=False,device=device,is_sbert_model=True)<jupyter_output>2021-10-19 08:23:00 - INFO - __init__ - 41 : Load pretrained SentenceTransformer: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/
2021-10-19 08:23:00 - INFO - __init__ - 107 : Load SentenceTransformer from folder: /data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unSimCSE_STS-B/<jupyter_text>构造evaluator<jupyter_code>#dev_sentences=[example.text_list for example in dev_examples]
#dev_labels=[example.label for example in dev_examples]
print(dev_sentences[0],dev_labels[0])
sentences1_list=[sen[0] for sen in dev_sentences]
sentences2_list=[sen[1] for sen in dev_sentences]
dev_labels=[int(score) for score in dev_labels]
evaluator=stsEvaluator(sentences1=sentences1_list,sentences2=sentences2_list,batch_size=64,write_csv=True,scores=dev_labels)
evaluator(simcse)
sentences1_list[0]
sentences2_list[0]
dev_labels[0]
sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)
sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)
sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)
sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)
(sentences1_embeddings-sentences2_embeddings).norm(dim=1,or).size()
sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)
sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)
(sentences1_embeddings-sentences2_embeddings).norm(dim=1,or).size()
s1=torch.pdist(sentences1_embeddings,p=2).pow(2).mul(-2).exp().mean().log()
s2=torch.pdist(sentences2_embeddings,p=2).pow(2).mul(-2).exp().mean().log()
(s1+s2)/2
sentence="数据转换方式决定了最终学习的向量表示的不变性"
v1=simcse.encode(sentence,normalize_embeddings=True,show_progress_bar=False)
v2=simcse.encode(sentence[::-1],normalize_embeddings=True,show_progress_bar=False)
(v1*v2).sum()
sentence[::-1]
bert_encoder=SimCSE(model_path3,is_sbert_model=False,is_distilbert=False)<jupyter_output>Some weights of the model checkpoint at /data/nfs14/nfs/aisearch/asr/xhsun/CommonModel/chinese-roberta-wwm/ were not used when initializing BertModel: ['cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.seq_relationship.bias', 'cls.predictions.transform.dense.weight']
- This IS expected if you are initializing BertModel from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).
- This IS NOT expected if you are initializing BertModel from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).<jupyter_text>展示contrastive loss的计算过程  首先将每一个句子编码为向量<jupyter_code>s1="这几行代码为了展示对比损失函数的计算"
s1_pie=s1
s2="今天组会的内容是对比学习在文本相似度计算任务的应用"
s2_pie=s2
z1=bert_encoder.encode(s1)
z1_pie=bert_encoder.encode(s1_pie)
z2=bert_encoder.encode(s2)
z2_pie=bert_encoder.encode(s2_pie)<jupyter_output><empty_output><jupyter_text>normalize每一个向量<jupyter_code>v1=z1/np.linalg.norm(z1,ord=2)
v1_pie=z1_pie/np.linalg.norm(z1_pie,ord=2)
v2=z2/np.linalg.norm(z2,ord=2)
v2_pie=z2_pie/np.linalg.norm(z2_pie,ord=2)<jupyter_output><empty_output><jupyter_text>得到logits和labels<jupyter_code>logits=torch.tensor([[np.sum(v1*v1_pie),np.sum(v1*v2_pie)],
                     [np.sum(v2*v1_pie),np.sum(v2*v2_pie)]])
labels=torch.LongTensor([[1,0],[0,1]])
print(logits)
print(labels)<jupyter_output>tensor([[1.0000, 0.8564],
        [0.8564, 1.0000]])
tensor([[1, 0],
        [0, 1]])<jupyter_text>对logits进行softmax<jupyter_code>log_probs=torch.nn.functional.log_softmax(logits,dim=1)
log_probs
contrastive_loss=-(log_probs[0][0] + log_probs[1][1])/2
contrastive_loss
contrastive_loss=-torch.sum(torch.nn.functional.log_softmax(logits,dim=1)*labels)/2
contrastive_loss
3+-1
from scipy.stats import pearsonr,spearmanr
X=[0.9,0.7,0.3,0.8,0.6]
Y=[4,5,1,3,2]
rank_X=[5,3,1,4,2]
d_i=[-1,2,0,1,0]
d_i2=[1,4,0,1,0]
print("spearman rank correlation coefficient: ",1-(6*sum(d_i2))/(5*(5**2-1)))
print("spearman rank correlation coefficient: ",spearmanr(a=rank_X,b=Y)[0])
print("pearson correlation coefficient: ",pearsonr(X,Y)[0])<jupyter_output>pearson correlation coefficient:  0.7554831017177897<jupyter_text>ConSERT模型<jupyter_code>class ConSERT(nn.Module):
    def __init__(self,
                 bert_model_path,
                 is_sbert_model=True,
                temperature=0.05,
                is_distilbert=False,
                 cutoff_rate=0.15,
                device='cpu',
                close_dropout=True):
        super(ConSERT,self).__init__()
        if is_sbert_model:
            self.encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)
        else:
            word_embedding_model = models.Transformer(bert_model_path, max_seq_length=max_seq_len)
            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
            self.encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)
        self.temperature=temperature
        self.cutoff_rate=cutoff_rate
        self.is_distilbert=is_distilbert#蒸馏版本的BERT不支持token_type_ids
        self.close_dropout=close_dropout
        
    def cal_cos_sim(self,embeddings1,embeddings2):
        embeddings1_norm=torch.nn.functional.normalize(embeddings1,p=2,dim=1)
        embeddings2_norm=torch.nn.functional.normalize(embeddings2,p=2,dim=1)
        return torch.mm(embeddings1_norm,embeddings2_norm.transpose(0,1))#(batch_size,batch_size)

    def shuffle_and_cutoff(self,sentence_feature):
        input_ids, attention_mask=sentence_feature['input_ids'],sentence_feature['attention_mask']
        bsz, seq_len = input_ids.shape
        shuffled_input_ids=[]
        cutoff_attention_mask=[]
        for bsz_id in range(bsz):
            sample_mask = attention_mask[bsz_id]
            num_tokens = sample_mask.sum().int().item()
            cur_input_ids=input_ids[bsz_id]
            if 102 not in cur_input_ids:
                indexes = list(range(num_tokens))[1:]
                random.shuffle(indexes)
                indexes=[0]+indexes#保证第一个位置是0
            else:
                indexes = list(range(num_tokens))[1:-1]
                random.shuffle(indexes)
                indexes=[0]+indexes+[num_tokens-1]#保证第一个位置是0，最后一个位置是SEP不变
            rest_indexes = list(range(num_tokens, seq_len))
            total_indexes = indexes + rest_indexes
            shuffled_input_id=input_ids[bsz_id][total_indexes]
            #print(shuffled_input_id,indexes)
            if self.cutoff_rate>0.0:
                sample_len=max(int(num_tokens*(1-self.cutoff_rate)),1)#if true_len is 32, cutoff_rate is 0.15 then sample_len is 27
                start_id = np.random.randint(1, high=num_tokens-sample_len+1)# start_id random select from (0,6)，避免删除CLS
                cutoff_mask=[1]*seq_len
                for idx in range(start_id, start_id+sample_len):
                    cutoff_mask[idx]=0#这些位置是0，bool之后就变成了False，而masked_fill是选择True的位置替换为value的
                cutoff_mask[0]=0#避免CLS被替换
                cutoff_mask[num_tokens-1]=0#避免SEP被替换
                cutoff_mask=torch.ByteTensor(cutoff_mask).bool().to(input_ids.device)
                shuffled_input_id=shuffled_input_id.masked_fill(cutoff_mask,value=0).to(input_ids.device)
                sample_mask=sample_mask.masked_fill(cutoff_mask,value=0).to(input_ids.device)

            shuffled_input_ids.append(shuffled_input_id)
            cutoff_attention_mask.append(sample_mask)
        shuffled_input_ids=torch.vstack(shuffled_input_ids)
        cutoff_attention_mask=torch.vstack(cutoff_attention_mask)
        return shuffled_input_ids,cutoff_attention_mask
        
    def forward(self,batch_inputs):
        '''
        为了实现兼容，所有model的batch_inputs最后一个位置必须是labels，即使为None
        get token_embeddings,cls_token_embeddings,sentence_embeddings
        sentence_embeddings是经过Pooling层后concat的embedding。维度=768*k，其中k取决于pooling的策略
        一般来讲，只会取一种pooling策略，要么直接cls要么mean last or mean last2 or mean first and last layer，所以sentence_embeddings的维度也是768
        '''
        batch1_features,batch2_features,_=batch_inputs
        if self.is_distilbert:
            del batch1_features['token_type_ids']
            del batch2_features['token_type_ids']
        batch1_embeddings=self.encoder(batch1_features)['sentence_embedding']
        shuffled_input_ids,cutoff_attention_mask=self.shuffle_and_cutoff(sentence_feature=batch1_features)
        #new_features{'input_ids'}=shuffled_input_ids
        batch2_features['input_ids']=shuffled_input_ids
        batch2_features['attention_mask']=cutoff_attention_mask
        orig_attention_probs_dropout_prob=self.encoder[0].auto_model.encoder.config.attention_probs_dropout_prob
        orig_hidden_dropout_prob=self.encoder[0].auto_model.encoder.config.hidden_dropout_prob
        if self.close_dropout:
            self.encoder[0].auto_model.encoder.config.attention_probs_dropout_prob=0.0
            self.encoder[0].auto_model.encoder.config.hidden_dropout_prob=0.0
        batch2_embeddings=self.encoder(batch2_features)['sentence_embedding']
        if self.close_dropout:
            self.encoder[0].auto_model.encoder.config.attention_probs_dropout_prob=orig_attention_probs_dropout_prob
            self.encoder[0].auto_model.encoder.config.hidden_dropout_prob=orig_hidden_dropout_prob
            
        cos_sim=self.cal_cos_sim(batch1_embeddings,batch2_embeddings)/self.temperature#(batch_size,batch_size)
        batch_size=cos_sim.size(0)
        assert cos_sim.size()==(batch_size,batch_size)
        labels=torch.arange(batch_size).to(cos_sim.device)
        return nn.CrossEntropyLoss()(cos_sim,labels)
    
    def encode(self, sentences,
               batch_size: int = 32,
               show_progress_bar: bool = None,
               output_value: str = 'sentence_embedding',
               convert_to_numpy: bool = True,
               convert_to_tensor: bool = False,
               device: str = None,
               normalize_embeddings: bool = False):
        '''
        传进来的sentences只能是single_batch
        '''
        return self.encoder.encode(sentences=sentences,
                                         batch_size=batch_size,
                                         show_progress_bar=show_progress_bar,
                                         output_value=output_value,
                                         convert_to_numpy=convert_to_numpy,
                                         convert_to_tensor=convert_to_tensor,
                                         device=device,
                                         normalize_embeddings=normalize_embeddings)
    
    def save(self,output_path):
        os.makedirs(output_path,exist_ok=True)
        with open(os.path.join(output_path, 'model_param_config.json'), 'w') as fOut:
            json.dump(self.get_config_dict(output_path), fOut)
        self.encoder.save(output_path)
        
    def get_config_dict(self,output_path):
        '''
        一定要有dict，这样才能初始化Model
        '''
        return {'bert_model_path':output_path,'temperature': self.temperature, 'is_distilbert': self.is_distilbert,
               'close_dropout':self.close_dropout,'cutoff_rate':self.cutoff_rate}
    @staticmethod
    def load(input_path):
        with open(os.path.join(input_path, 'model_param_config.json')) as fIn:
            config = json.load(fIn)
            if 'bert_model_path' not in config:
                config['bert_model_path']=config['output_path']
                del config['output_path']
        return ConSERT(**config)
consert_closedropout=ConSERT.load("/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/unConSERT_STS-B_closedropout/")
evaluator(consert_closedropout)
consert_donotclosedropout=ConSERT(bert_model_path=model_path3,is_distilbert=False,is_sbert_model=False,close_dropout=False)
evaluator(consert_donotclosedropout)<jupyter_output>2021-10-13 16:40:02 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:<jupyter_text>EsimCSE<jupyter_code>from queue import Queue
class ESimCSE(nn.Module):
    def __init__(self,
                 bert_model_path,
                 q_size=256,
                 dup_rate=0.32,
                 is_sbert_model=True,
                temperature=0.05,
                is_distilbert=False,
                 gamma=0.99,
                device='cpu'):
        super(ESimCSE,self).__init__()
        if is_sbert_model:
            self.encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)
            self.moco_encoder=SentenceTransformer(model_name_or_path=bert_model_path,device=device)
        else:
            word_embedding_model = models.Transformer(bert_model_path, max_seq_length=max_seq_len)
            pooling_model = models.Pooling(word_embedding_model.get_word_embedding_dimension())
            self.encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)
            self.moco_encoder=SentenceTransformer(modules=[word_embedding_model, pooling_model],device=device)
        self.gamma=gamma
        self.q=[]
        self.q_size=q_size
        self.dup_rate=dup_rate
        self.temperature=temperature
        self.is_distilbert=is_distilbert#蒸馏版本的BERT不支持token_type_ids
    def cal_cos_sim(self,embeddings1,embeddings2):
        embeddings1_norm=torch.nn.functional.normalize(embeddings1,p=2,dim=1)
        embeddings2_norm=torch.nn.functional.normalize(embeddings2,p=2,dim=1)
        return torch.mm(embeddings1_norm,embeddings2_norm.transpose(0,1))#(batch_size,batch_size)

    def word_repetition(self,sentence_feature):
        input_ids, attention_mask, token_type_ids=sentence_feature['input_ids'].cpu().tolist(),sentence_feature['attention_mask'].cpu().tolist(),sentence_feature['token_type_ids'].cpu().tolist()
        bsz, seq_len = len(input_ids),len(input_ids[0])
        #print(bsz,seq_len)
        repetitied_input_ids=[]
        repetitied_attention_mask=[]
        repetitied_token_type_ids=[]
        rep_seq_len=seq_len
        for bsz_id in range(bsz):
            sample_mask = attention_mask[bsz_id]
            actual_len = sum(sample_mask)

            cur_input_id=input_ids[bsz_id]
            dup_len=random.randint(a=0,b=max(2,int(self.dup_rate*actual_len)))
            dup_word_index=random.sample(list(range(1,actual_len)),k=dup_len)
            
            r_input_id=[]
            r_attention_mask=[]
            r_token_type_ids=[]
            for index,word_id in enumerate(cur_input_id):
                if index in dup_word_index:
                    r_input_id.append(word_id)
                    r_attention_mask.append(sample_mask[index])
                    r_token_type_ids.append(token_type_ids[bsz_id][index])

                r_input_id.append(word_id)
                r_attention_mask.append(sample_mask[index])
                r_token_type_ids.append(token_type_ids[bsz_id][index])

            after_dup_len=len(r_input_id)
            #assert after_dup_len==actual_len+dup_len
            repetitied_input_ids.append(r_input_id)#+rest_input_ids)
            repetitied_attention_mask.append(r_attention_mask)#+rest_attention_mask)
            repetitied_token_type_ids.append(r_token_type_ids)#+rest_token_type_ids)

            assert after_dup_len==dup_len+seq_len
            if after_dup_len>rep_seq_len:
                rep_seq_len=after_dup_len

        for i in range(bsz):
            after_dup_len=len(repetitied_input_ids[i])
            pad_len=rep_seq_len-after_dup_len
            repetitied_input_ids[i]+=[0]*pad_len
            repetitied_attention_mask[i]+=[0]*pad_len
            repetitied_token_type_ids[i]+=[0]*pad_len

        repetitied_input_ids=torch.LongTensor(repetitied_input_ids)
        repetitied_attention_mask=torch.LongTensor(repetitied_attention_mask)
        repetitied_token_type_ids=torch.LongTensor(repetitied_token_type_ids)
        return {"input_ids":repetitied_input_ids,'attention_mask':repetitied_attention_mask,'token_type_ids':repetitied_token_type_ids}

    def forward(self,batch_inputs):
        '''
        为了实现兼容，所有model的batch_inputs最后一个位置必须是labels，即使为None
        get token_embeddings,cls_token_embeddings,sentence_embeddings
        sentence_embeddings是经过Pooling层后concat的embedding。维度=768*k，其中k取决于pooling的策略
        一般来讲，只会取一种pooling策略，要么直接cls要么mean last or mean last2 or mean first and last layer，所以sentence_embeddings的维度也是768
        '''
        batch1_features,batch2_features,_=batch_inputs
        if self.is_distilbert:
            del batch1_features['token_type_ids']
            del batch2_features['token_type_ids']
        batch1_embeddings=self.encoder(batch1_features)['sentence_embedding']
        batch2_features=self.word_repetition(sentence_feature=batch2_features)
        batch2_embeddings=self.encoder(batch2_features)['sentence_embedding']
        cos_sim=self.cal_cos_sim(batch1_embeddings,batch2_embeddings)/self.temperature#(batch_size,batch_size)
        batch_size=cos_sim.size(0)
        assert cos_sim.size()==(batch_size,batch_size)
        labels=torch.arange(batch_size).to(cos_sim.device)
        negative_samples=None
        if len(self.q)>0:
            negative_samples=torch.vstack(self.q[:self.q_size])#(q_size,768)
        if len(self.q)+batch_size>=self.q_size:
            del self.q[:batch_size]
            
        with torch.no_grad():
            self.moco_encoder[0].auto_model.encoder.config.attention_probs_dropout_prob=0.0
            self.moco_encoder[0].auto_model.encoder.config.hidden_dropout_prob=0.0
            self.q.extend(self.moco_encoder(batch1_features)['sentence_embedding'])
            
        if negative_samples is not None:
            batch_size+=negative_samples.size(0)#(N+M)
            cos_sim_with_neg=self.cal_cos_sim(batch1_embeddings,negative_samples)/self.temperature#(N,M) not (N,N) N is bsz
            cos_sim=torch.cat([cos_sim,cos_sim_with_neg],dim=1)#(N,N+M)
            #labels=
        for encoder_param,moco_encoder_param in zip(self.encoder.parameters(),self.moco_encoder.parameters()):
            moco_encoder_param.data=self.gamma*moco_encoder_param.data+(1.-self.gamma)*encoder_param.data
            
        return nn.CrossEntropyLoss()(cos_sim,labels)
    
    def encode(self, sentences,
               batch_size: int = 32,
               show_progress_bar: bool = None,
               output_value: str = 'sentence_embedding',
               convert_to_numpy: bool = True,
               convert_to_tensor: bool = False,
               device: str = None,
               normalize_embeddings: bool = False):
        '''
        传进来的sentences只能是single_batch
        '''
        return self.encoder.encode(sentences=sentences,
                                         batch_size=batch_size,
                                         show_progress_bar=show_progress_bar,
                                         output_value=output_value,
                                         convert_to_numpy=convert_to_numpy,
                                         convert_to_tensor=convert_to_tensor,
                                         device=device,
                                         normalize_embeddings=normalize_embeddings)
    
    def save(self,output_path):
        os.makedirs(output_path,exist_ok=True)
        with open(os.path.join(output_path, 'model_param_config.json'), 'w') as fOut:
            json.dump(self.get_config_dict(output_path), fOut)
        self.encoder.save(output_path)
        
    def get_config_dict(self,output_path):
        '''
        一定要有dict，这样才能初始化Model
        '''
        return {'bert_model_path':output_path,'temperature': self.temperature, 'is_distilbert': self.is_distilbert,
                'q_size':self.q_size,'dup_rate':self.dup_rate,'gamma':self.gamma}
    @staticmethod
    def load(input_path):
        with open(os.path.join(input_path, 'model_param_config.json')) as fIn:
            config = json.load(fIn)
        return ESimCSE(**config)
device='cpu'
esimcse=ESimCSE(bert_model_path='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE',
                is_distilbert=False,
                is_sbert_model=True,
                dup_rate=0.32,gamma=0.99,
                device=device)
evaluator(esimcse)
sentences1_list=[sen[0] for sen in test_sentences]
sentences2_list=[sen[1] for sen in test_sentences]
sentences1_embeddings=esimcse.encode(sentences1_list,convert_to_tensor=True)
sentences2_embeddings=esimcse.encode(sentences2_list,convert_to_tensor=True)
print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mean())
s1=torch.pdist(sentences1_embeddings,p=2).pow(2).mul(-2).exp().mean().log()
print(s1)
s2=torch.pdist(sentences2_embeddings,p=2).pow(2).mul(-2).exp().mean().log()
print(s2)
print((s1+s2)/2)
print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mul(-2).exp().mean())#.log())
mengbi_esimcse=(sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2)
def uniform_loss(x, t=2):
    return torch.pdist(x, p=2).pow(2).mul(-t).exp().mean().log()
uniform_loss(sentences1_embeddings)
uniform_loss(sentences2_embeddings)
def align_loss(x, y, alpha=2):
    return (x - y).norm(p=2, dim=1).pow(alpha).mean()
align_loss(sentences1_embeddings,sentences2_embeddings)
sentences1_embeddings=simcse.encode(sentences1_list,convert_to_tensor=True)
sentences2_embeddings=simcse.encode(sentences2_list,convert_to_tensor=True)
print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mean())
s1=torch.pdist(sentences1_embeddings,p=2).pow(2).mul(-2).exp().mean().log()
print(s1)
s2=torch.pdist(sentences2_embeddings,p=2).pow(2).mul(-2).exp().mean().log()
print(s2)
print((s1+s2)/2)
print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mul(-2).exp().mean().log())
mengbi_simcse=(sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2)
sum((mengbi_esimcse==mengbi_simcse).bool())
mengbi_esimcse.mul(-2).exp().mean()
mengbi_simcse.mul(-2).exp().mean()
bert_model=SimCSE(model_path3,is_sbert_model=False,is_distilbert=False)
sentences1_embeddings=bert_model.encode(sentences1_list,convert_to_tensor=True)
sentences2_embeddings=bert_model.encode(sentences2_list,convert_to_tensor=True)
print((sentences1_embeddings-sentences2_embeddings).norm(dim=1).pow(2).mean())
s1=torch.pdist(sentences1_embeddings,p=2).pow(2).mul(-2).exp().mean().log()
print(s1)
s2=torch.pdist(sentences2_embeddings,p=2).pow(2).mul(-2).exp().mean().log()
print(s2)
print((s1+s2)/2)<jupyter_output><empty_output><jupyter_text>train model<jupyter_code>epochs=5
output_path='/data/nfs14/nfs/aisearch/asr/xhsun/bwbd_recall/unsupervisedSTSModel/ESimCSE'
tensorboard_logdir=os.path.join(output_path,'log')<jupyter_output><empty_output><jupyter_text>get optimizer<jupyter_code>optimizer_type='AdamW'
scheduler='WarmupLinear'
warmup_proportion=0.1
optimizer_params={'lr': 2e-5}
weight_decay=0.01
num_train_steps = int(len(train_dataloader) * epochs)
warmup_steps = num_train_steps*warmup_proportion
optimizer = get_optimizer(model=esimcse,optimizer_type=optimizer_type,weight_decay=weight_decay,optimizer_params=optimizer_params)
scheduler = get_scheduler(optimizer, scheduler=scheduler, warmup_steps=warmup_steps, t_total=num_train_steps)<jupyter_output><empty_output><jupyter_text>get Trainer<jupyter_code>trainer=Trainer(epochs=epochs,output_path=output_path,tensorboard_logdir=tensorboard_logdir,early_stop_patience=20)
trainer.train(train_dataloader=train_dataloader,
             model=esimcse,
             optimizer=optimizer,
             scheduler=scheduler,
             evaluator=evaluator,
             )
evaluator(consert_donotclosedropout)<jupyter_output>2021-10-13 18:23:51 - INFO - __call__ - 72 : EmbeddingSimilarityEvaluator: Evaluating the model on  dataset:
{'fraction_non_alphanumeric': 0.1057668172690763, 'fraction_numerical': 0.019829317269076305, 'mean_word_length': 3.9415503875968994, 'pattern_counts': {'":': 1, '<': 54, '<?xml version=': 0, '>': 58, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 42, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '567967', 'license': 'MIT', 'n_tokens_mistral': 11485, 'n_tokens_neox': 11252, 'n_words': 1071, 'path': 'STS-B.ipynb', 'repo_name': 'xianghuisun/nlp-basictasks', 'size': '31872'}