<jupyter_start><jupyter_text>run BertForSequenceClassification on annotation-test<jupyter_code>import numpy as np

import torch

from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM, BertForSequenceClassification

# Load pre-trained model (weights)
model = BertForSequenceClassification.from_pretrained('bert-base-chinese')
# Load pre-trained model tokenizer (vocabulary)
tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
model.to("cuda")
# f = open("/data/wyang/ShortTextSemanticSimilarity/data/corpora/annotation/annotation_test.csv")
# acc = 0
# total = 0
# for l in f:
#     label, a, b = l.replace("\n", "").split("\t")
#     a_index = tokenize_index(a, tokenizer)
#     b_index = tokenize_index(b, tokenizer)
#     combine_index = a_index + b_index
#     segments_ids = [0] * len(a_index) + [1] * len(b_index)
#     # Convert inputs to PyTorch tensors
#     tokens_tensor = torch.tensor([combine_index])
#     segments_tensors = torch.tensor([segments_ids])
#     # Predict all tokens
#     predictions = model(tokens_tensor, segments_tensors)
#     predicted_index = torch.argmax(predictions).item()
#     if predicted_index == int(label):
#         acc += 1
#     total += 1

def get_p1(prediction_score_list, labels):
    f = open("/data/wyang/ShortTextSemanticSimilarity/data/corpora/annotation/annotation_test.csv")
    a2score_label = {}
    for line, p, s in zip(f, prediction_score_list, labels):
        label, a, b = line.replace("\n", "").split("\t")
        if a not in a2prediction_label:
            a2score_label[a] = []
        a2score_label[a].append((p, s))
    
    acc = 0
    for a in a2score_label:
        a2score_label[a] = sorted(a2score_label[a], key=lambda x: x[1], reverse=True)
        if a2score_label[a][0][0] > 0:
            acc += 1
            
    p1 = acc / len(a2score_label)
    
    return p1
        
        
def eval(model):
    model.eval()
    f = open("/data/wyang/ShortTextSemanticSimilarity/data/corpora/annotation/annotation_test.csv")
    prediction_score_list, prediction_index_list = [], []
    labels = []
    test_batch, testid_batch = [], []
    batch_size = 16
    device = "cuda"
    for l in f:
        label, a, b = l.replace("\n", "").split("\t")
        labels.append(int(label))
        a_index = tokenize_index(a, tokenizer)
        b_index = tokenize_index(b, tokenizer)
        combine_index = a_index + b_index
        segments_ids = [0] * len(a_index) + [1] * len(b_index)
        test_batch.append(torch.tensor(combine_index))
        testid_batch.append(torch.tensor(segments_ids))
        if len(test_batch) >= batch_size:
            # Convert inputs to PyTorch tensors
            tokens_tensor = torch.nn.utils.rnn.pad_sequence(test_batch, batch_first=True, padding_value=0).to(device)
            segments_tensors = torch.nn.utils.rnn.pad_sequence(testid_batch, batch_first=True, padding_value=0).to(device)
            # Predict all tokens
            predictions = model(tokens_tensor, segments_tensors)
            predicted_index = list(torch.argmax(predictions, dim=1).cpu().numpy())
            prediction_index_list += predicted_index
            predicted_score = list(predictions[:, 1].cpu().detach().numpy())
            prediction_score_list.extend(predicted_score)
            
    if len(test_batch) > 0:
        # Convert inputs to PyTorch tensors
        tokens_tensor = torch.nn.utils.rnn.pad_sequence(test_batch, batch_first=True, padding_value=0).to(device)
        segments_tensors = torch.nn.utils.rnn.pad_sequence(testid_batch, batch_first=True, padding_value=0).to(device)
        # Predict all tokens
        predictions = model(tokens_tensor, segments_tensors)
        predicted_index = list(torch.argmax(predictions, dim=1).cpu().numpy())
        prediction_index_list += predicted_index
        predicted_score = list(predictions[:, 1].cpu().detach().numpy())
        prediction_score_list.extend(predicted_score)
    
    return prediction_score_list, prediction_index_list, labels
prediction_score_list, prediction_index_list, labels = eval(model)
import numpy as np
sum(np.array(prediction_list) == np.array(labels))
np.array(prediction_list).shape<jupyter_output><empty_output><jupyter_text>fine tune BertForSequenceClassification on annotation-train<jupyter_code>def tokenize_index(text, tokenizer):
    tokenized_text = tokenizer.tokenize(text)
    # Convert token to vocabulary indices
    indexed_tokens = tokenizer.convert_tokens_to_ids(tokenized_text)
    return indexed_tokens
from tqdm import tqdm
import random 

import torch

from pytorch_pretrained_bert import BertTokenizer, BertModel, BertForMaskedLM, BertForSequenceClassification
from pytorch_pretrained_bert.optimization import BertAdam


# Load pre-trained model (weights)
model = BertForSequenceClassification.from_pretrained('bert-base-chinese')
# Load pre-trained model tokenizer (vocabulary)
tokenizer = BertTokenizer.from_pretrained('bert-base-chinese')
model.to("cuda")
f = open("/data/wyang/ShortTextSemanticSimilarity/data/corpora/annotation/annotation_train.csv")
predictions = []
labels = []
test_batch, testid_batch, mask_batch, label_batch = [], [], [], []
batch_size = 16
train_data_set = []
device = "cuda"
for l in f:
    label, a, b = l.replace("\n", "").split("\t")
    labels.append(int(label))
    a_index = tokenize_index(a, tokenizer)
    b_index = tokenize_index(b, tokenizer)
    combine_index = a_index + b_index
    segments_ids = [0] * len(a_index) + [1] * len(b_index)
    test_batch.append(torch.tensor(combine_index))
    testid_batch.append(torch.tensor(segments_ids))
    mask_batch.append(torch.ones(len(combine_index)))
    label_batch.append(int(label))
    if len(test_batch) >= batch_size:
        # Convert inputs to PyTorch tensors
        tokens_tensor = torch.nn.utils.rnn.pad_sequence(test_batch, batch_first=True, padding_value=0).to(device)
        segments_tensor = torch.nn.utils.rnn.pad_sequence(testid_batch, batch_first=True, padding_value=0).to(device)
        mask_tensor = torch.nn.utils.rnn.pad_sequence(mask_batch, batch_first=True, padding_value=0).to(device)
        label_tensor = torch.tensor(label_batch, device=device)
        train_data_set.append((tokens_tensor, segments_tensor, mask_tensor, label_tensor))
        test_batch, testid_batch, mask_batch, label_batch = [], [], [], []

if len(test_batch) != 0:
    tokens_tensor = torch.nn.utils.rnn.pad_sequence(test_batch, batch_first=True, padding_value=0).to(device)
    segments_tensor = torch.nn.utils.rnn.pad_sequence(testid_batch, batch_first=True, padding_value=0).to(device)
    mask_tensor = torch.nn.utils.rnn.pad_sequence(mask_batch, batch_first=True, padding_value=0).to(device)
    label_tensor = torch.tensor(label_batch, device=device)
    train_data_set.append((tokens_tensor, segments_tensor, mask_tensor, label_tensor))
    test_batch, testid_batch, mask_batch, label_batch = [], [], [], []

learning_rate = 5e-5
warmup_proportion = 0.1 
"Proportion of training to perform linear learning rate warmup for. "
"E.g., 0.1 = 10%% of training."
num_train_epochs = 10

param_optimizer = list(model.named_parameters())
no_decay = ['bias', 'gamma', 'beta']
num_train_steps = len(train_data_set) * num_train_epochs
optimizer_grouped_parameters = [
    {'params': [p for n, p in param_optimizer if n not in no_decay], 'weight_decay_rate': 0.01},
    {'params': [p for n, p in param_optimizer if n in no_decay], 'weight_decay_rate': 0.0}
    ]
    
optimizer = BertAdam(optimizer_grouped_parameters,
                lr=learning_rate,
                warmup=warmup_proportion,
                t_total=num_train_steps)
model.train()
global_step = 0
for epoch in range(1, num_train_epochs+1):
    tr_loss = 0
    random.shuffle(train_data_set)
    for step, batch in enumerate(tqdm(train_data_set)):
        tokens_tensor, segments_tensor, mask_tensor, label_tensor = batch
        # Predict all tokens
        loss, logits = model(tokens_tensor, segments_tensor, mask_tensor, label_tensor)
        loss.backward()
        tr_loss += loss.item()
        optimizer.step()
        model.zero_grad()
        global_step += 1
pytorch_dump_path = "model/annotation_finetuned.pt"
# Save pytorch-model
print("Save PyTorch model to {}".format(pytorch_dump_path))
torch.save(model.state_dict(), pytorch_dump_path)<jupyter_output>Save PyTorch model to model/annotation_finetuned.pt
{'fraction_non_alphanumeric': 0.11593857874062612, 'fraction_numerical': 0.008094274491132009, 'mean_word_length': 4.049278846153846, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '2', 'dir': 'jupyter-structured-clean-dedup', 'id': '256451', 'license': 'MIT', 'n_tokens_mistral': 2805, 'n_tokens_neox': 2777, 'n_words': 512, 'path': 'fine_tune_chinese.ipynb', 'repo_name': 'HTAustin/BERT-Fine_tune', 'size': '8401'}