<jupyter_start><jupyter_text>Analyze training result<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns; sns.set()

from collections import defaultdict
from collections import Counter

from sklearn.feature_extraction.text import CountVectorizer
from sklearn.model_selection import StratifiedKFold
from sklearn.metrics import accuracy_score, f1_score

plt.style.use('ggplot')

import datetime as dt
import re
import pickle
from tqdm import tqdm_notebook as tqdm
import os
import sys
import time
import logging
import random

import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.data import (DataLoader, RandomSampler, SequentialSampler, TensorDataset)

from transformers import *<jupyter_output><empty_output><jupyter_text>Load saved state and predict new text<jupyter_code>model = BertForSequenceClassification.from_pretrained(
    "bert-base-uncased",
    num_labels = 3,
    output_attentions = False,
    output_hidden_states = False
)
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

model.load_state_dict(torch.load('model_statement_fold_1.bin', map_location=torch.device('cpu')))
model.eval()
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

def bert_encoder(text, max_len=512):
    """ Return embedded text vector as a list in max_len with a mask list"""
    text_token = tokenizer.tokenize(text)
    text_token = text_token[:max_len-2]
    text_token = ["[CLS]"] + text_token + ["[SEP]"]
    text_ids = tokenizer.convert_tokens_to_ids(text_token)
    text_ids += [0] * (max_len - len(text_token))
    pad_masks = [1] * len(text_token) + [0] * (max_len - len(text_token))
    segment_ids = [0] * len(text_token) + [0] * (max_len - len(text_token))
    
    return text_ids, pad_masks, segment_ids
file = open("../data/FOMC/train_all_df.pickle", "rb")
df = pickle.load(file)
file.close()
test_text = df.loc[df['type']=='speech']['text'].iloc[-1]
tokenizer.tokenize(test_text)
input_ids, masks, segment_ids = bert_encoder(test_text)
input_ids = torch.tensor(input_ids).unsqueeze(0)
output = model(input_ids)[0].detach()
F.softmax(output, dim=1).numpy()[0]<jupyter_output><empty_output><jupyter_text>DataFrame with Prediction<jupyter_code>file = open("trained_df.pickle", 'rb')
trained_df = pickle.load(file)

trained_df.tail(10)
sum(trained_df['target']==trained_df['pred_target'])
sum(trained_df['target']==trained_df['pred_target']) / len(trained_df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11954384585135666, 'fraction_numerical': 0.007471490365709791, 'mean_word_length': 6.482352941176471, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '161158', 'license': 'MIT', 'n_tokens_mistral': 894, 'n_tokens_neox': 885, 'n_words': 184, 'path': 'src/FOMC_Post_Training_BERT.ipynb', 'repo_name': 'yuki678/centralbank_analysis', 'size': '2543'}