<jupyter_start><jupyter_text>Model path<jupyter_code>glob('models/*')
path = 'models/20190418_211742/'<jupyter_output><empty_output><jupyter_text>Prepare<jupyter_code>data_test = load_file('data/LREC/test2011')
data_test_asr = load_file('data/LREC/test2011asr')
# data_test = load_file('data/NPR-podcasts/test')
with open(path+'hyperparameters.json', 'r') as f:
    hyperparameters = json.load(f)
hyperparameters
tokenizer = BertTokenizer.from_pretrained('bert-base-uncased', do_lower_case=True)

punctuation_enc = {
    'O': 0,
    'COMMA': 1,
    'PERIOD': 2,
    'QUESTION': 3
}

# punctuation_enc = {
#     'O': 0,
#     'PERIOD': 1
# }

segment_size = hyperparameters['segment_size']
X_test, y_test = preprocess_data(data_test, tokenizer, punctuation_enc, segment_size)
X_test_asr, y_test_asr = preprocess_data(data_test_asr, tokenizer, punctuation_enc, segment_size)
output_size = len(punctuation_enc)
dropout = hyperparameters['dropout']
bert_punc = nn.DataParallel(BertPunc(segment_size, output_size, dropout).cuda())<jupyter_output><empty_output><jupyter_text>Model progress<jupyter_code>progress = pd.read_csv(path+'progress.csv', delimiter=';')
progress
progress[['training loss', 'loss']].plot();
progress[['accuracy', 'f1_space', 'f1_comma', 'f1_period', 'f1_question']].plot();
# progress[['accuracy', 'f1_O', 'f1_PERIOD']].plot();<jupyter_output><empty_output><jupyter_text>Model evaluation<jupyter_code>bert_punc.load_state_dict(torch.load(path+'model'))
bert_punc.eval();
batch_size = 1024
data_loader_test = create_data_loader(X_test, y_test, False, batch_size)
data_loader_test_asr = create_data_loader(X_test_asr, y_test_asr, False, batch_size)
def predictions(data_loader):
    y_pred = []
    y_true = []
    for inputs, labels in tqdm(data_loader, total=len(data_loader)):
        with torch.no_grad():
            inputs, labels = inputs.cuda(), labels.cuda()
            output = bert_punc(inputs)
            y_pred += list(output.argmax(dim=1).cpu().data.numpy().flatten())
            y_true += list(labels.cpu().data.numpy().flatten())
    return y_pred, y_true
def evaluation(y_pred, y_test):
    precision, recall, f1, _ = metrics.precision_recall_fscore_support(
        y_test, y_pred, average=None, labels=[1, 2, 3])
    overall = metrics.precision_recall_fscore_support(
        y_test, y_pred, average='macro', labels=[1, 2, 3])
    result = pd.DataFrame(
        np.array([precision, recall, f1]), 
        columns=list(punctuation_enc.keys())[1:], 
        index=['Precision', 'Recall', 'F1']
    )
    result['OVERALL'] = overall[:3]
    return result
# def evaluation(y_pred, y_test):
#     precision, recall, f1, _ = metrics.precision_recall_fscore_support(
#         y_test, y_pred, average=None, labels=[1])
#     overall = metrics.precision_recall_fscore_support(
#         y_test, y_pred, average='macro', labels=[1])
#     result = pd.DataFrame(
#         np.array([precision, recall, f1]), 
#         columns=list(punctuation_enc.keys())[1:], 
#         index=['Precision', 'Recall', 'F1']
#     )
#     result['OVERALL'] = overall[:3]
#     return result<jupyter_output><empty_output><jupyter_text>Test<jupyter_code>y_pred_test, y_true_test = predictions(data_loader_test)
eval_test = evaluation(y_pred_test, y_true_test)
eval_test
# https://www.isca-speech.org/archive/Interspeech_2016/pdfs/1517.PDF
ref_test = pd.DataFrame({
    'COMMA':    [0.655, 0.471, 0.548],
    'PERIOD':   [0.733, 0.725, 0.729],
    'QUESTION': [0.707, 0.630, 0.667],
    'OVERALL':  [0.700, 0.597, 0.644]
}, index=['Precision', 'Recall', 'F1'])
ref_test
for col in ref_test.columns:
    pd.DataFrame({'Reference': ref_test[col], 'BertPunc': eval_test[col]}).plot.bar(
        title=col, figsize=(12, 4))<jupyter_output><empty_output><jupyter_text>Test ASR<jupyter_code>y_pred_test_asr, y_true_test_asr = predictions(data_loader_test_asr)
eval_test_asr = evaluation(y_pred_test_asr, y_true_test_asr)
eval_test_asr
# https://www.isca-speech.org/archive/Interspeech_2016/pdfs/1517.PDF
ref_test_asr = pd.DataFrame({
    'COMMA':    [0.596, 0.429, 0.499],
    'PERIOD':   [0.707, 0.720, 0.714],
    'QUESTION': [0.607, 0.486, 0.540],
    'OVERALL':  [0.660, 0.573, 0.614]
}, index=['Precision', 'Recall', 'F1'])
ref_test_asr
for col in ref_test_asr.columns:
    pd.DataFrame({'Reference': ref_test_asr[col], 'BertPunc': eval_test_asr[col]}).plot.bar(
        title=col, figsize=(12, 4))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16073038773669973, 'fraction_numerical': 0.03990081154192967, 'mean_word_length': 5.08641975308642, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '591922', 'license': 'Apache-2.0', 'n_tokens_mistral': 1842, 'n_tokens_neox': 1712, 'n_words': 255, 'path': 'evaluate.ipynb', 'repo_name': 'mec-correcaotextual/BertPunc', 'size': '4436'}