<jupyter_start><jupyter_text>Test with a small data<jupyter_code>resume = 'data/checkpoints/0906_214036/model_best.pth'
resume = 'data/checkpoints/0906_214036/checkpoint.pth'
resume = 'data/checkpoints/test_checkpoint/checkpoint.pth'
import json
import torch
import argparse
from tqdm import tqdm
from tabulate import tabulate

from utils.unique import unique
import model.loss as module_loss
import model.model as module_arch
import model.metric as module_metric
from parse_config import ConfigParser
import data_loader.data_loaders as module_data

PAD = '<pad>'
# Load:: config, sample data, and checkpoint
args = argparse.ArgumentParser(description='PyTorch Template')
args.add_argument('-c', '--config', default=None, type=str, help='config file path (default: None)')
args.add_argument('-r', '--resume', default=f"{resume}", type=str, help='path to latest checkpoint (default: None)')
args.add_argument('-d', '--device', default=None, type=str, help='indices of GPUs to enable (default: all)')
args.add_argument('-f', '--file', default=None, type=str, help='Error')

config = ConfigParser.from_args(args)
logger = config.get_logger('test')

data_loader = config.init_obj('data_loader', module_data)
test_data_loader = data_loader.get_test()

# build model architecturea
model = config.init_obj('arch', module_arch)
# logger.info(model)

# get function handles of loss and metrics
criterion = getattr(module_loss, config['loss'])
metric_fns = [getattr(module_metric, met) for met in config['metrics']]

logger.info('Loading checkpoint: {} ...'.format(config.resume))
checkpoint = torch.load(config.resume)
state_dict = checkpoint['state_dict']
if config['n_gpu'] > 1:
    model = torch.nn.DataParallel(model)
model.load_state_dict(state_dict)
layers_train = config._config['trainer']['layers_train']

# prepare model for testing
device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
model = model.to(device)
model.eval()

total_loss = 0.0
total_metrics = torch.zeros(len(metric_fns))<jupyter_output>Loading checkpoint: data/checkpoints/test_checkpoint/checkpoint.pth ...<jupyter_text>Making a prediction<jupyter_code>from utils.prediction import predict, get_dict_prediction, show

text = """
เมื่อวันที่29ก.ค.55 สำนักข่าวบีบีซีรายงานว่า มหาวิทยาลัยออกซ์ฟอร์ดของอังกฤษได้แก้ไขกฎการแต่งกายภายในมหาวิทยาลัย หลังชมรมเพื่อความหลากหลายทางเพศยื่นคำร้องว่ากฎที่มีอยู่เดิมไม่เป็นธรรมกับกลุ่มนศข้ามเพศtransgenderใน
มหาวิทยาลัย
"""

tokens, out = predict(model, text, data_loader, config)
tokens = [tk for tk in tokens if tk!=PAD]

print("|".join(tokens), "\n")
[show(x) for x in out];<jupyter_output><s>|เมื่อ|วันที่||29||ก|.|ค|.||55||สํานักข่าว||บีบีซี||รายงาน||ว่า||มหาวิทยาลั|ย||ออก||ซ์||ฟอร์ด||ของ||อังกฤษ||ได้||แก้ไข||กฎ|การ||แต่งกาย||ภายใน||มหาวิทยาลั|ย|หลัง||ชมรม|เพื่อ|ความ|หลากหลาย||ทางเพศ||ย|ื่นคําร้อง||ว่า||กฎ||ที่|มี||อยู่||เดิม|ไม่||เป็นธรรม||กับ|กลุ่ม||นศ||ข้าม||เพศ||trans|g|ender||ใน||มหาวิทยาลั|ย|</s> 

[2, 5]         day            วันที่29
[2, 12]        date           วันที่29ก.ค.55
[3, 5]         cardinal       29
[5, 10]        month          ก.ค.
[10, 12]       year           55
[12, 16]       media          สํานักข่าวบีบีซี
[14, 16]       media          บีบีซี
[20, 29]       org_edu        มหาวิทยาลัยออกซ์ฟอร์ด
[23, 29]       org_edu        ออกซ์ฟอร์ด
[31, 33]       country        อังกฤษ
[81, 85]       norp_others    transgender<jupyter_text>Test with a small data<jupyter_code>from tqdm import tqdm
results = []
with torch.no_grad():
    for batch_idx, instance in tqdm(enumerate(test_data_loader)):
        input_ids = torch.tensor(instance['input_ids']).to(device)
        attention_mask = torch.tensor(instance['attention_mask']).to(device)
        batch_size = input_ids.shape[0]
        output = model(input_ids, attention_mask)
        
        loss = 0
        nested_lm_conll_ids = {l:None for l in range(len(layers_train))}
        for index, layer in enumerate(layers_train):
            temp_nested_lm_conll_ids = torch.tensor(instance['nested_lm_conll_ids'][layer])
            temp_nested_lm_conll_ids = temp_nested_lm_conll_ids.to(device)
            nested_lm_conll_ids[index]=temp_nested_lm_conll_ids
            loss+=criterion(output[index], temp_nested_lm_conll_ids)
            
        total_loss += loss.item() * batch_size
        predictions = {x:[] for x in range(batch_size)}
        lm_entities = {x:[] for x in range(batch_size)}
        for sent_ids in range(batch_size):
            for layer in range(len(output)):
                predictions[sent_ids].append(output[layer][sent_ids].argmax(axis=0))
                lm_entities[sent_ids].append(nested_lm_conll_ids[layer][sent_ids])
                
        for sent_ids in range(batch_size):
            tokens = instance['lm_tokens'][sent_ids]
            tokens = [w for w in tokens if w!=PAD]
            preds = []
            for index in range(len(layers_train)):
                preds+=get_dict_prediction(
                        tokens, 
                        predictions[sent_ids][index], 
                        attention_mask[sent_ids], 
                        data_loader.ids2tag)
            entities_labels = []
            
            for index in range(len(layers_train)):
                entities_labels+=get_dict_prediction(
                    tokens, 
                    lm_entities[sent_ids][index], 
                    attention_mask[sent_ids], 
                    data_loader.ids2tag)
            results.append({
                'sentence_id': instance['sentence_id'][sent_ids],
                'tokens': tokens,
                'entities': entities_labels,
                'predictions':preds})
            for i, metric in enumerate(metric_fns):
                total_metrics[i] += metric( 
                        output, nested_lm_conll_ids, attention_mask, 
                        data_loader.boundary_type, info=False, ids2tag=data_loader.ids2tag
            ) * batch_size<jupyter_output>0it [00:00, ?it/s]/home/weerayut/anaconda3/envs/workspace/lib/python3.6/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Precision and F-score are ill-defined and being set to 0.0 in labels with no predicted samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
/home/weerayut/anaconda3/envs/workspace/lib/python3.6/site-packages/seqeval/metrics/v1.py:57: UndefinedMetricWarning: Recall and F-score are ill-defined and being set to 0.0 in labels with no true samples. Use `zero_division` parameter to control this behavior.
  _warn_prf(average, modifier, msg_start, len(result))
5it [00:34,  6.92s/it]<jupyter_text>Results<jupyter_code>for ids, item in enumerate(results[:2]):
    print('\n',"|".join(item["tokens"]),'\n')
    sid = item['sentence_id']
    entities=item['entities']
    predictions=item['predictions']
    
    print("Predictions")
    for pred in predictions:
        tag = "/" if pred in entities else "X"
        out = [f"SID:{sid}, Index{pred['span']}", tag, ''.join(pred['text']), pred['entity_type']]
        print(out)
    
    print("Not answer")
    for en in entities:
        if en not in predictions:
            out = [f"SID:{sid}, Index{en['span']}", "X", "".join(en['text']), en['entity_type']]
            print(out)
## Can input both BIESO and BIO
from model.eval import ClassEvaluator
results_eval, conll_results = ClassEvaluator()(results)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1379819965067849, 'fraction_numerical': 0.015585113529490796, 'mean_word_length': 3.462829736211031, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '515357', 'license': 'MIT', 'n_tokens_mistral': 2809, 'n_tokens_neox': 2768, 'n_words': 486, 'path': 'test.ipynb', 'repo_name': 'vistec-AI/Thai-NNER', 'size': '7443'}