<jupyter_start><jupyter_text>目前進度：我在閱讀 SimpleTransformer 的仔入資料的程式碼並嘗試複製因為我要把它改為多標籤的資料但我看不太懂，還在努力中多花一點時間相信我可以掌握，加油0623 19:00 進度我把關鍵的部分抓出來了，load_and_cache_examples 就是把檔名轉為 Tensor 的部分！明天繼續<jupyter_code>from __future__ import absolute_import, division, print_function

import logging
import math
import os
import random
import tempfile
import warnings
from dataclasses import asdict
from pathlib import Path

import numpy as np
import pandas as pd
import torch
from seqeval.metrics import (
    classification_report,
    f1_score,
    precision_score,
    recall_score,
)
from simpletransformers.config.model_args import NERArgs
from simpletransformers.config.utils import sweep_config_to_sweep_values
from simpletransformers.ner.ner_utils import (
    InputExample,
    LazyNERDataset,
    convert_examples_to_features,
    get_examples_from_df,
    load_hf_dataset,
    read_examples_from_file,
)
from tensorboardX import SummaryWriter
from torch.nn import CrossEntropyLoss
from torch.utils.data import DataLoader, RandomSampler, SequentialSampler, TensorDataset
from tqdm.auto import tqdm, trange
from transformers import (
    AlbertConfig,
    AlbertForTokenClassification,
    AlbertTokenizer,
    AutoConfig,
    AutoModelForTokenClassification,
    AutoTokenizer,
    BertConfig,
    BertForTokenClassification,
    BertTokenizer,
    BertweetTokenizer,
    BigBirdConfig,
    BigBirdForTokenClassification,
    BigBirdTokenizer,
    CamembertConfig,
    CamembertForTokenClassification,
    CamembertTokenizer,
    DebertaConfig,
    DebertaForTokenClassification,
    DebertaTokenizer,
    DebertaV2Config,
    DebertaV2ForTokenClassification,
    DebertaV2Tokenizer,
    DistilBertConfig,
    DistilBertForTokenClassification,
    DistilBertTokenizer,
    ElectraConfig,
    ElectraForTokenClassification,
    ElectraTokenizer,
    LayoutLMConfig,
    LayoutLMForTokenClassification,
    LayoutLMTokenizer,
    LongformerConfig,
    LongformerForTokenClassification,
    LongformerTokenizer,
    MPNetConfig,
    MPNetForTokenClassification,
    MPNetTokenizer,
    MobileBertConfig,
    MobileBertForTokenClassification,
    MobileBertTokenizer,
    RobertaConfig,
    RobertaForTokenClassification,
    RobertaTokenizerFast,
    SqueezeBertConfig,
    SqueezeBertForTokenClassification,
    SqueezeBertTokenizer,
    XLMConfig,
    XLMForTokenClassification,
    XLMTokenizer,
    XLMRobertaConfig,
    XLMRobertaForTokenClassification,
    XLMRobertaTokenizer,
    XLNetConfig,
    XLNetForTokenClassification,
    XLNetTokenizerFast,
)
from transformers.convert_graph_to_onnx import convert, quantize
from transformers.optimization import AdamW, Adafactor
from transformers.optimization import (
    get_constant_schedule,
    get_constant_schedule_with_warmup,
    get_linear_schedule_with_warmup,
    get_cosine_schedule_with_warmup,
    get_cosine_with_hard_restarts_schedule_with_warmup,
    get_polynomial_decay_schedule_with_warmup,
)


from simpletransformers.ner import NERModel
# Create a NERModel
#model = NERModel('bert', 'bert-base-cased')
model = NERModel('bert', 'dslim/bert-base-NER', args={
    'learning_rate': 2e-5,
    'overwrite_output_dir': True,
    'reprocess_input_data': True,
    'num_train_epochs': 1,
    "train_batch_size": 15})
file = "./train.txt"
out = []
with open(file) as f:
    lines = f.readlines()
    
    
for i in range(len(lines)):
    out.append(" ".join(lines[i].replace("\n", "").split(" ")[:-1] + ["O\n"]))
out[:10]
with open("./test_test.txt", "w") as f:
    f.writelines(out)
lines[:10]
lines[4].replace("\n", "").split(" ")[:-1]
" ".join(lines[4].replace("\n", "").split(" ")[:-1] + ["O\n"])
out = lines
lines[-2:]
tokenizer = model.tokenizer
tokenizer.encode("Hello")
tokenizer.encode("Hello World Eason")
tokenizer.encode("World")
tokenizer.encode("Eason")
logger = logging.getLogger(__name__)
MODELS_WITH_EXTRA_SEP_TOKEN = [
    "roberta",
    "camembert",
    "xlmroberta",
    "longformer",
    "mpnet",
]
data = "./train.txt"
evaluate=False
no_cache=False
to_predict=None

process_count = model.args.process_count

tokenizer = model.tokenizer
args = model.args

if not no_cache:
    no_cache = args.no_cache

mode = "dev" if evaluate else "train"


if to_predict:
        examples = to_predict
        no_cache = True
    else:
        if isinstance(data, str):
            examples = read_examples_from_file(
                data,
                mode,
                bbox=True if model.args.model_type == "layoutlm" else False,
            )
        else:
            if model.args.lazy_loading:
                raise ValueError(
                    "Input must be given as a path to a file when using lazy loading"
                )
            examples = get_examples_from_df(
                data,
                bbox=True if model.args.model_type == "layoutlm" else False,
            )

    cached_features_file = os.path.join(
        args.cache_dir,
        "cached_{}_{}_{}_{}_{}".format(
            mode,
            args.model_type,
            args.max_seq_length,
            model.num_labels,
            len(examples),
        ),
    )
    if not no_cache:
        os.makedirs(model.args.cache_dir, exist_ok=True)

    if os.path.exists(cached_features_file) and (
        (not args.reprocess_input_data and not no_cache)
        or (
            mode == "dev" and args.use_cached_eval_features and not no_cache
        )
    ):
        features = torch.load(cached_features_file)
        logger.info(
            f" Features loaded from cache at {cached_features_file}"
        )
    else:
        logger.info(" Converting to features started.")
        features = convert_examples_to_features(
            examples,
            model.args.labels_list,
            model.args.max_seq_length,
            model.tokenizer,
            # XLNet has a CLS token at the end
            cls_token_at_end=bool(args.model_type in ["xlnet"]),
            cls_token=tokenizer.cls_token,
            cls_token_segment_id=2 if args.model_type in ["xlnet"] else 0,
            sep_token=tokenizer.sep_token,
            # RoBERTa uses an extra separator b/w pairs of sentences,
            # cf. github.com/pytorch/fairseq/commit/1684e166e3da03f5b600dbb7855cb98ddfcd0805
            sep_token_extra=args.model_type in MODELS_WITH_EXTRA_SEP_TOKEN,
            # PAD on the left for XLNet
            pad_on_left=bool(args.model_type in ["xlnet"]),
            pad_token=tokenizer.pad_token_id,
            pad_token_segment_id=4 if args.model_type in ["xlnet"] else 0,
            pad_token_label_id=model.pad_token_label_id,
            process_count=process_count,
            silent=args.silent,
            use_multiprocessing=args.use_multiprocessing,
            chunksize=args.multiprocessing_chunksize,
            mode=mode,
            use_multiprocessing_for_evaluation=args.use_multiprocessing_for_evaluation,
        )

        if not no_cache:
            torch.save(features, cached_features_file)

    all_input_ids = torch.tensor(
        [f.input_ids for f in features], dtype=torch.long
    )
    all_input_mask = torch.tensor(
        [f.input_mask for f in features], dtype=torch.long
    )
    all_segment_ids = torch.tensor(
        [f.segment_ids for f in features], dtype=torch.long
    )
    all_label_ids = torch.tensor(
        [f.label_ids for f in features], dtype=torch.long
    )

    if model.args.model_type == "layoutlm":
        all_bboxes = torch.tensor(
            [f.bboxes for f in features], dtype=torch.long
        )

    if model.args.onnx:
        out_return = all_label_ids

    if model.args.model_type == "layoutlm":
        dataset = TensorDataset(
            all_input_ids,
            all_input_mask,
            all_segment_ids,
            all_label_ids,
            all_bboxes,
        )
    else:
        dataset = TensorDataset(
            all_input_ids, all_input_mask, all_segment_ids, all_label_ids
        )

out_return =  dataset
out_return<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.08206345241069198, 'fraction_numerical': 0.006869847614289283, 'mean_word_length': 2.959940652818991, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '399793', 'license': 'Apache-2.0', 'n_tokens_mistral': 2683, 'n_tokens_neox': 2636, 'n_words': 492, 'path': 'Practice/PyTorch/NER_test/0623_test_dataloader-Copy1.ipynb', 'repo_name': 'accordproject/labs-cicero-classify', 'size': '8006'}