<jupyter_start><jupyter_text>```// Copyright 2020 Twitter, Inc.// SPDX-License-Identifier: Apache-2.0``` Finetune NER modelTake an existing BERT model (with or without TPP pre-training) and fine-tune it on an NER dataset.  Setup libraries<jupyter_code>%pip install transformers==3.5.1 datasets==1.1.2 torch==1.4.0 seqeval==1.2.2 gensim==3.8.1<jupyter_output><empty_output><jupyter_text>Define parameters<jupyter_code>WIKI_TRAINING = False
SSEA_TRAINING = False

HOMEDIR = "./"
DATADIR = f"{HOMEDIR}/"
pre_trained_model_path = "bert-base-multilingual-uncased"<jupyter_output><empty_output><jupyter_text>Setup Helpers<jupyter_code>import json
import random
import re
from dataclasses import dataclass
from pathlib import Path
from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union

import numpy as np
import pandas as pd
import torch
from IPython.core.debugger import set_trace
from torch.nn.utils.rnn import pad_sequence
from tqdm import tqdm, trange
from transformers import (
    BertForTokenClassification,
    BertTokenizerFast,
    Pipeline,
    RobertaTokenizerFast,
    TokenClassificationPipeline,
    Trainer,
    TrainingArguments,
)
from transformers.tokenization_utils_base import (
    BatchEncoding,
    PaddingStrategy,
    PreTrainedTokenizerBase,
)

from seqeval.metrics import (
    accuracy_score,
    classification_report,
    f1_score,
    precision_score,
    recall_score,
)
URL_REGEX = re.compile(r"^http[s]?://[^ ]+")


def clean_tokens(token):
    # Sometimes a token starts or is equal to an empty string which matches the unicode char of id 65039
    if ord(token[0]) == 65039:
        token = token[1:]
    if token == chr(65039):
        return ""
    if token == "\n":
        return "[LF]"
    return URL_REGEX.sub("[URL]", token)


def read_ner_data(file_path):
    with open(file_path) as fp:
        all_tokens = []
        all_labels = []
        for line in tqdm(fp):
            line = json.loads(line)
            tokens = line["tokens"]
            labels = line["labels"]
            tokens = [clean_tokens(token) for token in tokens]
            # Remove empty tokens
            tokens, labels = tuple(
                zip(*[(t, l) for t, l in zip(tokens, labels) if t and t.strip()])
            )
            all_tokens.append(tokens)
            all_labels.append(labels)
    return all_tokens, all_labels


label2id = {
    "O_NOT_AN_ENTITY": 1,
    "B_PERSON": 2,
    "I_PERSON": 3,
    "B_PLACE": 4,
    "I_PLACE": 5,
    "B_PRODUCT": 6,
    "I_PRODUCT": 7,
    "B_ORGANIZATION": 8,
    "I_ORGANIZATION": 9,
    "B_OTHER": 10,
    "I_OTHER": 11,
    "O_PLACEHOLDER": 12,
}

if WIKI_TRAINING:
    label2id = {
        "O_NOT_AN_ENTITY": 1,
        "B_PERSON": 2,
        "I_PERSON": 3,
        "B_LOCATION": 4,
        "I_LOCATION": 5,
        "B_ORGANIZATION": 6,
        "I_ORGANIZATION": 7,
        "B_COMPANY": 8,
        "I_COMPANY": 9,
        "B_BAND": 10,
        "I_BAND": 11,
        "B_MOVIE": 12,
        "I_MOVIE": 13,
        "B_SPORTS_TEAM": 14,
        "I_SPORTS_TEAM": 15,
        "B_TV_SERIES": 16,
        "I_TV_SERIES": 17,
        "B_TV_SHOW": 18,
        "I_TV_SHOW": 19,
        "O_PLACEHOLDER": 20,
    }

LABEL_MAP = {
    k: k.replace("_", "-") if k != "O_NOT_AN_ENTITY" else "O" for k in label2id
}
label2id = {LABEL_MAP[k]: v - 1 for k, v in label2id.items()}

id2label = {v: k for k, v in label2id.items()}
class SplitTokenClassificationPipeline(TokenClassificationPipeline):
    def __init__(self, **kwargs):
        super().__init__(**kwargs)

    def __call__(self, inputs: Union[str, List[str]], **kwargs):
        """
    Classify each token of the text(s) given as inputs.
    Args:
        inputs (:obj:`str` or :obj:`List[str]`):
            One or several texts (or one list of texts) for token classification.
    Return:
        A list or a list of list of :obj:`dict`: Each result comes as a list of dictionaries (one for each token in
        the corresponding input, or each entity if this pipeline was instantiated with
        :obj:`grouped_entities=True`) with the following keys:
        - **word** (:obj:`str`) -- The token/word classified.
        - **score** (:obj:`float`) -- The corresponding probability for :obj:`entity`.
        - **entity** (:obj:`str`) -- The entity predicted for that token/word.
        - **index** (:obj:`int`, only present when ``self.grouped_entities=False``) -- The index of the
          corresponding token in the sentence.
    """
        inputs, offset_mappings = self._args_parser(
            inputs, **kwargs
        )

        answers = []

        for i, sentence in enumerate(
            inputs[0]
        ):  # Another addition to only select first list item in newer version
            # Manage correct placement of the tensors
            with self.device_placement():

                tokens = self.tokenizer(
                    sentence,
                    return_attention_mask=False,
                    return_tensors=self.framework,
                    truncation=True,
                    return_special_tokens_mask=True,
                    return_offsets_mapping=self.tokenizer.is_fast,
                    is_split_into_words=True,  # This is the new addition,
                    padding=True,
                    max_length=self.tokenizer.max_len,
                )
                if self.tokenizer.is_fast:
                    offset_mapping = tokens.pop("offset_mapping").cpu().numpy()[0]
                elif offset_mappings:
                    offset_mapping = offset_mappings[i]
                else:
                    offset_mapping = None

                special_tokens_mask = tokens.pop("special_tokens_mask").cpu().numpy()[0]

                # Forward
                if self.framework == "tf":
                    entities = self.model(tokens.data)[0][0].numpy()
                    input_ids = tokens["input_ids"].numpy()[0]
                else:
                    with torch.no_grad():
                        tokens = self.ensure_tensor_on_device(**tokens)
                        entities = self.model(**tokens)[0][0].cpu().numpy()
                        input_ids = tokens["input_ids"].cpu().numpy()[0]

            score = np.exp(entities) / np.exp(entities).sum(-1, keepdims=True)
            labels_idx = score.argmax(axis=-1)

            entities = []
            # Filter to labels not in `self.ignore_labels`
            # Filter special_tokens
            filtered_labels_idx = [
                (idx, label_idx)
                for idx, label_idx in enumerate(labels_idx)
                if (self.model.config.id2label[label_idx] not in self.ignore_labels)
                and not special_tokens_mask[idx]
            ]

            for idx, label_idx in filtered_labels_idx:
                if offset_mapping is not None:
                    start_ind, end_ind = offset_mapping[idx]
                    word_ref = sentence[start_ind:end_ind]
                    word = self.tokenizer.convert_ids_to_tokens([int(input_ids[idx])])[
                        0
                    ]
                    is_subword = len(word_ref) != len(word)

                    if int(input_ids[idx]) == self.tokenizer.unk_token_id:
                        word = word_ref
                        is_subword = False
                else:
                    word = self.tokenizer.convert_ids_to_tokens(int(input_ids[idx]))

                entity = {
                    "word": word,
                    "score": score[idx][label_idx].item(),
                    "entity": self.model.config.id2label[label_idx],
                    "index": idx,
                    "offset": (start_ind, end_ind),
                }

                entity["is_subword"] = is_subword  # Another addition

                if self.grouped_entities and self.ignore_subwords:
                    entity["is_subword"] = is_subword

                entities += [entity]

            if self.grouped_entities:
                answers += [self.group_entities(entities)]
            # Append ungrouped entities
            else:
                answers += [entities]

        if len(answers) == 1:
            return answers[0]
        return answers


# Metrics
def compute_metrics(all_preds, all_labels, return_report=False, extra_label_map=None):
    if extra_label_map is None:
        extra_label_map = {}

    def clean_pred(preds):
        return [p["entity"] for p in preds if p["offset"][0] == 0]

    def clean_label(labels):
        return [LABEL_MAP[extra_label_map.get(l, l)] for l in labels]

    true_predictions = [clean_pred(preds) for preds in all_preds]
    true_labels = [clean_label(labels) for labels in all_labels]
    tp = []
    tl = []
    num_errors = 0
    for i, (p, l) in enumerate(zip(true_predictions, true_labels)):
        if len(p) != len(l):
            num_errors += 1
            continue
        tp.append(p)
        tl.append(l)

    print(f"Found {num_errors} errors in length mismatch.")
    true_predictions = tp
    true_labels = tl

    metrics = {
        "accuracy_score": accuracy_score(true_labels, true_predictions),
        "precision": precision_score(true_labels, true_predictions),
        "recall": recall_score(true_labels, true_predictions),
        "f1": f1_score(true_labels, true_predictions),
    }

    if return_report:
        report = classification_report(true_labels, true_predictions, output_dict=True)
        return report

    return metrics


def get_preds(all_tokens, ner_pipeline):
    all_preds = []
    batch_size = 64
    for i in trange(0, len(all_tokens) + batch_size, batch_size):
        batch = all_tokens[i : i + batch_size]
        if batch:
            preds = ner_pipeline(batch)
            all_preds.extend(preds)
    return all_preds


def run_eval(data_path, ner_pipeline, extra_label_map=None):
    all_tokens, all_labels = read_ner_data(data_path)
    all_preds = get_preds(all_tokens, ner_pipeline)
    report = compute_metrics(
        all_preds, all_labels, return_report=True, extra_label_map=extra_label_map
    )
    df_report = pd.DataFrame.from_dict(report, orient="index")
    return df_report
@dataclass
class DataCollatorForTokenClassification:
    """
    NOTE: Code taken from huggingface transformers library
    Data collator that will dynamically pad the inputs received, as well as the labels.
    Args:
        tokenizer (:class:`~transformers.PreTrainedTokenizer` or :class:`~transformers.PreTrainedTokenizerFast`):
            The tokenizer used for encoding the data.
        padding (:obj:`bool`, :obj:`str` or :class:`~transformers.tokenization_utils_base.PaddingStrategy`, `optional`, defaults to :obj:`True`):
            Select a strategy to pad the returned sequences (according to the model's padding side and padding index)
            among:
            * :obj:`True` or :obj:`'longest'`: Pad to the longest sequence in the batch (or no padding if only a single
              sequence if provided).
            * :obj:`'max_length'`: Pad to a maximum length specified with the argument :obj:`max_length` or to the
              maximum acceptable input length for the model if that argument is not provided.
            * :obj:`False` or :obj:`'do_not_pad'` (default): No padding (i.e., can output a batch with sequences of
              different lengths).
        max_length (:obj:`int`, `optional`):
            Maximum length of the returned list and optionally padding length (see above).
        pad_to_multiple_of (:obj:`int`, `optional`):
            If set will pad the sequence to a multiple of the provided value.
            This is especially useful to enable the use of Tensor Cores on NVIDIA hardware with compute capability >=
            7.5 (Volta).
        label_pad_token_id (:obj:`int`, `optional`, defaults to -100):
            The id to use when padding the labels (-100 will be automatically ignore by PyTorch loss functions).
    """

    tokenizer: PreTrainedTokenizerBase
    padding: Union[bool, str, PaddingStrategy] = True
    max_length: Optional[int] = None
    pad_to_multiple_of: Optional[int] = None
    label_pad_token_id: int = -100

    def __call__(self, features):
        label_name = "label" if "label" in features[0].keys() else "labels"
        labels = (
            [feature[label_name] for feature in features]
            if label_name in features[0].keys()
            else None
        )
        for feature in features:
            feature.pop(label_name)
        batch = self.tokenizer.pad(
            features,
            padding=self.padding,
            max_length=self.max_length,
            pad_to_multiple_of=self.pad_to_multiple_of,
            # Conversion to tensors will fail if we have labels as they are not of the same length yet.
            return_tensors="pt" if labels is None else None,
        )

        if labels is None:
            return batch

        sequence_length = torch.tensor(batch["input_ids"]).shape[1]
        padding_side = self.tokenizer.padding_side
        if padding_side == "right":
            batch["labels"] = [
                label.tolist()
                + [self.label_pad_token_id] * (sequence_length - len(label))
                for label in labels
            ]
        else:
            batch["labels"] = [
                [self.label_pad_token_id] * (sequence_length - len(label))
                + label.tolist()
                for label in labels
            ]

        batch = {
            k: torch.tensor(v, dtype=torch.int64)[:, : self.max_length]
            for k, v in batch.items()
        }
        return batch
def encode_labels(labels, encodings):
    labels = [label2id[LABEL_MAP[label]] for label in labels]
    offset = encodings["offset_mapping"]
    # create an empty array of -100
    n = len(offset)
    doc_enc_labels = np.ones(n, dtype=int) * -100
    arr_offset = np.array(offset)
    positions = np.arange(n)
    mask = (arr_offset[:, 0] == 0) & (
        (positions != 0) & (positions != n - 1)
    )
    # set labels whose first offset position is 0 and the second is not 0
    doc_enc_labels[mask] = labels
    return doc_enc_labels.tolist()


class NERDataset(torch.utils.data.Dataset):
    def __init__(self, file_paths, tokenizer):
        self.file_paths = file_paths if isinstance(file_paths, list) else [file_paths]
        self.tokenizer = tokenizer
        self._setup()

    def _setup(self):
        all_tokens, all_labels = [], []
        for file_path in self.file_paths:
            all_tokens_fp, all_labels_fp = read_ner_data(file_path)
            all_tokens += all_tokens_fp
            all_labels += all_labels_fp
        self.data = []
        num_errors = 0
        all_encodings = encodings = self.tokenizer(
            all_tokens, is_split_into_words=True, return_offsets_mapping=True
        )

        for i, (tokens, labels) in tqdm(enumerate(zip(all_tokens, all_labels))):
            if len(tokens) == 0 or len(labels) == 0:
                num_errors += 1
                continue
            try:
                encodings = {k: all_encodings[k][i] for k in all_encodings}
                labels = encode_labels(labels, encodings)
                encodings.pop("offset_mapping")  # Don't pass to model
                self.data.append((encodings, labels))
            except ValueError as e:
                num_errors += 1
                continue
        print(
            f"Errors: {num_errors}, data={len(self.data)}, %error={num_errors*100./len(self.data)}"
        )

    def __getitem__(self, idx):
        encodings, labels = self.data[idx]
        item = {key: torch.tensor(val) for key, val in encodings.items()}
        item["labels"] = torch.tensor(labels)
        return item

    def __len__(self):
        return len(self.data)<jupyter_output><empty_output><jupyter_text>Run Training<jupyter_code># model_dir = Path(f"{HOMEDIR}/en_ner_model/").expanduser()
model_dir = pre_trained_model_path

tokenizer = BertTokenizerFast.from_pretrained(
    str(model_dir), max_len=512, truncation=True, padding=True
)
data_dir = Path(f"{DATADIR}/ner_data/").expanduser()
if SSEA_TRAINING:
    data_dir = Path(f"{DATADIR}/SSEA/processed/").expanduser()

langs = "ja"  # "en" # "**"
train_data_path = list(data_dir.glob(f"./{langs}/ner/*-training.json"))
val_data_path = list(data_dir.glob(f"./{langs}/ner/*-validation.json"))
test_data_path = list(data_dir.glob(f"./{langs}/ner/*-testing.json"))

if SSEA_TRAINING:
    train_data_path = list(data_dir.glob(f"./{langs}/aligned/*-training.json"))
    test_data_path = list(data_dir.glob(f"./{langs}/aligned/*-testing.json"))
train_data_path
train_dataset = NERDataset(train_data_path, tokenizer)
val_dataset = []
if not SSEA_TRAINING:
    val_dataset = NERDataset(val_data_path, tokenizer)
len(train_dataset), len(val_dataset)
model_prefix = "multi" if langs == "**" else langs
ner_model_dir = str(Path(f"{HOMEDIR}/{model_prefix}_ner_model").expanduser())
logging_dir = str(Path(f"{HOMEDIR}/{model_prefix}_ner_logs").expanduser())

data_collator = DataCollatorForTokenClassification(
    tokenizer=tokenizer, padding=True, max_length=tokenizer.max_len
)

eval_every_steps = 1  # -1 for eval at end

training_args = TrainingArguments(
    output_dir=str(ner_model_dir),  # output directory
    num_train_epochs=3,  # total number of training epochs
    per_device_train_batch_size=16,  # batch size per device during training
    per_device_eval_batch_size=64,  # batch size for evaluation
    warmup_steps=500,  # number of warmup steps for learning rate scheduler
    weight_decay=0.01,  # strength of weight decay
    logging_dir=str(logging_dir),  # directory for storing logs
    logging_steps=10,
    # evaluation_strategy="steps",
    # eval_steps=100,
    save_steps=500 if eval_every_steps < 1 else eval_every_steps,
    save_total_limit=2 if eval_every_steps < 1 else None,
    max_steps=-1 if eval_every_steps < 1 else 5,
    label_names=[id2label[i] for i in range(len(id2label))],
)


def train_model():
    model = BertForTokenClassification.from_pretrained(
        str(model_dir), num_labels=len(label2id), id2label=id2label, label2id=label2id
    )
    trainer = Trainer(
        model=model,  # the instantiated 🤗 Transformers model to be trained
        args=training_args,  # training arguments, defined above
        train_dataset=train_dataset,  # training dataset
        eval_dataset=val_dataset,  # evaluation dataset
        data_collator=data_collator,
        tokenizer=tokenizer,
    )
    trainer.train()
    trainer.save_model(ner_model_dir)
    tokenizer.save_pretrained(ner_model_dir)
    return model, trainer
%%time
model, trainer = train_model()<jupyter_output><empty_output><jupyter_text>Run evaluation<jupyter_code>from typing import Any, Callable, Dict, List, NewType, Optional, Tuple, Union

import numpy as np
import pandas as pd
import torch
from tqdm import trange
from transformers import (
    AutoModelForTokenClassification,
    AutoTokenizer,
    Pipeline,
    TokenClassificationPipeline,
    pipeline,
)

from seqeval.metrics import accuracy_score, f1_score, precision_score, recall_score
ner_model_dir = str(Path(f"{HOMEDIR}/{model_prefix}_ner_model/").expanduser())

model = AutoModelForTokenClassification.from_pretrained(ner_model_dir)
tokenizer = BertTokenizerFast.from_pretrained(
    str(ner_model_dir), max_len=512, truncation=True, padding=True
)
ner_pipeline = SplitTokenClassificationPipeline(
    model=model, tokenizer=tokenizer, grouped_entities=False, ignore_labels=[], device=0
)
data_dir = Path(f"{DATADIR}/ner_data/").expanduser()
langs = "**"
train_data_path = list(data_dir.glob(f"./{langs}/ner/*-training.json"))
val_data_path = list(data_dir.glob(f"./{langs}/ner/*-validation.json"))
test_data_path = list(data_dir.glob(f"./{langs}/ner/*-testing.json"))
extra_label_map = None
if WIKI_TRAINING:
    extra_label_map = {
        "B_PLACE": "B_LOCATION",
        "I_PLACE": "I_LOCATION",
        "B_PRODUCT": "O_NOT_AN_ENTITY",
        "I_PRODUCT": "O_NOT_AN_ENTITY",
        "B_OTHER": "O_NOT_AN_ENTITY",
        "I_OTHER": "O_NOT_AN_ENTITY",
    }
%%time
reports = {}
for test_path in test_data_path:
    lang = test_path.name.split("-")[1]
    print(lang, test_path.name)
    report = run_eval(test_path, ner_pipeline, extra_label_map=extra_label_map)
    reports[lang] = report
df_report = pd.concat(reports)
df_report
df_report.to_csv(Path(ner_model_dir) / "test_eval_report.txt", sep="\t")
df_report[df_report.index.isin(["micro avg"], level=1)]
df_report = pd.read_csv(
    Path(ner_model_dir) / "test_eval_report.txt", sep="\t", index_col=[0, 1]
)
df_report[df_report.index.isin(["micro avg"], level=1)]
if Path(f"{HOMEDIR}/test_eval_report.txt").expanduser().exists():
    df_report = pd.read_csv(
        f"{HOMEDIR}/test_eval_report.txt", sep="\t", index_col=[0, 1]
    )
df_report[df_report.index.isin(["micro avg"], level=1)]<jupyter_output><empty_output><jupyter_text>Evaluation on SSEA<jupyter_code>data_dir = Path(f"{DATADIR}/SSEA/processed/").expanduser()
langs = "**"
extra_label_map = None
train_data_path = list(data_dir.glob(f"./{langs}/aligned/*-training.json"))
val_data_path = list(data_dir.glob(f"./{langs}/aligned/*-validation.json"))
test_data_path = list(data_dir.glob(f"./{langs}/aligned/*-testing.json"))
len(train_data_path)
%%time
reports = {}
for test_path in test_data_path:
    lang = test_path.parts[-3]
    print(lang, test_path.name)
    report = run_eval(test_path, ner_pipeline)
    reports[lang] = report
df_report_ssea = pd.concat(reports)
df_report_ssea
df_report_ssea.to_csv(Path(ner_model_dir) / "ssea_test_eval_report.txt", sep="\t")
df_report_ssea[df_report_ssea.index.isin(["micro avg"], level=1)]
df_report_ssea = pd.read_csv(
    Path(ner_model_dir) / "ssea_test_eval_report.txt", sep="\t", index_col=[0, 1]
)
df_report_ssea[df_report_ssea.index.isin(["micro avg"], level=1)]
if Path(f"{HOMEDIR}/ssea_test_eval_report.txt").expanduser().exists():
    df_report_ssea = pd.read_csv(
        f"{HOMEDIR}/ssea_test_eval_report.txt", sep="\t", index_col=[0, 1]
    )
df_report_ssea[df_report_ssea.index.isin(["micro avg"], level=1)]
%%time
reports = {}
for test_path in train_data_path:
    lang = test_path.parts[-3]
    print(lang, test_path.name)
    report = run_eval(test_path, ner_pipeline)
    reports[lang] = report
df_report_ssea = pd.concat(reports)
df_report_ssea
df_report_ssea.to_csv(Path(ner_model_dir) / "ssea_train_eval_report.txt", sep="\t")
df_report_ssea[df_report_ssea.index.isin(["micro avg"], level=1)]<jupyter_output><empty_output><jupyter_text>Evaluate all checkpoints<jupyter_code>from IPython.display import display
%%time
eval_lang = "ja"
for ner_model_dir in (
    Path(f"{HOMEDIR}/{model_prefix}_ner_model/").expanduser().glob("checkpoint-*")
):
    print(ner_model_dir.name)
    model = AutoModelForTokenClassification.from_pretrained(ner_model_dir)
    tokenizer = BertTokenizerFast.from_pretrained(
        str(ner_model_dir), max_len=512, truncation=True, padding=True
    )
    ner_pipeline = SplitTokenClassificationPipeline(
        model=model,
        tokenizer=tokenizer,
        grouped_entities=False,
        ignore_labels=[],
        device=0,
    )
    reports = {}
    for test_path in test_data_path:
        lang = test_path.name.split("-")[1]
        if SSEA_TRAINING:
            lang = test_path.parts[-3]
            print(lang)
        if lang != eval_lang:
            continue
        print(lang, test_path.name)
        report = run_eval(test_path, ner_pipeline, extra_label_map=extra_label_map)
        reports[lang] = report
    df_report = pd.concat(reports)
    df_report.to_csv(Path(ner_model_dir) / f"test_eval_report.txt", sep="\t")
    display(df_report[df_report.index.isin(["micro avg"], level=1)])
df_all_reports = {}
for ner_model_dir in (
    Path(f"{HOMEDIR}/{model_prefix}_ner_model/").expanduser().glob("checkpoint-*")
):
    df_report = pd.read_csv(
        Path(ner_model_dir) / "test_eval_report.txt", sep="\t", index_col=[0, 1]
    )
    df_all_reports[
        ner_model_dir.name
    ] = df_report

df_all_reports = pd.concat(df_all_reports)
df_all_reports.to_csv(
    Path(ner_model_dir) / f"../test_eval_report_{eval_lang}_all_checkpoints.txt",
    sep="\t",
)
df_all_reports[df_all_reports.index.isin(["micro avg"], level=2)]<jupyter_output><empty_output><jupyter_text>Show Results<jupyter_code>df_report[df_report.index.isin(["micro avg"], level=1)]<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11371619965689078, 'fraction_numerical': 0.009231271954905645, 'mean_word_length': 2.828459734167318, 'pattern_counts': {'":': 50, '<': 36, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '207446', 'license': 'Apache-2.0', 'n_tokens_mistral': 8120, 'n_tokens_neox': 7838, 'n_words': 1688, 'path': 'notebooks/Finetune_NER.ipynb', 'repo_name': 'twitter-research/multilingual-alignment-tpp', 'size': '24482'}