<jupyter_start><jupyter_text>Install packages This version achieves a higher F1 score than v1 by assigning tag "X" to the splitted words that are not in the vocabulary of the "bert-base-uncased" model, therefore resolving the mismatch between word and corresponding labels. However, because of the newly introduced tag "X" and the large number of science "jargons" that the tokenizer does not recognizes, there are a lot "X" tags being present in the modified labels and the model is over-predicting them. (more false positives judging from the high recall and low precision score for "X" in the classification report)<jupyter_code>!pip3 install pytorch-transformers
!pip3 install seqeval
!pip3 install spacy
!pip3 install sklearn-crfsuite
import pandas as pd
import numpy as np
import torch
from tqdm import tqdm, trange
from torch.optim import Adam
from torch.utils.data import TensorDataset, DataLoader, RandomSampler, SequentialSampler
from keras.preprocessing.sequence import pad_sequences
from sklearn.model_selection import train_test_split
from pytorch_transformers import BertTokenizer, BertConfig, BertForTokenClassification, AdamW
from sklearn_crfsuite import metrics
from sklearn_crfsuite.metrics import flat_classification_report, flat_f1_score<jupyter_output>Using TensorFlow backend.<jupyter_text>Load data<jupyter_code>url = "https://raw.githubusercontent.com/rpw199912j/MatBERT/master/mat_ner_IOB_underscore.csv"
data = pd.read_csv(url).fillna("O")
data.head(10)<jupyter_output><empty_output><jupyter_text>Pre-processing Get the sentece and labels<jupyter_code>class SentenceGetter(object):

    def __init__(self, data):
        self.n_sent = 1
        self.data = data
        self.empty = False
        self.grouped = self.data.groupby("Sentence #").apply(
            lambda s: [(w, t) for w, t in zip(s["Word"].values.tolist(),
                                              s["Tag"].values.tolist())])
        self.sentences = [s for s in self.grouped]

    def get_next(self):
        try:
            s = self.grouped["Sentence: {}".format(self.n_sent)]
            self.n_sent += 1
            return s
        except:
            return None
getter = SentenceGetter(data)
print(getter.grouped)<jupyter_output>Sentence #
1       [(Magnetic, O), (and, O), (thermoelectric, O),...
2       [(In, O), (order, O), (to, O), (prevent, O), (...
3       [(The, O), (elements, O), (,, O), (Ba, B-precu...
4       [(The, O), (synthesis, O), (of, O), (the, O), ...
5       [(BaSe, B-material), (was, O), (prepared, B-op...
6       [(The, O), (mechanical, B-operation), (alloyin...
7       [(BaxCr5Se8, B-target), (was, O), (then, O), (...
8       [(In, O), (order, O), (to, O), (obtain, O), (5...
9       [(The, O), (mixture, B-unspecified_material), ...
10      [(The, O), (powder, B-unspecified_material), (...
11      [(About, O), (5, B-number), (g, B-amount_unit)...
12      [(The, O), (temperature, O), (was, O), (raised...
13      [(The, O), (pressure, O), (was, O), (raised, B...
14      [(Insights, O), (into, O), (the, O), (stabilit...
15      [(Au, B-target), (was, O), (deposited, B-opera...
16      [(This, O), (solution, B-unspecified_material)...
17      [(After, O), (stirring, B-operation), (for, O)...
18 [...]<jupyter_text>Take a look at the first sentence in the data<jupyter_code>sentences = [" ".join([s[0].lower() for s in sent]) for sent in getter.sentences]
print(sentences[0])<jupyter_output>magnetic and thermoelectric properties of the ternary pseudo-hollandite baxcr5se8 ( 0.5 < x < 0.55 ) solid solution .<jupyter_text>Get the word-level label<jupyter_code>labels = [[s[1] for s in sent] for sent in getter.sentences]
print(labels[0])<jupyter_output>['O', 'O', 'O', 'O', 'O', 'O', 'O', 'O', 'B-target', 'O', 'B-number', 'O', 'O', 'O', 'B-number', 'O', 'O', 'O', 'O']<jupyter_text>Create a dictionary that maps each word label into a number<jupyter_code>tags_vals = list(set(data["Tag"].values))
tags_vals.append("X")
print(tags_vals)
tag2idx = {t: i for i, t in enumerate(tags_vals)}
print(tag2idx)<jupyter_output>['I-apparatus_property_type', 'B-apparatus_descriptor', 'B-amount_misc', 'B-number', 'I-meta', 'I-number', 'I-characterization_apparatus', 'B-operation', 'B-brand', 'I-condition_misc', 'B-gas', 'B-target', 'B-condition_type', 'I-material', 'I-condition_type', 'B-reference', 'B-precursor', 'B-amount_unit', 'I-amount_misc', 'I-reference', 'B-nonrecipe_material', 'B-property_type', 'B-synthesis_apparatus', 'I-precursor', 'I-target', 'B-unspecified_material', 'I-gas', 'B-condition_unit', 'I-brand', 'B-property_misc', 'I-apparatus_unit', 'I-synthesis_apparatus', 'B-apparatus_unit', 'B-material_descriptor', 'B-condition_misc', 'I-property_unit', 'B-solvent', 'I-condition_unit', 'B-material', 'I-solvent', 'I-operation', 'B-meta', 'I-property_misc', 'I-apparatus_descriptor', 'B-apparatus_property_type', 'B-property_unit', 'I-unspecified_material', 'I-amount_unit', 'I-property_type', 'B-characterization_apparatus', 'O', 'I-nonrecipe_material', 'I-material_descriptor', 'X']
{'I-apparatus_propert[...]<jupyter_text>Apply BERT model Set constants and GPU processor<jupyter_code>MAX_LEN = 64
BATCH_SIZE = 32
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
n_gpu = torch.cuda.device_count()
print("The number of GPU: {}".format(n_gpu))
torch.cuda.get_device_name(0)<jupyter_output><empty_output><jupyter_text>Get the pre-trained uncased word embeddings<jupyter_code>tokenizer = BertTokenizer.from_pretrained("bert-base-uncased", do_lower_case=True)
# tokenizer.add_tokens(["stirrer", "teflon", "autoclave", "degc"])<jupyter_output><empty_output><jupyter_text>Tokenize all the sentences<jupyter_code>tokenizer.tokenize("stirrer")
tokenized_text = []
mylabels = []
for sent, tags in zip(sentences,labels):
  BERT_texts = []
  BERT_labels = np.array([])
  for word, tag in zip(sent.split(),tags):
    sub_words = tokenizer.wordpiece_tokenizer.tokenize(word)
    tags = np.array([tag for x in sub_words])
    tags[1:] = "X"
    BERT_texts += sub_words
    BERT_labels = np.append(BERT_labels,tags)
  tokenized_text.append(BERT_texts)
  mylabels.append(list(BERT_labels))

print(tokenized_text[1431])
print(mylabels[1431])
# print(sentences[1431])
# tokenized_text = [tokenizer.tokenize(sent) for sent in sentences]
# print(tokenized_text[0])
# print(labels[0])
# print(len(labels[0]))<jupyter_output><empty_output><jupyter_text>Pad all the tokenized sentences and labels to the same length<jupyter_code>input_ids = pad_sequences([tokenizer.convert_tokens_to_ids(txt) for txt in tokenized_text],
                          maxlen=MAX_LEN, dtype="long", truncating="post", padding="post")

tags = pad_sequences([[tag2idx.get(l) for l in lab] for lab in mylabels],
                     maxlen=MAX_LEN, value=tag2idx["O"], dtype="long", truncating="post", padding="post")

print(input_ids[0])
print(tags[0])<jupyter_output>[ 8060  1998  1996 10867  8913  2571 22601  5144  1997  1996 28774 24041
 18404 29624 14854  3122  4221  8670  2595 26775  2629  3366  2620  1006
  1014 29625  2629  1026  1060  1026  1014 29625 24087  1007  5024  5576
  1012     0     0     0     0     0     0     0     0     0     0     0
     0     0     0     0     0     0     0     0     0     0     0     0
     0     0     0     0]
[50 50 50 53 53 53 53 50 50 50 50 53 50 53 53 53 53 11 53 53 53 53 53 50
  3 53 53 50 50 50  3 53 53 50 50 50 50 50 50 50 50 50 50 50 50 50 50 50
 50 50 50 50 50 50 50 50 50 50 50 50 50 50 50 50]<jupyter_text>Create attention masks for the attention model<jupyter_code>attention_masks = [[float(i > 0) for i in ii] for ii in input_ids]
print(attention_masks)<jupyter_output>[[1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0], [1.0, 1.0, 1.0, 1.0, 1.0, 1.0, 1.[...]<jupyter_text>Split the data into 90% training set and 10% validation set<jupyter_code>tr_inputs, val_inputs, tr_tags, val_tags = train_test_split(input_ids, tags,
                                                            random_state=2018, test_size=0.1)
tr_masks, val_masks, _, _ = train_test_split(attention_masks, input_ids,
                                             random_state=2018, test_size=0.1)<jupyter_output><empty_output><jupyter_text>Convert the data into Torch tensor format for later processing<jupyter_code>tr_inputs = torch.tensor(tr_inputs)
val_inputs = torch.tensor(val_inputs)
tr_tags = torch.tensor(tr_tags)
val_tags = torch.tensor(val_tags)
tr_masks = torch.tensor(tr_masks)
val_masks = torch.tensor(val_masks)<jupyter_output><empty_output><jupyter_text>Define the training and validation data in the DataLoader for NLP model<jupyter_code>train_data = TensorDataset(tr_inputs, tr_masks, tr_tags)
train_sampler = RandomSampler(train_data)
train_dataloader = DataLoader(train_data, sampler=train_sampler, batch_size=BATCH_SIZE)

valid_data = TensorDataset(val_inputs, val_masks, val_tags)
valid_sampler = SequentialSampler(valid_data)
valid_dataloader = DataLoader(valid_data, sampler=valid_sampler, batch_size=BATCH_SIZE)<jupyter_output><empty_output><jupyter_text>Finetuning the BERT model<jupyter_code>model = BertForTokenClassification.from_pretrained("bert-base-uncased", num_labels=len(tag2idx))
# model.resize_token_embeddings(len(tokenizer))<jupyter_output><empty_output><jupyter_text>Load the data into GPU<jupyter_code>model.cuda()
FULL_FINETUNING = True
if FULL_FINETUNING:
    param_optimizer = list(model.named_parameters())
    no_decay = ['bias', 'gamma', 'beta']
    optimizer_grouped_parameters = [
        {'params': [p for n, p in param_optimizer if not any(nd in n for nd in no_decay)],
         'weight_decay_rate': 0.01},
        {'params': [p for n, p in param_optimizer if any(nd in n for nd in no_decay)],
         'weight_decay_rate': 0.0}
    ]
else:
    param_optimizer = list(model.classifier.named_parameters()) 
    optimizer_grouped_parameters = [{"params": [p for n, p in param_optimizer]}]
# USE ADAM for gradient descent
optimizer = Adam(optimizer_grouped_parameters, lr=3e-5)<jupyter_output><empty_output><jupyter_text>Define metrics for finetuning<jupyter_code>from seqeval.metrics import f1_score, classification_report

def flat_accuracy(preds, labels):
    pred_flat = np.argmax(preds, axis=2).flatten()
    labels_flat = labels.flatten()
    return np.sum(pred_flat == labels_flat) / len(labels_flat)
def flat_accuracy_no_x(preds, labels):
  pred_flat = np.argmax(preds, axis=2).flatten()
  labels_flat = labels.flatten()
  labels_non_x_indices = (labels_flat != tag2idx["X"])
  labels_non_x = labels_flat[labels_non_x_indices]
  pred_non_x = pred_flat[labels_non_x_indices]
  return np.sum(pred_non_x == labels_non_x) / len(labels_non_x)
epochs = 5
max_grad_norm = 1.0
counter = 0
s_labels = None
for batch in train_dataloader:
  if counter > 0:
    break
  _,_, s_labels = batch
  counter += 1
print(s_labels)
for _ in trange(epochs, desc="Epoch"):
    model.train()
    tr_loss = 0
    nb_tr_examples, nb_tr_steps = 0, 0
    for step, batch in enumerate(train_dataloader):
        # Use batch training to speed up training speed
        batch = tuple(t.to(device) for t in batch)
        b_input_ids, b_input_mask, b_labels = batch
#         b_labels[b_labels == tag2idx["X"]] = -1
        # Forward-prop pass and loss computing
        loss, _ = model(b_input_ids, token_type_ids=None,
                        attention_mask=b_input_mask, labels=b_labels)
        # Back-prop
        loss.backward()
        # track training loss
        tr_loss += loss.item()
        nb_tr_examples += b_input_ids.size(0)
        nb_tr_steps += 1
        # Gradient clipping to prevent gradient explosion
        torch.nn.utils.clip_grad_norm_(parameters=model.parameters(), 
                                       max_norm=max_grad_norm)
        # Update parameters
        optimizer.step()
        model.zero_grad()
    print("Avg Training Loss Per Epoch: {}".format(tr_loss/nb_tr_steps))

    # Validation
    model.eval()
    eval_loss, eval_accuracy = 0, 0
    nb_eval_steps, nb_eval_examples = 0, 0
    predictions, true_labels = [], []
    for batch in valid_dataloader:
        batch = tuple(t.to(device) for t in batch)
        b_input_ids, b_input_mask, b_labels = batch
#         b_labels[b_labels == tag2idx["X"]] = -1
        with torch.no_grad():
            outputs = model(b_input_ids, token_type_ids=None,
                            attention_mask=b_input_mask, labels=b_labels)
            tmp_eval_loss, logits = outputs[:2]
        logits = logits.detach().cpu().numpy()
        label_ids = b_labels.to("cpu").numpy()
        predictions.extend([list(p) for p in np.argmax(logits, axis=2)])
        true_labels.append(label_ids)

        tmp_eval_accuracy = flat_accuracy_no_x(logits, label_ids)

        eval_loss += tmp_eval_loss.mean().item()
        eval_accuracy += tmp_eval_accuracy

        nb_eval_examples += b_input_ids.size(0)
        nb_eval_steps += 1
    eval_loss = eval_loss / nb_eval_steps
    print("Validation loss: {}".format(eval_loss))
    print("Validation accuracy: {}".format(eval_accuracy/nb_eval_steps))
    pred_tags = [tags_vals[p_i] for p in predictions for p_i in p]
    valid_tags = [tags_vals[l_ii] for l in true_labels for l_i in l for l_ii in l_i]
    print("F1 score: {}".format(f1_score(pred_tags, valid_tags)))<jupyter_output>Epoch:   0%|          | 0/5 [00:00<?, ?it/s][A<jupyter_text>Evaluate the model<jupyter_code>model.eval()
predictions = []
true_labels = []
eval_loss, eval_accuracy = 0, 0
nb_eval_steps, nb_eval_examples = 0, 0
for batch in valid_dataloader:
    batch = tuple(t.to(device) for t in batch)
    b_input_ids, b_input_mask, b_labels = batch
#     print(b_input_ids)
#     print(b_input_ids[1,:].tolist())
    with torch.no_grad():
        tmp_eval_loss, logits = model(b_input_ids, token_type_ids=None,
                                      attention_mask=b_input_mask, labels=b_labels)[:2]
        
    logits = logits.detach().cpu().numpy()
#     print(logits)
    predictions.extend([list(p) for p in np.argmax(logits, axis=2)])
    label_ids = b_labels.to('cpu').numpy()
    true_labels.append(label_ids)
    tmp_eval_accuracy = flat_accuracy_no_x(logits, label_ids)

    eval_loss += tmp_eval_loss.mean().item()
    eval_accuracy += tmp_eval_accuracy

    nb_eval_examples += b_input_ids.size(0)
    nb_eval_steps += 1

pred_tags = [[tags_vals[p_i] for p_i in p] for p in predictions]
valid_tags = [[tags_vals[l_ii] for l_ii in l_i] for l in true_labels for l_i in l]
print("Validation loss: {}".format(eval_loss/nb_eval_steps))
print("Validation Accuracy: {}".format(eval_accuracy/nb_eval_steps))
print("Validation F1-Score: {}".format(f1_score(pred_tags, valid_tags)))
valid_tags[0] == "X"
print("{}".format(classification_report(valid_tags, pred_tags)))
tags_vals.remove("O")
tags_vals.remove("X")
sorted_labels = sorted(tags_vals)
print("{}".format(flat_classification_report(valid_tags, pred_tags, labels=sorted_labels)))
print(pred_tags[1][:43])
print(valid_tags[1][:43])
ids = [2044, 2582, 18385, 2007, 1037, 8060, 16130, 14544, 2005, 1020, 1044, 2012, 2282, 4860, 1010, 1996, 21500, 8150, 2001, 4015, 2000, 1037, 8915, 10258, 2239, 1011, 7732, 18676, 1011, 3886, 8285, 23650, 1998, 9685, 2012, 8574, 2139, 18195, 2005, 1023, 2420, 2104, 21552, 1006, 3438, 11575, 1007, 1012, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
print(tokenizer.convert_ids_to_tokens(ids[:43]))<jupyter_output>['after', 'further', 'stirring', 'with', 'a', 'magnetic', 'stir', '##rer', 'for', '6', 'h', 'at', 'room', 'temperature', ',', 'the', 'gel', 'mixture', 'was', 'transferred', 'to', 'a', 'te', '##fl', '##on', '-', 'lined', 'stainless', '-', 'steel', 'auto', '##clave', 'and', 'heated', 'at', '140', 'de', '##gc', 'for', '9', 'days', 'under', 'tumbling']<jupyter_text>NER Visualization<jupyter_code>from spacy import displacy
tags_uppercase = [tag.upper() for tag in tags_vals]
print(tags_uppercase)
print(len(tags_uppercase))
COLORS = {"AMOUNT_MISC": "linear-gradient(90deg, #aa9cfc, #fc9ce7)",
          "NUMBER": "linear-gradient(90deg, orange, cyan)",
          "AMOUNT_UNIT": "linear-gradient(90deg, red, orange)",
          "PROPERTY_MISC": "linear-gradient(90deg, purple 40%, yellow)",
          "MATERIAL": "#aa9cfc",
          "NONRECIPE_MATERIAL": "red",
          "TARGET": "#a4893d",
          "META": "yellow",
          "UNSPECIFIED_MATERIAL": "#0074D9",
          "APPARATUS_UNIT": "linear-gradient(90deg, #e66465, #9198e5)",
          "MATERIAL_DESCRIPTOR": "#9198e5",
          "SOLVENT": "#e66465",
          "PROPERTY_TYPE": "brown",
          "PRECURSOR": "pink",
          "CONDITION_MISC": "#fc9ce7",
          "APPARATUS_PROPERTY_TYPE": "orange",
          "PROPERTY_UNIT": "linear-gradient(217deg, rgba(255,0,0,.8), rgba(255,0,0,0) 70.71%)",
          "CONDITION_UNIT": "linear-gradient(217deg, rgba(800,0,0,.8), yellow 70.71%)",
          "APPARATUS_DESCRIPTOR": "#fea49f",
          "SYNTHESIS_APPARATUS": "#bf4aa8",
          "OPERATION": "#9e363a",
          "CHARACTERIZATION_APPARATUS": "#4f5f76",
          "BRAND": "#e4decd",
          "CONDITION_TYPE": "#8bf0ba",
          "GAS": "#ffdc6a",
          "REFERENCE": "#feda6a"
          }
def ner_visualize(sentence, tags, colors=COLORS):
    sentence_concat = " ".join(sentence)
    ents = []
    start = 0
    end = 0
    for word, tag in zip(sentence, tags):
        end = start + len(word) - 1
        ents.append({"start": start, "end": end+1,
                     "label": tag[2:].upper() if tag != "O" else tag.upper()})
        start = end + 2

    test = [{"text": sentence_concat,
             "ents": ents,
             "title": None}]

    options = {"ents": [tag[2:].upper() for tag in set(tags) if tag not in ["O"]], "colors": colors}
    displacy.render(test, style="ent", manual=True, options=options)<jupyter_output><empty_output><jupyter_text>Example labels<jupyter_code>text = ['after', 'further', 'stirring', 'with', 'a', 'magnetic', 'stir', '##rer', 'for', '6', 'h', 'at', 'room', 'temperature', ',', 'the', 'gel', 'mixture', 'was', 'transferred', 'to', 'a', 'te', '##fl', '##on', '-', 'lined', 'stainless', '-', 'steel', 'auto', '##clave', 'and', 'heated', 'at', '140', 'de', '##gc', 'for', '9', 'days', 'under', 'tumbling']
true_labels = ['O', 'O', 'B-operation', 'O', 'O', 'B-synthesis_apparatus', 'I-synthesis_apparatus', 'X', 'O', 'B-number', 'B-condition_unit', 'O', 'B-condition_misc', 'I-condition_misc', 'O', 'O', 'B-unspecified_material', 'I-unspecified_material', 'O', 'B-operation', 'O', 'O', 'B-apparatus_descriptor', 'X', 'X', 'I-apparatus_descriptor', 'I-apparatus_descriptor', 'I-apparatus_descriptor', 'I-apparatus_descriptor', 'I-apparatus_descriptor', 'B-synthesis_apparatus', 'X', 'O', 'B-operation', 'O', 'B-number', 'B-condition_unit', 'X', 'O', 'B-number', 'B-condition_unit', 'O', 'B-operation']
pred_labels = ['O', 'O', 'B-operation', 'O', 'O', 'B-synthesis_apparatus', 'B-synthesis_apparatus', 'X', 'O', 'B-number', 'B-condition_unit', 'O', 'B-condition_misc', 'I-condition_misc', 'O', 'O', 'B-unspecified_material', 'B-unspecified_material', 'O', 'B-operation', 'O', 'O', 'B-apparatus_descriptor', 'X', 'X', 'I-apparatus_descriptor', 'I-apparatus_descriptor', 'I-apparatus_descriptor', 'I-apparatus_descriptor', 'I-apparatus_descriptor', 'B-synthesis_apparatus', 'X', 'O', 'B-operation', 'O', 'B-number', 'B-condition_unit', 'X', 'O', 'B-number', 'B-condition_unit', 'O', 'B-operation']<jupyter_output><empty_output><jupyter_text>True labels<jupyter_code>ner_visualize(text, true_labels)<jupyter_output><empty_output><jupyter_text>Predicted labelsThe color coding shows that the model makes a perfect prediction for this sentence, but in reality, the model makes a wrong prediction for the word "stir" in the model.<jupyter_code>ner_visualize(text, pred_labels)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16378644525970265, 'fraction_numerical': 0.058264143657076654, 'mean_word_length': 3.9092417061611373, 'pattern_counts': {'":': 35, '<': 83, '<?xml version=': 0, '>': 82, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 32, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '22', 'dir': 'jupyter-structured-clean-dedup', 'id': '78000', 'license': 'MIT', 'n_tokens_mistral': 8730, 'n_tokens_neox': 7839, 'n_words': 1839, 'path': 'MatNER_v2.ipynb', 'repo_name': 'rpw199912j/MatBERT', 'size': '20716'}