<jupyter_start><jupyter_text>Conll 2003 Name Entity Recognitiondataset link: https://huggingface.co/datasets/conll2003<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
! pip install transformers
import numpy as np
import pandas as pd

from transformers import AutoTokenizer

from tqdm import tqdm
import tensorflow as tf

import matplotlib.pyplot as plt
def load_sentences(filepath):

    final = []
    sentences = []

    with open(filepath, 'r') as f:
        
        for line in f.readlines():
            
            if (line == ('-DOCSTART- -X- -X- O\n') or line == '\n'):
                if len(sentences) > 0:
                    final.append(sentences)
                    sentences = []
            else:
                l = line.split(' ')
                sentences.append((l[0], l[3].strip('\n')))
    
    return final
base_path = './drive/MyDrive/berkeley/capstone/demo/conll2003/'

train_samples = load_sentences(base_path + 'train.txt')
test_samples = load_sentences(base_path + 'test.txt')
valid_samples = load_sentences(base_path + 'valid.txt')

samples = train_samples + test_samples

schema = ['_'] + sorted({tag for sentence in samples 
                             for _, tag in sentence})
# dataset quick view
train_data = []
sentences=[]
with open(base_path + 'train.txt', 'r') as f:
  for line in f.readlines():
      if (line == ('-DOCSTART- -X- -X- O\n') or line == '\n'):
        if len(sentences) > 0:
          train_data.append(sentences)
          sentences = []
        else:
          l = line.split(" ")
          sentences.append((l[i].strip('\n') for i in range(4)))
l[0].strip('\n')<jupyter_output>[1;30;43mStreaming output truncated to the last 5000 lines.[0m


































































































-DOCSTART- -X- -X- O













-DOCSTART- -X- -X- O

















-DOCSTART- -X- -X- O











































-DOCSTART- -X- -X- O































































































-DOCSTART- -X- -X- O











-DOCSTART- -X- -X- O





























































-DOCSTART- -X- -X- O



































-DOCSTART- -X- -X- O

























































































































































































-DOCSTART- -X- -X- O



















-DOCSTART- -X- -X- O





























-DOCSTART- -X- -X- O











































-DOCSTART- -X- -X- O





























-DOCS[...]<jupyter_text>Model<jupyter_code>from transformers import AutoConfig, TFAutoModelForTokenClassification

MODEL_NAME = 'bert-base-cased' 

config = AutoConfig.from_pretrained(MODEL_NAME, num_labels=len(schema))
model = TFAutoModelForTokenClassification.from_pretrained(MODEL_NAME, 
                                                          config=config)
model.summary()
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)

def tokenize_sample(sample):
    seq = [
               (subtoken, tag)
               for token, tag in sample
               for subtoken in tokenizer(token)['input_ids'][1:-1]
           ]
    return [(3, 'O')] + seq + [(4, 'O')]

def preprocess(samples):
    tag_index = {tag: i for i, tag in enumerate(schema)}
    tokenized_samples = list(tqdm(map(tokenize_sample, samples)))
    max_len = max(map(len, tokenized_samples))
    X = np.zeros((len(samples), max_len), dtype=np.int32)
    y = np.zeros((len(samples), max_len), dtype=np.int32)
    for i, sentence in enumerate(tokenized_samples):
        for j, (subtoken_id, tag) in enumerate(sentence):
            X[i, j] = subtoken_id
            y[i,j] = tag_index[tag]
    return X, y

X_train, y_train = preprocess(train_samples)
X_test, y_test = preprocess(test_samples)
X_valid, y_valid = preprocess(valid_samples)
EPOCHS=5
BATCH_SIZE=8

optimizer = tf.keras.optimizers.Adam(lr=0.000001)
loss = tf.keras.losses.SparseCategoricalCrossentropy(from_logits=True)
model.compile(optimizer=optimizer, loss=loss, metrics='accuracy')
history = model.fit(tf.constant(X_train), tf.constant(y_train),
                    validation_data=(X_test, y_test), 
                    epochs=EPOCHS, 
                    batch_size=BATCH_SIZE)
plt.figure(figsize=(14,8))
plt.title('Losses')
plt.plot(history.history['loss'], label='Train Loss')
plt.plot(history.history['val_loss'], label='Valid Loss')
plt.ylabel('Loss')
plt.xlabel('Epochs')
plt.legend()
plt.show()<jupyter_output><empty_output><jupyter_text><jupyter_code><jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12163892445582586, 'fraction_numerical': 0.010243277848911651, 'mean_word_length': 1.7265852239674229, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 14, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '418770', 'license': 'MIT', 'n_tokens_mistral': 1455, 'n_tokens_neox': 1385, 'n_words': 305, 'path': 'conll_2003baseline.ipynb', 'repo_name': 'OscarChang46/de-identify-sensitive-data-for-cybersecurity-use-cases-e.g.-PII-and-PHI-data-in-unstructured-text-an', 'size': '4686'}