<jupyter_start><jupyter_text>Notebook for training NER for CoNL2003Achives 91.5-97.7 f1 measure<jupyter_code>word_dim = 100
word_embeddings_folder = '/home/saif/Downloads/'
word_embeddings_file = word_embeddings_folder + 'glove.6B.{}d.txt'.format(word_dim)

dataset_folder = '/home/saif/Downloads/'
import numpy as np
import os
import tensorflow as tf
import string
import random
import math
import sys
import time

from ner_model import NerModel
from dataset_encoder import DatasetEncoder
from ner_model_saver import NerModelSaver
def read_glove(file):
    word2id = {}
    vectors = []
    
    def add_vector(word, vector):
        vectors.append(vector)
        word2id[word] = len(word2id)        
    
    dummy_added = False
    with open(file) as f:
        for line in f:
            items = line.split(' ')
            word = items[0]
            vector = [float(x) for x in items[1:]]
            
            if not dummy_added:
                add_vector('__oov__', [0] * len(vector))
                dummy_added = True
            
            add_vector(DatasetEncoder.normalize(word), vector)
    
    return word2id, vectors


# Returns sentences, each sentence is an array of tuples: (word, tag)
def read_conll(folder):
    
    # array of tuple (word, tag)
    sentence = []
    
    for root, subdirs, files in os.walk(folder):
        #print("root is: " + str(root))
        #print("subdirs are: " + str(subdirs))
        #print("files are: " + str(files))
        for file in files:
            curfile = os.path.join(root, file)
            #print("parsing conll file: " + str(curfile))
            with open(curfile) as f:
                for line in f:                
                    items = line.strip().split(' ')
                    if len(items) < 4 or items[0] == '-DOCSTART-':
                        if len(sentence) > 0:
                            yield sentence
                            
                            sentence = []
                    else:
                        word = items[0]
                        tag = items[3]

                        sentence.append((word, tag))
                
    if len(sentence) > 0:
        yield sentence
word2id, embeddings = read_glove(word_embeddings_file)
print(len(word2id))
encoder = DatasetEncoder(word2id, embeddings)
train = list(encoder.encode(read_conll(dataset_folder)))
valid = train
test = train
def words_in_embeddings(dataset):
    zero = 0
    other = 0
    for sentence in dataset:
        for word_id in sentence["word_ids"]:
            if word_id == 0:
                zero += 1
            else:
                other += 1
    
    return (zero, other)

(zero, other) = words_in_embeddings(train)
print('train word embeddings coverage: {}'.format(other / (zero + other)))

(zero, other) = words_in_embeddings(valid)
print('valid word embeddings coverage: {}'.format(other / (zero + other)))

(zero, other) = words_in_embeddings(test)
print('test word embeddings coverage: {}'.format(other / (zero + other)))
print(len(list(encoder.encode(read_conll(dataset_folder)))))

labels = set()
for item in read_conll(dataset_folder):
    labels  = labels | set(([label for (word, label) in item]))
    
    
print(labels)
print('chars: {}'.format(len(encoder.char2id)))

all_chars = set()
for item in read_conll(dataset_folder):
    for (word, label) in item:
        all_chars = all_chars | set(word)
    
print('chars: {}'.format(len(all_chars)))
ner = NerModel()
ner.add_cnn_char_repr(101, 25, 30)
ner.add_pretrained_word_embeddings(word_dim)
ner.add_context_repr(10, 200)
ner.add_inference_layer(False)
ner.add_training_op(5.0)

ner.init_variables()
for i in range(0, 1):
    t = time.time()
    ner.train(train, 
          valid, 
          lr = 0.2,
          po = 0.05,
          batch_size = 90,
          dropout = 0.5, 
          epoch_start = i, 
          epoch_end = i + 1
    )
    (prec, rec, f1) = ner.measure(test)
    print('Test quality prec: {}, rec: {}, f1: {}'.format(prec, rec, f1))
    print("epoch time: " + str(time.time() - t))
    
saver = NerModelSaver(ner, encoder, embeddings_file=word_embeddings_file)
saver.save('conll_model')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11752085816448153, 'fraction_numerical': 0.015733015494636474, 'mean_word_length': 2.2909803921568628, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '102893', 'license': 'Apache-2.0', 'n_tokens_mistral': 1380, 'n_tokens_neox': 1319, 'n_words': 327, 'path': 'python/tensorflow/ner/conll2003_folderBased.ipynb', 'repo_name': 'PaddyK92/spark-nlp', 'size': '4195'}