<jupyter_start><jupyter_text>Data Loading<jupyter_code>data = pd.read_csv("ner_dataset.csv", encoding="latin1")
data = data.drop(['POS'], axis =1)
data.head()
plt.style.use("ggplot")
data = pd.read_csv("ner_dataset.csv", encoding="latin1")
data = data.drop(['POS'], axis =1)
data = data.fillna(method="ffill")
words = set(list(data['Word'].values)) #Vocabulary
words.add('PADword')
n_words = len(words)
tags = list(set(data["Tag"].values))
n_tags = len(tags)
print(n_words,n_tags)<jupyter_output>35179 17<jupyter_text>Data Preprocessing<jupyter_code>class SentenceGetter(object):
  def __init__(self, data):
      self.n_sent = 1
      self.data = data
      self.empty = False
      agg_func = lambda s: [(w, t) for w, t in zip(s["Word"].
                                                   values.tolist(),s["Tag"].values.tolist())]
      self.grouped = self.data.groupby("Sentence #").apply(agg_func)
      self.sentences = [s for s in self.grouped]

  def get_next(self):
      try:
          s = self.grouped["Sentence: {}".format(self.n_sent)]
          self.n_sent += 1
          return s
      except:
          return None
getter = SentenceGetter(data)

sent = getter.get_next()
print(sent)
sentences = getter.sentences
print(len(sentences))
largest_sen = max(len(sen) for sen in sentences)
print('biggest sentence has {} words'.format(largest_sen))
%matplotlib inline
plt.hist([len(sen) for sen in sentences],bins=50)
plt.xlabel('Sentence Length')
plt.ylabel('Frequency')
plt.show()
max_len = 50
X = [[w[0]for w in s] for s in sentences]
new_X = []
for seq in X:
    new_seq = []
    for i in range(max_len):
        try:
            new_seq.append(seq[i])
        except:
            new_seq.append("PADword")
    new_X.append(new_seq)
print(new_X[0])
sentences[0]
list(enumerate(tags))

tags2index
from keras.preprocessing.sequence import pad_sequences
tags2index = {t:i for i,t in enumerate(tags)}
y = [[tags2index[w[1]] for w in s] for s in sentences]
y = pad_sequences(maxlen=max_len, sequences=y, padding="post", value=tags2index["O"])
y<jupyter_output>Using TensorFlow backend.<jupyter_text>Model Building and Training<jupyter_code>from sklearn.model_selection import train_test_split
import tensorflow as tf
import tensorflow_hub as hub
from keras import backend as K
X_tr, X_te, y_tr, y_te = train_test_split(new_X, y, test_size=0.1, random_state=2018)
sess = tf.Session()
K.set_session(sess)
elmo_model = hub.Module("https://tfhub.dev/google/elmo/2", trainable=True)
sess.run(tf.global_variables_initializer())
sess.run(tf.tables_initializer())
batch_size = 32
def ElmoEmbedding(x):
    return elmo_model(inputs={"tokens": tf.squeeze(tf.cast(x,
                                                           tf.string)),"sequence_len": tf.constant(batch_size*[max_len])
                     },signature="tokens",as_dict=True)["elmo"]

from keras.models import Model, Input
from keras.layers.merge import add
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional, Lambda
input_text = Input(shape=(max_len,), dtype=tf.string)
embedding = Lambda(ElmoEmbedding, output_shape=(max_len, 1024))(input_text)
x = Bidirectional(LSTM(units=512, return_sequences=True,
                       recurrent_dropout=0.2, dropout=0.2))(embedding)
x_rnn = Bidirectional(LSTM(units=512, return_sequences=True,
                           recurrent_dropout=0.2, dropout=0.2))(x)
x = add([x, x_rnn])  # residual connection to the first biLSTM
out = TimeDistributed(Dense(n_tags, activation="softmax"))(x)
model = Model(input_text, out)
model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
model.summary()
X_tr, X_val = X_tr[:1213*batch_size], X_tr[-135*batch_size:]
y_tr, y_val = y_tr[:1213*batch_size], y_tr[-135*batch_size:]
y_tr = y_tr.reshape(y_tr.shape[0], y_tr.shape[1], 1)
y_val = y_val.reshape(y_val.shape[0], y_val.shape[1], 1)
history = model.fit(np.array(X_tr), y_tr, validation_data=(np.array(X_val), y_val),batch_size=batch_size, epochs=3, verbose=1)
model.save_weights('bilstm_model.hdf5')
!pip install seqeval<jupyter_output><empty_output><jupyter_text>Model Evaluation<jupyter_code>from seqeval.metrics import precision_score, recall_score, f1_score, classification_report
X_te = X_te[:149*batch_size]
test_pred = model.predict(np.array(X_te), verbose=1)
idx2tag = {i: w for w, i in tags2index.items()}
def pred2label(pred):
    out = []
    for pred_i in pred:
        out_i = []
        for p in pred_i:
            p_i = np.argmax(p)
            out_i.append(idx2tag[p_i].replace("PADword", "O"))
        out.append(out_i)
    return out
def test2label(pred):
    out = []
    for pred_i in pred:
        out_i = []
        for p in pred_i:
            out_i.append(idx2tag[p].replace("PADword", "O"))
        out.append(out_i)
    return out
    
pred_labels = pred2label(test_pred)
test_labels = test2label(y_te[:149*32])
print(classification_report(test_labels, pred_labels))
i = 390
p = model.predict(np.array(X_te[i:i+batch_size]))[0]
p = np.argmax(p, axis=-1)
print("{:15} {:5}: ({})".format("Word", "Pred", "True"))
print("="*30)
for w, true, pred in zip(X_te[i], y_te[i], p):
    if w != "__PAD__":
        print("{:15}:{:5} ({})".format(w, tags[pred], tags[true]))
history.history
?(figsize=(12,12))
?(history.history["acc"],c = 'b')
?(history.history["val_acc"], c = 'g')
plt.show()
test_sentence = [["Hawking", "is", "a", "Fellow", "of", "the", "Royal", "Society", ",", "a", "lifetime", "member",
                 "of", "the", "Pontifical", "Academy", "of", "Sciences", ",", "and", "a", "recipient", "of",
                 "the", "Presidential", "Medal", "of", "Freedom", ",", "the", "highest", "civilian", "award",
                 "in", "the", "United", "States", "."]]
max_len = 50
X_test = [[w for w in s] for s in test_sentence]
new_X_test = []
for seq in X_test:
    new_seq = []
    for i in range(max_len):
        try:
            new_seq.append(seq[i])
        except:
            new_seq.append("PADword")
    new_X_test.append(new_seq)
new_X_test
np.array(new_X_test,dtype='<U26')
np.array(X_te)[1]<jupyter_output><empty_output><jupyter_text>Inference<jupyter_code>#model.load_weights('bilstm_model.hdf5')
#model.compile(optimizer="adam", loss="sparse_categorical_crossentropy", metrics=["accuracy"])
p = ?(np.array(new_X_test*32,dtype='<U26'))[0]
p = ?(p, axis=-1)
print("{:15} {:5}".format("Word", "Pred"))
print("="*30)
for w, pred in zip(new_X_test[0], p):
    if w != "__PAD__":
        print("{:15}:{:5}".format(w, tags[pred]))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16722458916615945, 'fraction_numerical': 0.019933049300060863, 'mean_word_length': 4.25, 'pattern_counts': {'":': 4, '<': 21, '<?xml version=': 0, '>': 19, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '4690', 'license': 'MIT', 'n_tokens_mistral': 2546, 'n_tokens_neox': 2440, 'n_words': 476, 'path': '1 Supervised Learning/Classification/Sequence_Tagging_NER.ipynb', 'repo_name': 'HJJ256/Machine-Learning-2018-present-', 'size': '6572'}