<jupyter_start><jupyter_text>Feature Generation<jupyter_code>word2idx = {w: i for i, w in enumerate(words)}
tag2idx = {t: i for i, t in enumerate(tags)}
print(type(word2idx))
dict_items = word2idx.items()
first_5 = list(dict_items)[:5]
print(first_5)

X = [[word2idx[w[0]] for w in s] for s in sentences]
X = pad_sequences(maxlen=150, sequences=X, padding="post",value=n_words - 1)
y = [[tag2idx[w[1]] for w in s] for s in sentences]
y = pad_sequences(maxlen=150, sequences=y, padding="post", value=tag2idx["O"])
from keras.utils import to_categorical
y = [to_categorical(i, num_classes=n_tags) for i in y]
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.2)
from keras.models import Model, Input
from keras.layers import LSTM, Embedding, Dense, TimeDistributed, Dropout, Bidirectional
input = Input(shape=(150,))
model = Embedding(input_dim=n_words, output_dim=150, input_length=150)(input)
model = Dropout(0.1)(model)
model = Bidirectional(LSTM(units=150, return_sequences=True, recurrent_dropout=0.1))(model)
out = TimeDistributed(Dense(n_tags, activation="softmax"))(model)
model = Model(input, out)
model.compile(optimizer="adam", loss="categorical_crossentropy", metrics=["accuracy"])
history = model.fit(X_train, np.array(y_train), batch_size=32, epochs=10, validation_split=0.2, verbose=1)
i = 2
p = model.predict(np.array([X_test[i]]))
p = np.argmax(p, axis=-1)
print("{:14} ({:5}): {}".format("Word", "True", "Pred"))
for w,pred in zip(X_test[i],p[0]):
    print("{:15}: {}".format(words[w],tags[pred]))
pred = []
for i in range(len(y_test)):
    p = model.predict(np.array([X_test[i]]))
    p = np.argmax(p, axis=-1)
    pred.append(p)
pred_temp = [x[0] for x in pred]
y_temp = []
for i in range(len(y_test)):
    y_temp.append(np.array([np.where(y_test[i][j]==1)[0][0] for j in range(len(y_test[i]))]))
actual = y_temp[0]
for i in range(1, len(y_temp)):
    actual = np.concatenate((actual, y_temp[i]), axis=None)
predicted = pred_temp[0]
for i in range(1, len(pred_temp)):
    predicted = np.concatenate((predicted, pred_temp[i]), axis=None)
import scikitplot as skplt
from sklearn.metrics import classification_report
skplt.metrics.plot_confusion_matrix(actual, predicted)
print(classification_report(actual, predicted, target_names=tags))<jupyter_output>precision    recall  f1-score   support

       B-PER       0.93      0.90      0.92       935
       B-LOC       1.00      0.96      0.98      1857
           O       1.00      1.00      1.00    314472
       I-PER       1.00      0.96      0.98      3266
       I-LOC       1.00      0.98      0.99      1393
       B-ORG       0.99      0.94      0.96      1330
       I-ORG       1.00      0.98      0.99      4347

    accuracy                           1.00    327600
   macro avg       0.99      0.96      0.97    327600
weighted avg       1.00      1.00      1.00    327600
{'fraction_non_alphanumeric': 0.14534883720930233, 'fraction_numerical': 0.06600547195622435, 'mean_word_length': 3.4385432473444615, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '488109', 'license': 'MIT', 'n_tokens_mistral': 1186, 'n_tokens_neox': 1099, 'n_words': 250, 'path': 'Base Models/BI-LSTM-Hindi.ipynb', 'repo_name': 'sagunesh-grover/capstone-research-project-1003', 'size': '2924'}