<jupyter_start><jupyter_text>Implementation of Continuous bag of Words Model  Importing dependencies<jupyter_code>from keras.preprocessing import text
from keras.utils.np_utils import to_categorical
from keras.layers import Embedding , Dense , Flatten , Lambda
from keras.preprocessing.sequence import pad_sequences
import numpy as np
import keras.backend as k
from keras.models import Sequential
import re
import pandas as pd
import nltk
nltk.download('gutenberg')
from nltk.corpus import gutenberg
gutenberg.fileids()
raw_text = gutenberg.raw('bible-kjv.txt')
raw_text = re.sub('\n',' ',raw_text)
import re 
corpus = re.compile('[0-9]+:[0-9]+').split(raw_text)
len(corpus)
corpus = corpus[:10000]
tokenizer = text.Tokenizer(num_words = 5000)
tokenizer.fit_on_texts(corpus)
word2id = tokenizer.word_index
word2id['<PAD>'] = 0
len(word2id)
id2word = {id:word for word , id in word2id.items()}
type(word2id)
vocab_size = len(word2id)
embedding_size = 128
window_size = 2
num_epochs = 5
wids = [[word2id[word] for word in text.text_to_word_sequence(doc)] for doc in corpus]
len(wids[4])<jupyter_output><empty_output><jupyter_text>defining function to generate context and target word pairs<jupyter_code>def generate_context_target_word_pairs(corpus , vocab_size , window_size):
    context_length = 2*window_size
    for words in corpus:
        sequence_length = len(words)
        for index , word in enumerate(words):
            start = index - window_size
            end = index + window_size + 1 
            context_words = []
            label_word = []
            context_words.append([words[i] for i in range(start,end) if 0 <= i < sequence_length and i != index])
            label_word.append(word)
            #x = context_words
            x = pad_sequences(context_words , maxlen = context_length , padding = 'pre' , truncating = 'pre')
            y = to_categorical(label_word , vocab_size)
            yield(x,y)<jupyter_output><empty_output><jupyter_text>Generating Context and target pairs<jupyter_code>i = 0
for x,y in generate_context_target_word_pairs(corpus = wids , vocab_size = vocab_size , window_size = window_size):
    if 0 not in x[0]:
        
        print("Context Words :" , [id2word[i] for i in x[0]] , '-> Target Word :' , id2word[np.argmax(y)])
        if i == 20:
            break
        i = i + 1

import keras.backend as K

model = Sequential()
model.add(Embedding(input_dim = vocab_size , output_dim = embedding_size , input_length = window_size * 2))
model.add(Lambda(lambda x: K.mean(x , axis  = 1) , output_shape = (embedding_size ,)))
model.add(Dense(output_dim = vocab_size , activation = 'softmax'))
model.compile(loss = 'categorical_crossentropy' , optimizer = 'rmsprop')

print(model.summary())<jupyter_output>_________________________________________________________________
Layer (type)                 Output Shape              Param #   
=================================================================
embedding_4 (Embedding)      (None, 4, 128)            873344    
_________________________________________________________________
lambda_4 (Lambda)            (None, 128)               0         
_________________________________________________________________
dense_4 (Dense)              (None, 6823)              880167    
=================================================================
Total params: 1,753,511
Trainable params: 1,753,511
Non-trainable params: 0
_________________________________________________________________
None<jupyter_text>Training the Model<jupyter_code>for epoch in range(num_epochs):
    loss = 0
    i = 0 
    for x , y in generate_context_target_word_pairs(corpus = wids , vocab_size = vocab_size , window_size = window_size):
        i = i + 1
        loss+=model.train_on_batch(x,y)
    print('Epoch :' , epoch , '-> Loss :' , loss)
    print()
            
weights = model.get_weights()[0]
weights = weights[1:]
print(weights.shape)
pd.DataFrame(weights, index=list(id2word.values())[1:]).head()
from sklearn.metrics.pairwise import euclidean_distances

# compute pairwise distance matrix
distance_matrix = euclidean_distances(weights)
print(distance_matrix.shape)

# view contextually similar words
similar_words = {search_term: [id2word[idx] for idx in distance_matrix[word2id[search_term]-1].argsort()[1:6]+1] 
                   for search_term in ['god', 'noah', 'egypt', 'commanded']}

similar_words<jupyter_output>(6822, 6822)
{'fraction_non_alphanumeric': 0.12965050732807215, 'fraction_numerical': 0.02367531003382187, 'mean_word_length': 4.110599078341014, 'pattern_counts': {'":': 0, '<': 18, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '542417', 'license': 'MIT', 'n_tokens_mistral': 1459, 'n_tokens_neox': 1364, 'n_words': 323, 'path': 'Word2Vec.ipynb', 'repo_name': 'python-freak/CBOW-Implementation-', 'size': '4435'}