<jupyter_start><jupyter_text>* In Question body and answer body we have to remove code parts and we have to get only text part so we will remove code part and will remove html tags and all punctuations and we will get clean text in lower case.*  We will add new column as preprocessed question body and preprocessed answer body.<jupyter_code># remove code part from question body 
removed_code_questions = []
for i,row in tqdm(df.iterrows()):
    removed_code_questions.append(remove_code(row.body))
# remove stopwords from removed_code_questions
preprocessed_questions = []
for i in tqdm(removed_code_questions):
    preprocessed_questions.append(remove_stopwords(i))

# preprocess title
preprocessed_title = []
for i,row in tqdm(df.iterrows()):
    preprocessed_title.append(preprocess_title(row.title))
# original title
df['title'][2]
# cleaned title
preprocessed_title[2]
# remove code from answers
removed_code_answers = []
for row in tqdm(df['answers']):
    removed_code_answers.append(remove_code(row))
# original answer at index 0
df['answers'][0]
# answers after removing code
removed_code_answers[0]
# remove stopwords from answers
preprocessed_answers = []
for i in tqdm(removed_code_answers):
    preprocessed_answers.append(remove_stopwords(i))
df['answers'][0]
preprocessed_answers[0]
# dump preprocessed question drive
joblib.dump(preprocessed_questions,'/content/drive/My Drive/self_case_study1/data/preprocessed_questions')
# dump preprocessed title in drive
joblib.dump(preprocessed_title,'/content/drive/My Drive/self_case_study1/data/preprocessed_title')
# dump preprocessed answers in drive
joblib.dump(preprocessed_answers,'/content/drive/My Drive/self_case_study1/data/preprocessed_answers')
# load preprocessed questions
preprocessed_question = joblib.load('/content/drive/My Drive/self_case_study1/data/preprocessed_questions')
print('Length of preprocessed questions = ',len(preprocessed_question))
# load preprocessed titles
preprocessed_title = joblib.load('/content/drive/My Drive/self_case_study1/data/preprocessed_title')
print('Length of preprocessed titles= ',len(preprocessed_title))
# load preprocessed answers
preprocessed_answer = joblib.load('/content/drive/My Drive/self_case_study1/data/preprocessed_answers')
print('Length of preprocessed answers= ',len(preprocessed_answer))
tags = df['tags']
id = df['id']
score = df['score']
Dict = { 'id' : id,
         'preprocessed_title' : preprocessed_title,
         'preprocessed_question': preprocessed_question,
         'preprocessed_answer' : preprocessed_answer,
         'score' : score,
         'tags': tags}
preprocessed_df = pd.DataFrame(Dict)
# save as csv
# preprocessed_df.to_csv('/content/drive/My Drive/self_case_study1/data/preprocessed_df.csv',index=False)<jupyter_output><empty_output><jupyter_text>Load preprocesed data<jupyter_code>preprocessed_df = pd.read_csv('/content/drive/My Drive/self_case_study1/data/preprocessed_df.csv')
preprocessed_df.head()
# cheking for null values
print('Total null values in preprocessed titles =',preprocessed_df['preprocessed_title'].isna().sum())
print('Total null values in preprocessed questions =',preprocessed_df['preprocessed_question'].isna().sum())
print('Total null values in preprocessed answers =',preprocessed_df['preprocessed_answer'].isna().sum())
print('Total null values in score =',preprocessed_df['score'].isna().sum())
print('Total null values in tags =',preprocessed_df['tags'].isna().sum())
# concatenate title and question body 
preprocessed_df['question_content'] = preprocessed_df['preprocessed_title'].fillna('') + ' ' + preprocessed_df['preprocessed_question'].fillna('')
# remove all character  which has length 1
def remove_single_char(text):
    l = [word for word in str(text).split() if len(word) > 1]
    return ' '.join(l)
# remove words with length 1 from question content and answer
preprocessed_df['question_content'] = preprocessed_df['question_content'].apply(lambda x: remove_single_char(x))
preprocessed_df['preprocessed_answer'] = preprocessed_df['preprocessed_answer'].apply(lambda x: remove_single_char(x))
# concatenate question content and preprocessed answer
preprocessed_df['all_text'] = preprocessed_df['question_content'].fillna('')+' '+preprocessed_df['preprocessed_answer'].fillna('')
preprocessed_df.head()<jupyter_output><empty_output><jupyter_text>Checking for null values in question content<jupyter_code># checking in preprocessed_df for null titles
preprocessed_df['question_content'].isna().sum()<jupyter_output><empty_output><jupyter_text>Checking for null values in all text<jupyter_code># checking in preprocessed_df for null answers
preprocessed_df['all_text'].isna().sum()
preprocessed_df['all_text'][0]<jupyter_output><empty_output><jupyter_text>Analysis of question content<jupyter_code># import FreqDist from nltk library to get word frequency from given text
from nltk import FreqDist
# get one single list of whole corpus words
all_words = []
for line in tqdm(preprocessed_df['question_content']):
    all_words.extend(line.split())    
# get frequency distribution of all words
word_freq = FreqDist(all_words) 
print('Number of unique words = ',len(word_freq))
# cheecking last 5 words from most common words
word_freq.most_common(20000)[-5:]<jupyter_output><empty_output><jupyter_text>here we are taking top 20000 words because last 5 words of top 20000 words are occuring atleasr 30 times<jupyter_code># top 20000 words
top_k_words = dict(word_freq.most_common(20000)).keys()
# set of top k words 
top_k_words = set(top_k_words)
print('Length of top k words is = ', len(top_k_words))
def keep_top_k_words(text):
    ''' This function will keep only top k words'''
    top_k =  [word for word in text.split() if word in top_k_words]
    return ' '.join(top_k)
# apply keep top words function on question content
preprocessed_df['question_content'] = preprocessed_df['question_content'].apply(lambda x : keep_top_k_words(x))<jupyter_output><empty_output><jupyter_text>Analysis on question content length<jupyter_code># make column of question content length
preprocessed_df['question_length'] = preprocessed_df['question_content'].apply(lambda x: len(x.split()))
preprocessed_df.head()
preprocessed_df.shape
print('50th percentile of question lenght is ==>',np.percentile(preprocessed_df['question_length'],50))
print('75th percentile of question lenght is ==>',np.percentile(preprocessed_df['question_length'],75))
print('90th percentile of question lenght is ==>',np.percentile(preprocessed_df['question_length'],90))
print('95th percentile of question lenght is ==>',np.percentile(preprocessed_df['question_length'],95))
print('98th percentile of question lenght is ==>',np.percentile(preprocessed_df['question_length'],98))<jupyter_output>50th percentile of question lenght is ==> 41.0
75th percentile of question lenght is ==> 60.0
90th percentile of question lenght is ==> 87.0
95th percentile of question lenght is ==> 109.0
98th percentile of question lenght is ==> 143.0<jupyter_text>plot CDF of question length<jupyter_code>import seaborn as sns
sns.kdeplot(preprocessed_df['question_length'],cumulative=True,shade=True)<jupyter_output><empty_output><jupyter_text>as we can see 75th percent of question length are 60 or less then 60 so we will consider all questions which have length less then or equal to 60.<jupyter_code>print('Total number of rows which have question length greater then 60 is  =======>')
print(preprocessed_df[preprocessed_df['question_length']>60].shape[0])
preprocessed_df = preprocessed_df[preprocessed_df['question_length']<60]
print('Total number of rows in preprocessed_df after dropping large length question = ', preprocessed_df.shape[0])
# reset index
preprocessed_df.reset_index(drop=True,inplace=True)
preprocessed_df.head()
# save final featurized dataframe
#preprocessed_df.to_csv('/content/drive/My Drive/self_case_study1/data/featurized_df.csv',index=False)
#featurized_df = pd.read_csv('/content/drive/My Drive/self_case_study1/data/featurized_df.csv')
print('Shape of featurized_df is ===> ',featurized_df.shape)
featurized_df.head(3)
# create dataframe of original title and id
title_df = df[['id','title']]
title_df.head()
# attach original title column using merge operation on id

featurized_df = featurized_df.merge(title_df,on='id')
featurized_df.head(2)<jupyter_output><empty_output><jupyter_text>Save final featurized data with original title as csv<jupyter_code># save final dataframe
featurized_df.to_csv('/content/drive/My Drive/self_case_study1/data/featurized_data.csv',index=False)<jupyter_output><empty_output><jupyter_text>Load featurized dataframe<jupyter_code># load final featurized dataframe
featurized_df = pd.read_csv('/content/drive/My Drive/self_case_study1/data/featurized_data.csv')
from sklearn.model_selection import train_test_split
X_train,X_test = train_test_split(featurized_df,test_size=0.5,random_state=42)

# reset index of train and test
X_train.reset_index(drop=True,inplace=True)
X_test.reset_index(drop=True,inplace=True)

print('Shape of X_train',X_train.shape)
print('Shape of X_test',X_test.shape)
X_train.head()<jupyter_output><empty_output><jupyter_text>**Model-1 : TFIDF word2vec using pretrained glove model**<jupyter_code># function to load glove vectors
def loadGloveModel(gloveFile):
    print ("Loading Glove Model")
    f = open(gloveFile,'r', encoding="utf8")
    model = {}
    for line in tqdm(f):
        splitLine = line.split()
        word = splitLine[0]
        embedding = np.array([float(val) for val in splitLine[1:]])
        model[word] = embedding
    print ("Done.",len(model)," words loaded!")
    return model
model = loadGloveModel('/content/drive/My Drive/Glove_42B/glove.42B.300d.txt')
# get list of lists for every sentence words
words = []
for line in tqdm(X_train['question_content']):
    words.append(line.split())
len(words)
# get one single list of whole corpus words
Words = []
for line in tqdm(X_train['question_content']):
    Words.extend(line.split())
print("All the words in the courpus", len(Words))
Words = set(Words)
print("The unique words in the courpus", len(Words))
inter_words = set(model.keys()).intersection(Words)
print("The number of words that are present in both glove vectors and our courpus", \
len(inter_words),"(",np.round(len(inter_words)/len(Words)*100,3),"%)")
from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer(min_df=10)
# fit and transform on question content
tfidf.fit_transform(X_train["question_content"])
# save tfidf 
joblib.dump(tfidf,'/content/drive/My Drive/self_case_study1/data/joblib_dumps_glove_model/tfidf1')
len(tfidf_words)
words_courpus = {}
words_glove = set(model.keys())
for i in Words:
    if i in words_glove:
        words_courpus[i] = model[i]
print("word 2 vec length", len(words_courpus))
# stronging variables into pickle files python: http://www.jessicayung.com/how-to-use-pickle-to-save-and-load-variables-in-python/

import pickle
with open('/content/drive/My Drive/self_case_study1/data/joblib_dumps_glove_model/glove_vectors', 'wb') as f:
    pickle.dump(words_courpus, f)<jupyter_output>word 2 vec length 18546<jupyter_text>load glove vectors<jupyter_code>with open('/content/drive/My Drive/self_case_study1/data/joblib_dumps_glove_model/glove_vectors', 'rb') as f:
    model = pickle.load(f)
    glove_words = set(model.keys())
# we are converting a dictionary with word as a key, and the idf as a value
tfidf = joblib.load('/content/drive/My Drive/self_case_study1/data/joblib_dumps_glove_model/tfidf1')
dictionary = dict(zip(tfidf.get_feature_names(), list(tfidf.idf_)))
tfidf_words = set(tfidf.get_feature_names())

tfidf_w2v_vector = [] # the weighted w2v for each sentence is stored in this list
for sentence in tqdm(words): # for each sentence words list
    vector = np.zeros(300) # as word vectors length is 300
    tf_idf_weight =0; # num of words with a valid vector in the sentence
    for word in sentence : # for each word in a review/sentence
        if (word in glove_words ) and (word in tfidf_words):
            vec = model[word] # getting the vector for each word
            # multiply idf value of word with tf of word
            tf_idf = dictionary[word]*(sentence.count(word)/len(sentence))
            vector += (vec * tf_idf) # calculating tfidf weighted w2v
            tf_idf_weight += tf_idf
    if tf_idf_weight != 0:
        vector /= tf_idf_weight
    tfidf_w2v_vector.append(vector)
# save tfidf w2v vectors
joblib.dump(tfidf_w2v_vector,'/content/drive/My Drive/self_case_study1/data/joblib_dumps_glove_model/tfidf_w2v_vector_glove')
# load tfidf w2v vectors
tfidf_w2v_vector = joblib.load('/content/drive/My Drive/self_case_study1/data/joblib_dumps_glove_model/tfidf_w2v_vector_glove')
# query question
text = 'how to make dictionary in python language'
text
# get glove words
#glove_words = set(model.keys())
# get tfidf words
#tfidf_words = set(tfidf.get_feature_names())
# dictionary of tfidf
#dictionary = dict(zip(tfidf.get_feature_names(), list(tfidf.idf_)))


def find_similarity(question,top_n):
    ''' This function will find top similar result for given query'''
    start = time.time()
    # initialize  vector for user query
    main_vec = np.zeros(300)
    # initialize tfidf weight
    weight_sum = 0
    # preprocess question
    text = preprocess_title(question)
    #splitting the sentence
    text_list = list(text.split())
    for word in text_list:
        #finding if word is present in tfidf and in w2v words
        if word in tfidf_words and word in glove_words :
            #finding vector of word from glove model
            vect = model[word]
            #compute tfidf
            tf_idf = dictionary[word]*(text_list.count(word)/len(text_list)) 
            # adding vector * tfidf to main_vec
            main_vec+= (vect*tf_idf)
            # summing tfidf values
            weight_sum += tf_idf
    if weight_sum !=0:
        # devide by weight_sum
        main_vec /= weight_sum
    # find cosine similarity
    similarities =  cosine_similarity((main_vec).reshape(1, -1), Y=tfidf_w2v_vector, dense_output=True)
    # sort similarities 
    #print(similarities[0])
    
    sort = np.argsort(similarities[0])
    # get top similarity indices  in descending order
    similarity_index = np.array(list(reversed(sort)))
    # finad top n similarities
    top_similarity_index = similarity_index[:top_n]
    # print top similarity values
    print('Top cosine similarities are ======>',similarities[0][top_similarity_index])
    # get original title of similar questions
    similar_questions = X_train['title'][top_similarity_index]
    #print(similar_questions)
    
    similar_question_list = []
    for q in similar_questions:
        similar_question_list.append(q)
        #print(q)
    total_time = (time.time() - start)
    print('Total time ===========> ',total_time)
    return similar_question_list

top_10_questions = find_similarity(text,10)
print('Top 10 similar questions Using weighted TFIDF and Glove vectors')
print('='*100)
print('\t')
for i,line in enumerate(top_10_questions):
    print('Question {} ===> {}'.format(i+1,line))
    print('='*100)<jupyter_output>Top 10 similar questions Using weighted TFIDF and Glove vectors
====================================================================================================
	
Question 1 ===> Handling dictionary object in python
====================================================================================================
Question 2 ===> Need help in translating language function in wordpress
====================================================================================================
Question 3 ===> Creating a words game in Java(Issues with dictionary File organisation)
====================================================================================================
Question 4 ===> Iteration of a nested dictionary in Python 3
====================================================================================================
Question 5 ===> Expressjs language library
====================================================================================================
Question 6 [...]<jupyter_text>Model-2 : train word2vec using gensim<jupyter_code>from gensim.models import Word2Vec
# train Word2vec model with vector size 60
w2v_model = Word2Vec(words,size=60,min_count=10, workers=-1,iter=50)
word_vectors = w2v_model.wv
# shape of w2v vectors
word_vectors.vectors.shape
# save word2vec model
joblib.dump(w2v_model,'/content/drive/My Drive/self_case_study1/data/joblib_dumps/gensim_w2v_model')
# get word2vec vocab
w2v_words = list(w2v_model.wv.vocab)
print("sample words =====>", w2v_words[0:10])
# number of unique word in word2vec vocab
print('Number of unique words in Word2vec ==> ',len(w2v_words))
# Words list have all the words which are in corpus
print("All the words in the courpus", len(Words))
# unique words from whole X_train
Words = set(Words)
print("The unique words in the courpus", len(Words))
# common word of w2v_words and Words
inter_words = set(w2v_words).intersection(Words)
print("The number of words that are present in both glove vectors and our courpus", \
len(inter_words),"(",np.round(len(inter_words)/len(Words)*100,3),"%)")<jupyter_output>All the words in the courpus 19725
The unique words in the courpus 19725
The number of words that are present in both glove vectors and our courpus 16226 ( 82.261 %)<jupyter_text>**TF-IDF weighted W2V using gensim**<jupyter_code># dictionary in which word as key and idf as value
dictionary = dict(zip(tfidf.get_feature_names(),tfidf.idf_))
# check for random word
dictionary['samsung']
# number of words in dictionary
len(dictionary)
# get tfidf words
tfidf_words = set(tfidf.get_feature_names())
tfidf_w2v_vectors_gensim = [] # the weighted w2v for each sentence is stored in this list
for sentence in tqdm(words): # for each sentence
    vector = np.zeros(60) # as word vectors length is 60
    tf_idf_weight =0; # num of words with a valid vector in the sentence
    for word in sentence : # for each word in a review/sentence
        if (word in w2v_words) and (word in tfidf_words):
            vec = w2v_model.wv[word] # getting the vector for each word
            # multiply idf value of word with tf of word
            tf_idf = dictionary[word]*(sentence.count(word)/len(sentence))
            vector += (vec * tf_idf) # calculating tfidf weighted w2v
            tf_idf_weight += tf_idf
    if tf_idf_weight != 0:
        vector /= tf_idf_weight
    tfidf_w2v_vectors_gensim.append(vector)

len(tfidf_w2v_vectors_gensim)
# save tfidf w2v vectors using gensim
joblib.dump(tfidf_w2v_vectors_gensim,'/content/drive/My Drive/self_case_study1/data/joblib_dumps/tfidf_w2v_vectors_gensim')
# total number of vectors 
print('Total number of vectors = ',len(tfidf_w2v_vectors_gensim))
# length of single vector
print('Size of single vector = ',len(tfidf_w2v_vectors_gensim[0]))<jupyter_output>Total number of vectors =  212583
Size of single vector =  60<jupyter_text>**Load all saved files**<jupyter_code>tfidf_gensim = joblib.load('/content/drive/My Drive/self_case_study1/data/joblib_dumps_glove_model/tfidf1')
w2v_model_gensim = joblib.load('/content/drive/My Drive/self_case_study1/data/joblib_dumps/gensim_w2v_model')
tfidf_w2v_vectors_gensim = joblib.load('/content/drive/My Drive/self_case_study1/data/joblib_dumps/tfidf_w2v_vectors_gensim')
# w2vec words vocabulary
w2v_words = list(w2v_model_gensim.wv.vocab)
# tfidf words
tfidf_words = set(tfidf_gensim.get_feature_names())
# dictionary of tfidf and idf values
dictionary = dict(zip(tfidf_gensim.get_feature_names(),tfidf_gensim.idf_))
text = 'how to make dictionary in python language'
def find_similarity_gensim_trained(question,top_n):
    ''' This function will find top similar result for given query using gensim w2v'''
    start = time.time()
    # initialize  vector for user query
    main_vec = np.zeros(60)
    # initialize tfidf weight
    weight_sum = 0
    # preprocess question
    text = preprocess_title(question)
    #splitting the sentence
    text_list = list(text.split())
    for word in text_list:
        #finding if word is present in tfidf and in w2v words
        if word in tfidf_words and word in w2v_words :
            #finding vector of word from glove model
            vect = w2v_model_gensim[word]
            #compute tfidf
            tf_idf = dictionary[word]*(text_list.count(word)/len(text_list)) 
            # adding vector * tfidf to main_vec
            main_vec+= (vect*tf_idf)
            # summing tfidf values
            weight_sum += tf_idf
    if weight_sum !=0:
        # devide by weight_sum
        main_vec /= weight_sum
    # find cosine similarity
    # tfidf word2vec have trained using gensim
    similarities =  cosine_similarity((main_vec).reshape(1, -1), Y=tfidf_w2v_vectors_gensim, dense_output=True)
    # sort similarities 
    sort = np.argsort(similarities[0])
    # get top similarity indices  in descending order
    similarity_index = np.array(list(reversed(sort)))
    # find top n similarities
    top_similarity_index = similarity_index[:top_n]
    # print top similarity values
    print('Top cosine similarities are ======>',similarities[0][top_similarity_index])
    similar_questions = X_train['title'][top_similarity_index]
    #print(similar_questions)
    similar_question_list = []
    for q in similar_questions:
        similar_question_list.append(q)
        #print(q)
    total_time = (time.time() - start)
    print('\t')
    print('Total time ===========> ',total_time)
    return similar_question_list
top_10_similar_q = find_similarity_gensim_trained(text,10)
print('Top 10 similar questions Using weighted TFIDF and gensim Word2Vec')
print('='*100)
print('\t')
for i,line in enumerate(top_10_similar_q):
    print('Question {} ===> {}'.format(i+1,line))
    print('='*100)<jupyter_output>Top 10 similar questions Using weighted TFIDF and gensim Word2Vec
====================================================================================================
	
Question 1 ===> write a dictionary of dictionaries (with unknown size) as a matrix
====================================================================================================
Question 2 ===> how to order a dictionary based on key by daywise in python
====================================================================================================
Question 3 ===> Handling dictionary object in python
====================================================================================================
Question 4 ===> import nested dictionary in pandas (from yaml)
====================================================================================================
Question 5 ===> How to get a dictionary source?
====================================================================================================
Que[...]<jupyter_text>Model-3 : Doc2VecReference : https://radimrehurek.com/gensim/auto_examples/tutorials/run_doc2vec_lee.htmlsphx-glr-auto-examples-tutorials-run-doc2vec-lee-py**There are two implementations:**    1. Paragraph Vector - Distributed Memory (PV-DM)    2. Paragraph Vector - Distributed Bag of Words (PV-DBOW)<jupyter_code>def read_corpus(filename, tokens_only=False):
    for i, line in tqdm(enumerate(filename)):
        tokens = gensim.utils.simple_preprocess(line)
        if tokens_only:
            yield tokens
        else:
            # For training data, add tags
            yield gensim.models.doc2vec.TaggedDocument(tokens, [i])

# train corpus
train_corpus = list(read_corpus(X_train['question_content']))
# check for first 5 questions
train_corpus[:5]
X_train.shape<jupyter_output><empty_output><jupyter_text>Training the model<jupyter_code># define model
model = gensim.models.doc2vec.Doc2Vec(vector_size=100, min_count=10, epochs=60,workers=-1)
# build vocab
model.build_vocab(train_corpus)
#Essentially, the vocabulary is a dictionary (accessible via model.wv.vocab) of all of the unique words extracted 
#from the training corpus along with the count (e.g., model.wv.vocab['penalty'].count for counts for the word penalty).
# count of word python in vocab
print(model.wv.vocab['python'])
print(model.corpus_count)
# train model
model.train(train_corpus, total_examples=model.corpus_count, epochs=model.epochs)
# save model
joblib.dump(model,'/content/drive/My Drive/self_case_study1/data/doc2vec/doc2vecmodel')
# load model 
model = joblib.load('/content/drive/My Drive/self_case_study1/data/doc2vec/doc2vecmodel')
# shape of the vector
model.docvecs[0].shape
test_corpus = list(read_corpus('how to make dictionary in python', tokens_only=True))
vector = model.infer_vector(test_corpus[0])
simi = (model.docvecs.most_similar([vector], topn=10))#this algorithm by default calculates cosine similarity as a distance metric
print(simi)
sim_doc2vec = []
for i in range(len(simi)):
  idx = (simi[i][0])
  se = X_train["title"][idx]
  print(se)
  print('='*100)
  #sim_doc2vec.append(se)<jupyter_output>Multiple Parameters with POST and MVC
====================================================================================================
Center menu items in responsive navbar
====================================================================================================
pandas write dataframe to parquet format with append
====================================================================================================
Load VideoJS with AJAX
====================================================================================================
How to prepopulate form in struts2 from database values
====================================================================================================
Jqgrid - How to perform search locally for entire data set,without considering the pagination
====================================================================================================
How does rails activerecord where clause accepts dynamic parameter?
=======================[...]<jupyter_text>Model-4 : Universal sentence encoderReference : https://tfhub.dev/google/universal-sentence-encoder/4<jupyter_code>#tf.keras.backend.clear_session()
import tensorflow_hub as hub

import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()


tf.logging.set_verbosity(tf.logging.ERROR)
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")
def embed_text(sentences):
    ''' This function will convert sentence into vector using universal sentence encoder'''
    # Reduce logging output.
    tf.logging.set_verbosity(tf.logging.ERROR)

    with tf.Session() as session:
        session.run([tf.global_variables_initializer(), tf.tables_initializer()])
        # create vectors from sentences
        embeddings = session.run(embed(sentences))
        # return  embeddings
        return embeddings<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
Instructions for updating:
non-resource variables are not supported in the long term<jupyter_text>**Now we will doing sentence embedding in 4 parts because i have around 2 lacks rows so my colab will crash so i am doing sentence embedding in 4 parts and after that i will concatenate**<jupyter_code>start = time.time()
question_embedding1 = embed_text(X_train["question_content"][:50000])
print('Total time ====> ',(time.time() - start))
start = time.time()
question_embedding2 = embed_text(X_train["question_content"][50000:100000])
print('Total time ====> ',(time.time() - start))
start = time.time()
question_embedding3 = embed_text(X_train["question_content"][100000:150000])
print('Total time ====> ',(time.time() - start))
start = time.time()
question_embedding4 = embed_text(X_train["question_content"][150000:212583])
print('Total time ====> ',(time.time() - start))
print('Shape of question_embedding1 is = ',question_embedding1.shape)
print('Shape of question_embedding1 is = ',question_embedding2.shape)
print('Shape of question_embedding1 is = ',question_embedding3.shape)
print('Shape of question_embedding1 is = ',question_embedding4.shape)
# concatenate all embeddings on axis 0 (vertically)
question_embeddings = np.concatenate((question_embedding1,question_embedding2,question_embedding3,question_embedding4),axis=0)
print('Shape of final question embedding is ===> ',question_embeddings.shape)
# store question embeddings using joblib
joblib.dump(question_embedddings,'/content/drive/My Drive/self_case_study1/data/universal_embed/universal_question_embedding')
# load question embeddings
question_embeddings = joblib.load('/content/drive/My Drive/self_case_study1/data/universal_embed/universal_question_embedding')<jupyter_output><empty_output><jupyter_text>Find similarity for user query<jupyter_code>def find_similarity_universal_sentence_encoder(question,top_n):
    ''' This function will find top similar result for given query using universal sentence encoder'''
    start = time.time()
    # initialize  vector for user query
    query_vector = embed_text([question])
    similarities =  cosine_similarity((query_vector).reshape(1, -1), Y=question_embedddings, dense_output=True)
    # sort similarities 
    sort = np.argsort(similarities[0])
    # get top similarity indices  in descending order
    similarity_index = np.array(list(reversed(sort)))
    # find top n similarities
    top_similarity_index = similarity_index[:top_n]
    # print top similarity values
    print('Top cosine similarities are ======>',similarities[0][top_similarity_index])
    # get similar questions title
    similar_questions = X_train['title'][top_similarity_index]
    # create similar questions title list
    similar_question_list = []
    for q in similar_questions:
        similar_question_list.append(q)
    total_time = (time.time() - start)
    print('\t')
    print('Total time ===========> ',total_time)
    return similar_question_list
query = 'how to make dictionary in python'
USE = find_similarity_universal_sentence_encoder(query,top_n=10)
print('Top 10 similar questions Using Universal sentence encoder')
print('='*100)
print('\t')
for i,line in enumerate(USE):
    print('Question {} ===> {}'.format(i+1,line))
    print('='*100)<jupyter_output>Top 10 similar questions Using Universal sentence encoder
====================================================================================================
	
Question 1 ===> Python - Unnest List of Dictionaries
====================================================================================================
Question 2 ===> How to convert string to dictionary into python
====================================================================================================
Question 3 ===> How to convert a list into a dictionary in python
====================================================================================================
Question 4 ===> How to ignore \ before " in simplejson?
====================================================================================================
Question 5 ===> How to use python dict in my write order
====================================================================================================
Question 6 ===> Python converting a li[...]<jupyter_text>Model-5 : Variational Auto EncoderReference : 1. https://s4sarath.github.io/2016/11/23/variational_autoenocder_for_Natural_Language_Processing  2. https://medium.com/@adriensieg/text-similarities-da019229c894<jupyter_code>import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
os.chdir('/content/drive/My Drive/self_case_study1/data/VAE')
from text_loader_utils import TextLoader
A = TextLoader(list(X_train['question_content']) , min_count = 10)
# save to drive
joblib.dump(A,'/content/drive/My Drive/self_case_study1/data/VAE/textloaded.pickle')
# load from drive
text_loaded = joblib.load('/content/drive/My Drive/self_case_study1/data/VAE/textloaded.pickle')
len(text_loaded.vocab)
batch_size = 100
# restricting memory usage, TensorFlow is greedy and will use all memory otherwise
gpu_opts = tf.GPUOptions(per_process_gpu_memory_fraction=0.9)
sess = tf.InteractiveSession(config=tf.ConfigProto(gpu_options=gpu_opts))
from variational_model import NVDM
from tf_common_utils import load_model , save_model
np.random.seed(0)
tf.set_random_seed(0)

def xavier_init(fan_in , fan_out, constant=1): 
    """ Xavier initialization of network weights"""
    # https://stackoverflow.com/questions/33640581/how-to-do-xavier-initialization-on-tensorflow
#     fan_in = in_and_out[0]
#     fan_out = in_and_out[1]
    low = -constant*np.sqrt(6.0/(fan_in + fan_out)) 
    high = constant*np.sqrt(6.0/(fan_in + fan_out))
    return tf.random_uniform((fan_in, fan_out), 
                             minval=low, maxval=high, 
                             dtype=tf.float32)
vae = NVDM(sess , len(text_loaded.vocab), 10, [100 , 100] , 
           transfer_fct=tf.nn.tanh , output_activation=tf.nn.softmax,
           batch_size=100, initializer=xavier_init)
vae.start_the_model()
save_model(vae)
# load_model will restore all state of model and checkpoints
load_model(model=vae)
# create embedding matrix of mean
embedding_matrix = vae.SESS.run(vae.Weights_generator['out_mean'])
print('Shape of embedding matrix ====>',embedding_matrix.shape)
from sklearn.preprocessing import normalize
embedding_matrix = embedding_matrix.transpose()
print('Shape of transposed embedding matrix',embedding_matrix.shape)
# normalise embedding matrix
embedding_matrix = normalize(embedding_matrix,axis=1) 
embedding_matrix.shape
def find_similar(embedded_mat , query_vec,top_n):
    # find similarities
    similarity = np.dot(embedded_mat , query_vec)
    # sort similarities index in ascending order
    indices_asc = np.argsort(similarity)
    # get top similarity indices  in descending order
    similarity_index = np.array(list(reversed(indices_asc)))
    # find top n similarities
    top_similarity_index = similarity_index[:top_n]
    # print top similarity values
    top_k_sim = similarity[top_similarity_index]
    print('Top cosine similarities are ======>',top_k_sim)
    return  top_similarity_index , top_k_sim
def word_match(norm_matrix , word , vocab, vocab_inverse , top_n = 10):
    # index of query word
    idx = vocab[word]
    # find top n index and similarity
    top_n_index, top_n_simialrity = find_similar(norm_matrix , norm_matrix[idx],top_n)
    words = [vocab_inverse[i_x_] for i_x_ in top_n_index]
    return list(zip(words , top_n_simialrity))
word_match(embedding_matrix , 'python' , text_loaded.vocab , text_loaded.vocab_inverse)
batch_size = 100
question_content = []
batch_data = text_loaded.get_batch(batch_size)
batch_id = 0
for batch_ in batch_data:

            batch_id += 1
            collected_data = [chunks for chunks in batch_]
            batch_xs , mask_xs , mask_negative  = text_loaded._bag_of_words(collected_data)
            

            feed_dict = {vae.X: batch_xs , vae.dynamic_batch_size:batch_xs.shape[0],
                                            vae.MASK:mask_xs}

            h_batch    = vae.SESS.run(vae.z, feed_dict= feed_dict)
            question_content.extend(h_batch)

question_content = np.array(question_content)
joblib.dump(question_content,'/content/drive/My Drive/self_case_study1/data/VAE/question_content_vec')
question_content = joblib.load('/content/drive/My Drive/self_case_study1/data/VAE/question_content_vec')
question_content[0]
def find_similiar_docs(index, top_n = 10):

    ind , similarities = find_similar(normalize(question_content), normalize(question_content)[index],top_n)
    source = X_train['title'][index]
    similar_results = [X_train['title'][i] for i in ind]
    
    return source , list(zip(similar_results, similarities))
query , results = find_similiar_docs(24)
query
results
print('Top 10 Similar results')
print('\t\t')
for i,que in enumerate(results):
    print('Question {} ====> '.format(i+1),que[0],'  ***Similarity*** ==> {}'.format(que[1]))
    print('='*100)<jupyter_output>Top 10 Similar results
		
Question 1 ====>  Grails using ajax submit a form   ***Similarity*** ==> 1.0
====================================================================================================
Question 2 ====>  ReferenceError when running javascript in a Laravel Blade   ***Similarity*** ==> 0.9382911920547485
====================================================================================================
Question 3 ====>  How to find eclipse cdt version?   ***Similarity*** ==> 0.9235158562660217
====================================================================================================
Question 4 ====>  Excel Selecting a Row   ***Similarity*** ==> 0.9220808744430542
====================================================================================================
Question 5 ====>  replacing the process with file url instead of file upload in php   ***Similarity*** ==> 0.9208033084869385
==========================================================================[...]<jupyter_text>BERT EmbeddingReference : https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2<jupyter_code>import tensorflow_hub as hub
from tensorflow.keras.models import Model
os.chdir('/content/drive/My Drive/self_case_study1/data/BERT')
def create_bert_model(seq_length):
    # clear tensorflow backend
    tf.keras.backend.clear_session()

    # Max sequence length
    max_seq_length = seq_length # Your choice here.
    # input words represented as integer
    input_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,
                                        name="input_word_ids")
    # mask vector for padding
    input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,
                                    name="input_mask")

    #
    segment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32,
                                        name="segment_ids")
    # bert layer
    bert_layer = hub.KerasLayer("https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/2",
                                trainable=True)
    # output from bert layer
    pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])
    # bert model
    bert_model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=pooled_output)
    # return model
    return bert_model

bert_model = create_bert_model(60)
bert_model.summary()
# reference : https://github.com/tensorflow/models/blob/master/official/nlp/bert/tokenization.py
!pip install sentencepiece
from tokenization import FullTokenizer
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
tokenizer = FullTokenizer(vocab_file, do_lower_case)
def tokenize_for_bert(text):
    ''' This function will return tokens as integer encoded and mask tokens and segment'''
    tokens = tokenizer.tokenize(text) # tokenizer will break sentent into tokens
    tokens = tokens[0:max_seq_length-2] # consider tokens only max seq length-2
    # because we have to pad [CLS] at starting of tokens and [SEP] at the end of tokens list
    tokens = ['[CLS]',*tokens,'[SEP]'] # add [CLS] at starting and [SEP] at the end.
    # masked_tokens have 1's only when word is present otherwise 0.
    masked_tokens = list([1]*len(tokens)+[0]*(max_seq_length-len(tokens)))
    # segment tokens
    segmented_tokens = list([0]*max_seq_length)
    # rest tokens means those words which are not present in text 
    rest_tokens = max_seq_length-(len(tokens))
    # pad 0's for rest_tokens
    tokens.extend(['[PAD]' for i in range(rest_tokens)])
    # convert final tokens into indexes
    indexed_tokens = list(tokenizer.convert_tokens_to_ids(tokens))
    # return indexed_tokens, masaked_tokens and segmented tokens 
    return indexed_tokens, masked_tokens, segmented_tokens

# convert train data into tokens, mask and segments for bert input
# initialize token,mask and segment as zeros
X_train_token = np.zeros(shape = (len(X_train),max_seq_length))
X_train_mask = np.zeros(shape = (len(X_train),max_seq_length))
X_train_segment = np.zeros(shape = (len(X_train),max_seq_length))

#iterate through X_train
for i in tqdm(range(len(X_train))):
    tokens, mask, segment = tokenize_for_bert(X_train['question_content'][i])
    X_train_token[i] = tokens
    X_train_mask[i] = mask
    X_train_segment[i] = segment
print('Shape of X_train token',X_train_token.shape)
print('Shape of X_train mask',X_train_mask.shape)
print('Shape of X_train segment',X_train_segment.shape)
# get output from bert model
X_train_pooled = bert_model.predict([X_train_token, X_train_mask,X_train_segment] )<jupyter_output><empty_output><jupyter_text>There are two outputs: a pooled_output of shape [batch_size, 768] with representations for the entire input sequences and a sequence_output of shape [batch_size, max_seq_length, 768] with representations for each input token (in context).**we need pooled_output because it gives vector for entire sentence**<jupyter_code># dump pooled output using joblib
joblib.dump(X_train_pooled,'/content/drive/My Drive/self_case_study1/data/BERT/X_train_pooled.pickle')
# load X_train_pooled output
X_train_pooled = joblib.load('/content/drive/My Drive/self_case_study1/data/BERT/X_train_pooled.pickle')
X_train_pooled.shape<jupyter_output><empty_output><jupyter_text>Find most similar documents<jupyter_code>def bert_similarity(query_text,top_n):
    start = time.time()
    # initialize tokens,mask and segment of query text
    query_tokens = np.zeros(shape = (1,max_seq_length))
    query_mask = np.zeros(shape = (1,max_seq_length))
    query_segment = np.zeros(shape = (1,max_seq_length))
    # get token,mask and segment
    q_token,q_mask,q_segment = tokenize_for_bert(query_text)
    # assign tokens
    query_tokens[0] = q_token
    # assign mask
    query_mask[0] = q_mask
    # assign segment
    query_segment[0] = q_segment
    # pooled output of query text
    query_pooled_output = bert_model.predict([query_tokens,query_mask,query_segment])
    # find cosine similarity
    similarities =  cosine_similarity(query_pooled_output, Y=X_train_pooled, dense_output=True)
    # sort similarities 
    sort = np.argsort(similarities[0])
    # get top similarity indices  in descending order
    similarity_index = np.array(list(reversed(sort)))
    # find top n similarities
    top_similarity_index = similarity_index[:top_n]
    # print top similarity values
    print('Top cosine similarities are ======>',similarities[0][top_similarity_index])
    # get similar questions title
    similar_questions = X_train['title'][top_similarity_index]
    # create similar questions title list
    total_time = (time.time() - start)
    print('\t')
    print('Total time ===========> ',total_time)
    return list(similar_questions)
query = 'how to make dictionary in python language'
similarity = bert_similarity(query,10)
similarity
print('Top 10 similar questions Using BERT embedding ')
print('='*100)
print('\t')
for i,line in enumerate(similarity):
    print('Question {} ===> {}'.format(i+1,line))
    print('='*100)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.16613249094975674, 'fraction_numerical': 0.01616361917498674, 'mean_word_length': 5.496405032953865, 'pattern_counts': {'":': 0, '<': 92, '<?xml version=': 0, '>': 154, 'https://': 9, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '25', 'dir': 'jupyter-structured-clean-dedup', 'id': '335040', 'license': 'MIT', 'n_tokens_mistral': 13932, 'n_tokens_neox': 13002, 'n_words': 3208, 'path': 'Preprocessing_sc1.ipynb', 'repo_name': 'jrderek/Stackoverflow-search', 'size': '43369'}