<jupyter_start><jupyter_text>3.2 Data Cleaning<jupyter_code>DATAPOINTS = 112357#number of datapoints to consider
import pandas as pd
df = pd.read_csv('/content/drive/My Drive/case study data/Copy of file1.csv',nrows = DATAPOINTS)<jupyter_output><empty_output><jupyter_text>3.2.1 finding duplicate rows and removing them<jupyter_code>print("number of duplicated rows",len(df[df.duplicated()]))
#drop duplictes
df.drop_duplicates(keep='first')
df.head()
df.info()
print('~> Total number of questions :\n   {}'.format(len(df)))<jupyter_output>~> Total number of questions :
   112357<jupyter_text>3.2.2 Checking whether there are any rows with null values<jupyter_code>nan_rows = df[df.isnull().any(1)]
print (nan_rows)<jupyter_output>Empty DataFrame
Columns: [Unnamed: 0, title, body]
Index: []<jupyter_text>3.2.3 finding max and min length of questions<jupyter_code>s = df['body'].str.len()
print('Minimum length of the question :',min(s))
print('Maximum length of the question :',max(s))
from wordcloud import WordCloud, STOPWORDS
from nltk.corpus import stopwords
from nltk.stem import SnowballStemmer
from sklearn.manifold import TSNE
import nltk
import matplotlib.pyplot as plt
nltk.download('stopwords')<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.<jupyter_text>3.2.4 most common words<jupyter_code>stopwords1 = set(STOPWORDS)
wc = WordCloud(background_color="black", max_words=len(df),width=1600,
                          height=800, stopwords=stopwords1)
wc.generate(str(df["body"]))
print ("Word Cloud for Question ")
fig = plt.figure(figsize=(15,10))
plt.imshow(wc, interpolation='bilinear')
plt.axis("off")
plt.tight_layout(pad=0)
plt.show()<jupyter_output>Word Cloud for Question<jupyter_text>3.3 Cleaning and preprocessing of Questions   3.3.1 Preprocessing        Separate out code-snippets from Body      Remove Spcial characters from Question title and description (not in code)     Remove stop words (Except 'C')      Remove HTML Tags      Convert all the characters into small letters      Use SnowballStemmer to stem the words<jupyter_code>from bs4 import BeautifulSoup
import re
def striphtml(data):
    soup = BeautifulSoup(data, 'lxml')
    cleantext = soup.get_text()
    soup1 = BeautifulSoup(cleantext, 'html5lib')
    cleantext = soup1.get_text()
    cleantext = re.sub('<.*?>', ' ', str(cleantext))  
    cleantext = re.sub('\\n', ' ', str(cleantext))  
    cleantext=re.sub('\sdiv\s', ' ', str(cleantext), flags=re.MULTILINE|re.DOTALL)

    return cleantext
stop_words = set(stopwords.words('english'))
stemmer = SnowballStemmer("english")
print(df['body'][1])
striphtml(df['body'][1].encode('utf-8'))
from tqdm import tqdm
start = datetime.now()
preprocessed_data_list=[]
preprocessed_data_list_code = []
questions_with_code=0
len_pre=0
len_post=0
questions_proccesed = 0
for row in tqdm(range(len(df))):
    is_code = 0
    title, question = df['title'][row], df['body'][row]
    if '<code>' in question:
        questions_with_code+=1
        is_code = 1
    x = len(question)+len(title)

    len_pre+=x

    code = str(re.findall(r'<code>(.*?)</code>', question, flags=re.DOTALL))
    preprocessed_data_list_code.append(code)
    question = re.sub('<code>(.*?)</code>', '', question, flags=re.MULTILINE|re.DOTALL)#keeping the code content
    question=striphtml(question.encode('utf-8'))
    question=str(title)+" "+str(question)

    question=re.sub(r'[^A-Za-z]+',' ',question)
    words=word_tokenize(str(question.lower()))

    #Removing all single letter and and stopwords from question exceptt for the letter 'c'
    question=' '.join(str(stemmer.stem(j)) for j in words if j not in stop_words and (len(j)!=1 or j=='c'))
    preprocessed_data_list.append(question)
    len_post+=len(question)

    questions_proccesed += 1

no_dup_avg_len_pre=(len_pre*1.0)/questions_proccesed
no_dup_avg_len_post=(len_post*1.0)/questions_proccesed

print( "\nAvg. length of questions(Title+Body) before processing: %d"%no_dup_avg_len_pre)
print( "\nAvg. length of questions(Title+Body) after processing: %d"%no_dup_avg_len_post)
print ("\nPercent of questions containing code: %d"%((questions_with_code*100.0)/questions_proccesed))
print("\nTime taken to run this cell :", datetime.now() - start)
print(len(preprocessed_data_list))
print(len(preprocessed_data_list_code))<jupyter_output>112357
112357<jupyter_text>3.3.2 sample output of Preprocessing<jupyter_code>k=1
print(df['body'][k])
print('='*100)
print(preprocessed_data_list[k])<jupyter_output><p>I have an absolutely positioned <code>div</code> containing several children, one of which is a relatively positioned <code>div</code>. When I use a <code>percentage-based width</code> on the child <code>div</code>, it collapses to <code>0 width</code> on IE7, but not on Firefox or Safari.</p>

<p>If I use <code>pixel width</code>, it works. If the parent is relatively positioned, the percentage width on the child works.</p>

<ol>
<li>Is there something I'm missing here?</li>
<li>Is there an easy fix for this besides the <code>pixel-based width</code> on the
child?</li>
<li>Is there an area of the CSS specification that covers this?</li>
</ol>

percentag width child element absolut posit parent internet explor absolut posit contain sever children one relat posit use child collaps ie firefox safari use work parent relat posit percentag width child work someth miss easi fix besid child area css specif cover<jupyter_text>3.3.3 converting preprocessed data into dataframe and storing it as csv<jupyter_code>preprocessed = pd.DataFrame(zip(df['title'],df['body'],preprocessed_data_list,preprocessed_data_list_code),columns= ["title","content","questions","code"])
preprocessed.head()
preprocessed.shape
preprocessed.to_csv("/content/drive/My Drive/case study data/preprocessed_1.5_lacs.csv",index=False)<jupyter_output><empty_output><jupyter_text>4. Machine Learning Models  4.1 Split the data into test and train<jupyter_code>import pandas as pd

preprocessed = pd.read_csv("/content/drive/My Drive/case study data/preprocessed_1.5_lacs.csv")
preprocessed.shape
print(preprocessed.index.values)
from sklearn.model_selection import train_test_split
train,test = train_test_split(preprocessed,test_size=0.05)
print(len(train))
print(len(test))
train.reset_index(inplace = True)
test.reset_index(inplace = True)
print(train.index)
print(test.index)
#finding length of each sentences
import numpy as np
def get_wordlen(x):
    return len(x.split())

preprocessed['len'] = preprocessed.questions.apply(get_wordlen)
#reviews = reviews[reviews.len<50]
leng = (preprocessed["len"].values)
print('95%of the questions contains{} words'.format(np.percentile(leng,q = 95)))
max_seq_length = 200
#all imports
import pandas as pd
import tensorflow as tf
import tensorflow_hub as hub
from tensorflow.keras.models import Model<jupyter_output><empty_output><jupyter_text>4.2 Model-1 : BERT   For this case sudy, we are using BERT uncased Base model. It uses L=12 hidden layers (i.e., Transformer blocks), a hidden size of H=768, and A=12 attention heads.  4.2.1 loading the model<jupyter_code>## Loading the Pretrained Model from tensorflow HUB
tf.keras.backend.clear_session()

# maximum length of a seq in the data we have, for now i am making it as 200. You can change this
max_seq_length = 200

#BERT takes 3 inputs

#this is input words. Sequence of words represented as integers
input_word_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32, name="input_word_ids")

#mask vector if you are padding anything
input_mask = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32, name="input_mask")

#segment vectors. If you are giving only one sentence for the classification, total seg vector is 0. 
#If you are giving two sentenced with [sep] token separated, first seq segment vectors are zeros and 
#second seq segment vector are 1's
segment_ids = tf.keras.layers.Input(shape=(max_seq_length,), dtype=tf.int32, name="segment_ids")

#bert layer 
bert_layer = hub.KerasLayer("https://tfhub.dev/tensorflow/bert_en_uncased_L-12_H-768_A-12/1", trainable=False)
pooled_output, sequence_output = bert_layer([input_word_ids, input_mask, segment_ids])

#Bert model
#We are using only pooled output not sequence out. 
#If you want to know about those, please read https://www.kaggle.com/questions-and-answers/86510
bert_model = Model(inputs=[input_word_ids, input_mask, segment_ids], outputs=pooled_output)<jupyter_output><empty_output><jupyter_text>4.2.2 summary of the model<jupyter_code>bert_model.summary()
bert_model.output<jupyter_output><empty_output><jupyter_text>4.2.3 preprocessing the text<jupyter_code>#getting Vocab file
vocab_file = bert_layer.resolved_object.vocab_file.asset_path.numpy()
do_lower_case = bert_layer.resolved_object.do_lower_case.numpy()
print(do_lower_case)
#import tokenization - We have given tokenization.py file
!pip install sentencepiece 
from tokenization import FullTokenizer
tokenizer = FullTokenizer(vocab_file,do_lower_case=do_lower_case)
def tok(text):
  tokens = tokenizer.tokenize(text)#tokenizing the input
  tokens = tokens[0:max_seq_length-2]
  tokens = ['[CLS]',*tokens,'[SEP]']#adding tokens at the START and END
  X_train_mask = list([1]*len(tokens)+[0]*(max_seq_length-len(tokens)))#creating mask tokens by putting zeros where there are not any tokens
  X_train_segment = list([0]*max_seq_length)#creating segment
  remain = max_seq_length-(len(tokens))
  tokens.extend(['[PAD]' for i in range(remain)])#padding the tokens
  X_train_tokens = list(tokenizer.convert_tokens_to_ids(tokens))#tokenizing the created tokens into ids
  return X_train_tokens,X_train_mask,X_train_segment<jupyter_output><empty_output><jupyter_text>4.2.2.1 converting train data into tokens,mask and segments<jupyter_code>from tqdm import tqdm
train_tokens = np.zeros(shape = (len(train),max_seq_length))#array for tokens
train_mask = np.zeros(shape = (len(train),max_seq_length))#array for mask
train_segment = np.zeros(shape = (len(train),max_seq_length))#array for segments
for rows in tqdm(range(len(train))):
  tokens,mask,segment = tok(train["questions"][rows])
  train_tokens[rows] = tokens
  train_mask[rows] = mask
  train_segment[rows] = segment<jupyter_output>100%|██████████| 475000/475000 [08:47<00:00, 900.87it/s]<jupyter_text>4.2.2.2 converting test data into tokens,mask and segments<jupyter_code>test_tokens = np.zeros(shape = (len(test),max_seq_length))
test_mask = np.zeros(shape = (len(test),max_seq_length))
test_segment = np.zeros(shape = (len(test),max_seq_length))

for rows in tqdm(range(len(test))):
  tokens,mask,segment = tok(test["questions"][rows])
  test_tokens[rows] = tokens
  test_mask[rows] = mask
  test_segment[rows] = segment<jupyter_output>100%|██████████| 25000/25000 [00:27<00:00, 903.03it/s]<jupyter_text>4.2.3 predicting the word embedding<jupyter_code># get the train output, BERT model will give one output so save in
train_pooled_output = bert_model.predict([train_tokens, train_mask,train_segment] )
import pickle

pickle.dump(train_pooled_output,open('/content/drive/My Drive/case study data/train_pooled_output.pkl','wb'))
# get the test output, BERT model will give one output so save in
test_pooled_output =bert_model.predict([ test_tokens, test_mask, test_segment] )
import pickle
pickle.dump(test_pooled_output,open('/content/drive/My Drive/case study data/test_pooled_output.pkl','wb'))
import pickle
train_pooled_output = pickle.load(open('/content/drive/My Drive/case study data/train_pooled_output.pkl','rb'))
test_pooled_output = pickle.load(open('/content/drive/My Drive/case study data/test_pooled_output.pkl','rb'))
test_pooled_output.shape<jupyter_output><empty_output><jupyter_text>4.2.4 finding most similar document<jupyter_code>import numpy as np
def most_sim__doc(text):
  pred_tokens = np.zeros(shape = (1,max_seq_length))
  pred_mask = np.zeros(shape = (1,max_seq_length))
  pred_segment = np.zeros(shape = (1,max_seq_length))

  tokens,mask,segment = tok(text)
  pred_tokens[0] = tokens
  pred_mask[0] = mask
  pred_segment[0] = segment
  pooled_output = bert_model.predict([pred_tokens,pred_mask,pred_segment])
  similarity =  cosine_similarity(pooled_output, Y=train_pooled_output, dense_output=True)
  #print('query :\n',train["content"][train_row])
  most_common = np.argsort(similarity[0])
  s = most_common[-5:]
  print(s)
  rev = np.flip(s)
  #print(similarity[train_row][rev])
  se = train["content"][rev].values
  lst = ["BERT"]
  for i in se:
    i = striphtml(i)

    sub_string = i.split()[:50]
    lst.append(" ".join(sub_string))
  return (lst)
l1 = most_sim__doc('what is superclass in object orient programming?')
print(l1)<jupyter_output><empty_output><jupyter_text>4.3 Model-2 : DOC2VEC MODEL 4.3.1 preprocessing the sentences<jupyter_code>#https://radimrehurek.com/gensim/auto_examples/tutorials/run_doc2vec_lee.html#sphx-glr-download-auto-examples-tutorials-run-doc2vec-lee-py

import smart_open
import os
import gensim
def read_corpus(fname, tokens_only=False):
        for i, line in enumerate(fname):
            tokens = gensim.utils.simple_preprocess(line)#preprocess the sentence
            if tokens_only:#for test data yield only tokens
                yield tokens
            else:
                # For training data, add tags
                yield gensim.models.doc2vec.TaggedDocument(tokens, [i])

train_corpus = list(read_corpus((train["questions"])))

train_corpus[5]
gensim.models.doc2vec.TaggedDocument.__doc__<jupyter_output><empty_output><jupyter_text>4.3.2 defining the model<jupyter_code>from gensim.models.doc2vec import Doc2Vec
model = Doc2Vec(vector_size=30,alpha=0.025,min_alpha=0.00025,min_count=20, epochs=50, workers=-1,dm =1)#
#dm defines the training algorithm. If dm=1 means ‘distributed memory’ (PV-DM) and dm =0 means ‘distributed bag of words’ (PV-DBOW). Distributed Memory model preserves the word order in a document whereas Distributed Bag of words just uses the bag of words approach, which doesn’t preserve any word order.<jupyter_output><empty_output><jupyter_text>4.3.3 building the vocabulary<jupyter_code>model.build_vocab(train_corpus)
train_corpus[:2]<jupyter_output><empty_output><jupyter_text>4.3.4 training the model<jupyter_code>model.train(train_corpus, total_examples=model.corpus_count,epochs=model.epochs)
model.docvecs[2]<jupyter_output><empty_output><jupyter_text>4.3.1 finding the most similar doc<jupyter_code>test_corpus = list(read_corpus(('what is superclass in object orient programming','what is superclass in object orient programming'), tokens_only=True))
vector = model.infer_vector(test_corpus[0])
simi = (model.docvecs.most_similar([vector], topn=5))#this algorithm by default calculates cosine similarity as a distance metric
print(simi)
sim_doc2vec = ["DOC2VEC"]
for i in range(len(simi)):
  idx = (simi[i][0])
  se = train["content"][idx].split()[:50]
  sim_doc2vec.append(" ".join(se))<jupyter_output><empty_output><jupyter_text>4.3.5 displaying result in pretty table<jupyter_code>from prettytable import PrettyTable
x = PrettyTable(['model','TOP result1','TOP result2','TOP result3','TOP result4','TOP result5'])
x.align = "l"
#x.add_row(l1)
x.add_row(sim_doc2vec)
print("Qurey : what is superclass in object orient programming" )
print(x)
Qurey : what is superclass in object orient programming
+---------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| model   | TOP result1                                                                                                                                                                                                                                                                                                                                                                                  | TOP result2                                                                                                                                                                                                                                                                                                                                                    | TOP result3                                                                                                                                                                                                                                                                                                         | TOP result4                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | TOP result5                                                                                                                                                                                                                                                                                           |
+---------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| BERT    | Custom view transition in OpenGL ES<p>I'm trying to create a custom transition, to serve as a replacement for a default transition you would get here, for example:</p> <pre><code>[self.navigationController pushViewController:someController animated:YES]; </code></pre> <p>I have prepared an OpenGL-based view that performs an effect on some static texture mapped to a plane (let's | Dynamic contact information data/design pattern: Is this in any way feasible?<p>I'm currently working on a web business application that has many entities (people,organizations) with lots of contact information ie. multiple postal addresses, email addresses, phone numbers etc. </p> <p>At the moment the database schema is such that persons table has | Databinding with Silverlight<p>If I want to bind a collection to a some form of listing control in Silverlight. Is the only way to do it so make the underlying objects in the collection implement INotifyPropertyChanged and for the collection to be an Observablecollection?</p> <p>If I was using some sort of | SharePoint Default Styles<p>I'm building a custom web part for SharePoint and I'm trying to use the default styles so when people theme the site, it will theme with it.</p> <p>I've found a couple of decent sites that show the different styles like: <a href="http://www.sharepointcustomization.com/resources/tipstricks/wss_cssguide.htm" rel="nofollow noreferrer">http://www.sharepointcustomization.com/resources/tipstricks/wss_cssguide.htm</a></p> <p>but I'm looking for | passing void to a generic class<p>I'm trying to create a form that will animate something while processing a particular task (passed as a delegate to the constructor). It's working fine, but the problem I'm having is that I can't instantiate a copy of my generic class if the particular method |
| DOC2VEC | What is software development at your company really like (methodologies, tools, ...)?<p>Since I've started my first job as a professional software developer about two years ago, I've read many articles about commonly accepted methodologies (e.g. Scrum, XP), technologies (e.g. EJB, Spring), techniques (e.g. TDD, code reviews), tools (bug tracking, wikis)                          | Best reference / crib sheet for AWK<p>In a series of similar questions, what is the best AWK reference you've ever seen? </p> <p>If there isn't really one (I've yet to find the grail), perhaps we could compile one in a separate question.</p>                                                                                                              | control lost focus event when using keyboard shortcut<p>For both .NET Winforms and Windows Presentation Foundation, if I have a text box that the user has just entered text into, and a button, if the user clicks the button the "LostFocus" event fires before the button click event fires. However if          | Best free Java .class viewer?<p>I've used <a href="http://members.fortunecity.com/neshkov/dj.html" rel="noreferrer">DJ Java Decompiler</a>, which has a handy GUI, but it seems as if the latest version is only a trial and forces you to purchase the software after some period of days (I recall using an earlier free version about a year                                                                                                                                       | How do I turn a python program into an .egg file?<p>How do I turn a python program into an .egg file?</p>                                                                                                                                                                                             |
+---------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+<jupyter_output><empty_output><jupyter_text>4.4 Model-3 TFIDF- W2V 4.4.1 converting sentences into list of sentences<jupyter_code>list_of_sentance=[]
for sentance in (train["questions"]):
    list_of_sentance.append(sentance.split())<jupyter_output><empty_output><jupyter_text>4.4.2 tfidf vectorization of sentences<jupyter_code>from sklearn.feature_extraction.text import TfidfVectorizer

tfidf = TfidfVectorizer()
tfidf.fit_transform(train["questions"])
# we are converting a dictionary with word as a key, and the idf as a value

dictionary = dict(zip(tfidf.get_feature_names(),tfidf.idf_))#this dictionary will contain tfidf value of the particular word
zip for Converting two lists into a dictionary
# refer : https://stackoverflow.com/questions/209840/convert-two-lists-into-a-dictionary
import pickle
pickle.dump(tfidf,open('/content/drive/My Drive/case study data/tfidf','wb'))
dictionary['extract']<jupyter_output><empty_output><jupyter_text>4.4.3 training the model<jupyter_code>from gensim.models import Word2Vec
vec_size = 50#output vector size
w2v_model=Word2Vec(list_of_sentance,min_count=10,size=vec_size, workers=-1, iter=30)#for training w2v<jupyter_output><empty_output><jupyter_text>4.4.4 for pretrained w2v<jupyter_code>'''''from gensim.models import KeyedVectors
!wget -c "https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz"
w2v_model = KeyedVectors.load_word2vec_format('GoogleNews-vectors-negative300.bin.gz', binary=True)'''''
w2v_words = list(w2v_model.wv.vocab)
print("sample words ", w2v_words[0:10])<jupyter_output>sample words  ['wrong', 'use', 'rand', 'ms', 'sql', 'server', 'tri', 'select', 'random', 'sampl']<jupyter_text>4.4.5 TF-IDF weighted Word2Vec<jupyter_code>from tqdm import tqdm
import numpy as np
# final_tf_idf is the sparse matrix with row= sentence, col=word and cell_val = tfidf

tfidf_feat = tfidf.get_feature_names()
tfidf_sent_vectors = []# the tfidf-w2v for each sentence/review is stored in this list
row = 0
for sent in tqdm(list_of_sentance): # for each review/sentence 
  sent_vec = np.zeros(vec_size)# as word vectors are of zero length
  weight_sum = 0# num of words with a valid vector in the sentence/review
  for word in sent:# for each word in a review/sentence

    if word in tfidf_feat and word in w2v_words :
      vec = w2v_model.wv[word]
        # tf_idf = tf_idf_matrix[row, tfidf_feat.index(word)]
        # to reduce the computation we are 
        # dictionary[word] = idf value of word in whole courpus
        # sent.count(word) = tf valeus of word in this review
      tf_idf = dictionary[str(word)]*(sent.count(word)/len(sent))
      sent_vec+= (vec*tf_idf)
      weight_sum += tf_idf
  if weight_sum !=0:
    sent_vec /= weight_sum 
  tfidf_sent_vectors.append(sent_vec)
  row+=1
import pickle
pickle.dump(tfidf_sent_vectors,open('/content/drive/My Drive/case study data/tfidf_sent_vectors_500','wb'))
import pickle

tfidf_sent_vectors = pickle.load(open('/content/drive/My Drive/case study data/tfidf_sent_vectors_300','rb'))
tfidf = pickle.load(open('/content/drive/My Drive/case study data/tfidf','rb'))
w2v_words = pickle.load(open('/content/drive/My Drive/case study data/w2v_words','rb'))
w2v_model = pickle.load(open('/content/drive/My Drive/case study data/w2v_model','rb'))

print(len(tfidf_sent_vectors))<jupyter_output>106739<jupyter_text>4.4.6 finding most similar docs<jupyter_code>from bs4 import BeautifulSoup
import re
import nltk
from nltk.stem import SnowballStemmer
from nltk.tokenize import word_tokenize
nltk.download('punkt')

nltk.download('stopwords')
from nltk.corpus import stopwords

stop_words = set(stopwords.words('english'))
stemmer = SnowballStemmer("english")
def preprocess(data):
    soup = BeautifulSoup(data, 'lxml')
    cleantext = soup.get_text()
    soup1 = BeautifulSoup(cleantext, 'html5lib')
    cleantext = soup1.get_text()
    cleantext = re.sub('<.*?>', ' ', str(cleantext))  
    cleantext = re.sub('\\n', ' ', str(cleantext))  
    cleantext=re.sub('\sdiv\s', ' ', str(cleantext), flags=re.MULTILINE|re.DOTALL)
    cleantext = re.sub(r'[^A-Za-z]+',' ', str(cleantext))  
    words=word_tokenize(str(cleantext.lower()))
    cleantext=' '.join(str(stemmer.stem(j)) for j in words if j not in stop_words and (len(j)!=1 or j=='c'))

    return cleantext
import numpy as np
import time
from sklearn.metrics.pairwise import cosine_similarity
vec_size = 300
def most_sim__doc(text):
    start = time.time()#noting time for analysis

    sent_vec = np.zeros(vec_size)#creating a vector for test sentence
    weight_sum = 0
    #text = preprocess(text)

    lst_sent = list(text.split())#splitting the sentence
    for word in lst_sent:#iterating over words
        if word in tfidf_feat and word in w2v_words :#finding if word is present in tfidf and in w2v words
            vec = w2v_model[word]#finding vector of word
            tf_idf = dictionary[str(word)]*(lst_sent.count(word)/len(lst_sent))#computing tfidf 
            sent_vec+= (vec*tf_idf)#multiplying tfidf woth w2v value
            weight_sum += tf_idf#summing the words up
    if weight_sum !=0:
        sent_vec /= weight_sum #doing weighted sum

    similarity =  cosine_similarity((sent_vec).reshape(1, -1), Y=tfidf_sent_vectors, dense_output=True)#finding cosine similaroty of that particular vector with all the vectors in the corpus
    most_common = np.argsort(similarity[0])#performing argsort to get index of maximum similarity sentences
    s = most_common[-5:]#selecting top 5 results
    print("top cosine similarities:",similarity[0][s])
    rev = np.flip(s,axis = 0)#flipping the results to get results in decreasing order of similarity
    se = train["questions"][rev].values#getting the most similar values
    lst = ["TFIDF-W2V"]
    for i in se:
        #i = striphtml(i)#removing the html and preprocessing them
        sub_string = i.split()[:50]#considering only first 50 words
        lst.append(" ".join(sub_string))
    elapsed_time_fl = (time.time() - start) #counting the time taken to run this loop
    print(elapsed_time_fl)
    return (lst)#returning the result
tfidf_feat = tfidf.get_feature_names()
tfidf_w2v = most_sim__doc(str('what is superclass in object orient programming'))
print((tfidf_w2v))<jupyter_output>/usr/local/lib/python3.6/dist-packages/ipykernel_launcher.py:13: DeprecationWarning: Call to deprecated `__getitem__` (Method will be removed in 4.0.0, use self.wv.__getitem__() instead).
  del sys.path[0]<jupyter_text>4.4.7 displaying the results<jupyter_code>from prettytable import PrettyTable
x = PrettyTable(['model','TOP result1','TOP result2','TOP result3','TOP result4','TOP result5'])#defining the columns of table
x.align = "l"#alignment of the values

x.add_row(tfidf_w2v)#adding row to the table
print("Qurey : what is superclass in object orient programming" )
print(x)
300
Qurey : what is superclass in object orient programming
+-----------+-----------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+
| model     | TOP result1                                                                                                                 | TOP result2                                                                                                                                                                                                                                                                                                                                                                     | TOP result3                                                                                                                                                                     | TOP result4                                                                                                                                                                                                                                                                                            | TOP result5                                                                                                                                  |
+-----------+-----------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+
| TFIDF-W2V | find object class superclass c find object instanc certain class class superclass return true even object actual superclass | problem class design inherit flash problem design class three class one superclass two subclass one subclass animatedcharact made flash use display object screen characterphys made extend superclass problem object use type animatedcharact put variabl type characterphys tri sort decor pattern give object type characterphys refer object overrid method superclass pass | forc variabl superclass assign compil time superclass bunch variabl etc like guarante variabl get overridden assign someth use subclass inherit superclass compil time eleg way | object orient c would set nifti preprocessor hack ansi c iso c compat enabl kind ugli usabl object orient c familiar differ object orient languag pleas respond answer like learn c read object orient program ansi c bewar pdf format sever interest solut most interest see also write object orient | would one write object orient code c way write object orient code c especi regard polymorph see also stack overflow question object orient c |
+-----------+-----------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+
200
Qurey : what is superclass in object orient programming
+-----------+-----------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| model     | TOP result1                                                                                                                 | TOP result2                                                                                                                                             | TOP result3                                                                                                                                                                                                                                                                                            | TOP result4                                                                                                                                  | TOP result5                                                                                                                                                                                                                                                                                                                                                 |
+-----------+-----------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| TFIDF-W2V | find object class superclass c find object instanc certain class class superclass return true even object actual superclass | object orient model differ object orient program differ object orient model object orient program overheard convers subway train morn seem thing differ | object orient c would set nifti preprocessor hack ansi c iso c compat enabl kind ugli usabl object orient c familiar differ object orient languag pleas respond answer like learn c read object orient program ansi c bewar pdf format sever interest solut most interest see also write object orient | would one write object orient code c way write object orient code c especi regard polymorph see also stack overflow question object orient c | worth convert function javascript code object orient design current build small web applic includ fair amount javascript prototyp initi idea hack togeth function demonstr applic would eventu behav intend go forward write javascript object orient natur get implement phase find creat object orient javascript sake object orient seem overkil project |
+-----------+-----------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
100
Qurey : what is superclass in object orient programming
+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| model     | TOP result1                                                                                                                                                                                                                                                                                            | TOP result2                                                                                                                                  | TOP result3                                                                                                                                             | TOP result4                                                                                                                                                                                                                                                                                | TOP result5                                                                                                                                                                                                                                                                                                                                                 |
+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| TFIDF-W2V | object orient c would set nifti preprocessor hack ansi c iso c compat enabl kind ugli usabl object orient c familiar differ object orient languag pleas respond answer like learn c read object orient program ansi c bewar pdf format sever interest solut most interest see also write object orient | would one write object orient code c way write object orient code c especi regard polymorph see also stack overflow question object orient c | object orient model differ object orient program differ object orient model object orient program overheard convers subway train morn seem thing differ | make languag object orient sinc debat without meaning term meaningless figur would point eleph room ask exact make languag object orient look textbook answer one base experi oo languag work well domain whatev may relat question might help answer first archetyp object orient languag | worth convert function javascript code object orient design current build small web applic includ fair amount javascript prototyp initi idea hack togeth function demonstr applic would eventu behav intend go forward write javascript object orient natur get implement phase find creat object orient javascript sake object orient seem overkil project |
+-----------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+----------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+<jupyter_output><empty_output><jupyter_text>4.5 Model-4 : Universal Sentence Encoder  4.5.1 loading the model<jupyter_code>import tensorflow_hub as hub

import tensorflow.compat.v1 as tf
tf.disable_v2_behavior()
embed = hub.load("https://tfhub.dev/google/universal-sentence-encoder/4")#loading the tensorflow hub model

def use(sent):#converting sentence into vectors
  with tf.Session() as session:
      session.run([tf.global_variables_initializer(), tf.tables_initializer()])
      message_embeddings = session.run(embed(sent))#this will embed the sentence into vectors of size 512
      return message_embeddings<jupyter_output>WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/compat/v2_compat.py:96: disable_resource_variables (from tensorflow.python.ops.variable_scope) is deprecated and will be removed in a future version.
Instructions for updating:
non-resource variables are not supported in the long term
WARNING:tensorflow:From /usr/local/lib/python3.6/dist-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.<jupyter_text>4.5.2 finding word embedding<jupyter_code>message_embeddings = use(train["questions"][:50000])
print(message_embeddings.shape)
message_embeddings2 = use(train["questions"][50000:])
print(message_embeddings2.shape)
message_embeddings = np.concatenate((message_embeddings, message_embeddings2), axis=0)
message_embeddings.shape<jupyter_output><empty_output><jupyter_text>4.5.3 predicting the results<jupyter_code>import numpy as np
import time
from sklearn.metrics.pairwise import cosine_similarity
import re
def most_sim__doc(text):
    start = time.time()#noting the current time
    sent_vec = use([text])#predicting the embedding
    similarity =  cosine_similarity((sent_vec).reshape(1, -1), Y=message_embeddings, dense_output=True)#finding the cosine similarity
    most_common = np.argsort(similarity[0])#sorting the similarity to get top results
    s = most_common[-5:]#considering the top 5 results
    print("top cosine similarities:",similarity[0][s])
    rev = np.flip(s,axis = 0)
    se = train["content"][rev].values#finding the top indeces values
    lst = ["universal sentence decoder"]
    for i in se:
        i = striphtml(i)Epreprocess the output
        sub_string = i.split()[:50]#considering first 50 words
        lst.append(" ".join(sub_string))
    elapsed_time_fl = (time.time() - start) 
    print(elapsed_time_fl)
    return (lst)
uni_sen_dec = most_sim__doc('what is superclass in object orient programming')<jupyter_output>top cosine similarities: [0.51230574 0.5123432  0.5286257  0.52907324 0.5967799 ]
4.0943992137908936<jupyter_text>4.5.4 displaying the result<jupyter_code>from prettytable import PrettyTable
x = PrettyTable(['model','TOP result1','TOP result2','TOP result3','TOP result4','TOP result5'])
x.align = "l"
x.add_row(uni_sen_dec)
print("Qurey : what is superclass in object orient programming" )
print(x)


Qurey : what is superclass in object orient programming
+----------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| model                      | TOP result1                                                                                                                                                                                                                                         | TOP result2                                                                                                                                                                                                                                                                                                                              | TOP result3                                                                                                                                                                                                                                                                           | TOP result4                                                                                                                                                                                                                                                      | TOP result5                                                                                                                                                                                            |
+----------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| universal sentence decoder | Is JavaScript object-oriented?There have been some questions about whether or not JavaScript is an object-oriented language. Even a statement, "just because a language has objects doesn't make it OO." Is JavaScript an object-oriented language? | When is Object Oriented not the correct solution?I've encountered lately some opinions saying that Object Oriented design/programming should not always be used. Do you know some use-cases that will not benefit from and should not use Object Oriented design? For example: there are some problems (concerns) that will benefit from | Is Object-Oriented Modeling different from Object-Oriented Programming?What is the difference between Object-Oriented Modeling and Object-Oriented Programming? I overheard a conversation on my subway train this morning and it seems that these things are different. Aren't they? | Are there any good courses for learning about Object-Oriented Programming?I'm looking for some good courses on object oriented programming. I've been programming for about 4 years so far, but I don't feel like I have a SOLID grasp on OO. How did you learn? | How would one write object-oriented code in C?What are some ways to write object-oriented code in C? Especially with regard to polymorphism. See also Stack Overflow question Object-orientation in C. |
+----------------------------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.38324588324588327, 'fraction_numerical': 0.0099000099000099, 'mean_word_length': 0.9155788741742786, 'pattern_counts': {'":': 0, '<': 188, '<?xml version=': 0, '>': 187, 'https://': 6, 'lorem ipsum': 0, 'www.': 3, 'xml': 2}, 'pii_count': 2, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 324, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '33', 'dir': 'jupyter-structured-clean-dedup', 'id': '115389', 'license': 'Apache-2.0', 'n_tokens_mistral': 11242, 'n_tokens_neox': 10663, 'n_words': 3327, 'path': 'case_study_final.ipynb', 'repo_name': 'sezazqureshi/stackoverflow-search-engine', 'size': '60606'}