<jupyter_start><jupyter_text>Query pre_processing<jupyter_code>!pip install nltk
# Read/create the text data
#tweet_sample= "How to take control of your #debt https://personal.vanguard.com/us/insights/saving-investing/ debt-management.#Best advice for #family #financial #success (@PrepareToWin)"
import re
import string
from textblob import TextBlob
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
from textblob import Word
from nltk.util import ngrams
import re
from wordcloud import WordCloud, STOPWORDS
import nltk
nltk.download('stopwords')
nltk.download('wordnet')

from nltk.tokenize import word_tokenize
from nltk import ToktokTokenizer

tokenizer = ToktokTokenizer()
stopword_list = nltk.corpus.stopwords.words('english')

def remove_stopwords(text):
    text = text.lower()
    text = re.sub(r'[^\w\s]', ' ', text)
    if text.startswith(" "):
        text = text.lstrip()
    tokens = tokenizer.tokenize(text)
    texts = [word for word in tokens if word not in stopword_list]
    texts = ' '.join(texts)
    return texts

def remove_punctuation_marks(text):
    s = text
    for c in string.punctuation:
        s= s.replace(c,"")
    return s
    


#process the data
def pre_process_query(query1):
        
        #remove punctuation
        query = remove_punctuation_marks(query1)
        #remove stopwords
        tweet = remove_stopwords(query)  
        #Removes unicode strings like "\u002c" and "x96"
        try:
           tweet = re.sub(r'(\\u[0-9A-Fa-f]+)',r'', tweet)
           tweet = re.sub(r'[^\x00-\x7f]',r'',tweet)
        except:
          tweet = tweet
        #convert any url to URL
        tweet = re.sub('((www\.[^\s]+)|(https?://[^\s]+))','URL',tweet)
        #Convert any @Username to "AT_USER"
        tweet = re.sub('@[^\s]+','AT_USER',tweet)
        #Remove additional white spaces
        tweet = re.sub('[\s]+', ' ', tweet)
        tweet = re.sub('[\n]+', ' ', tweet)
        #Remove not alphanumeric symbols white spaces
        tweet = re.sub(r'[^\w]', ' ', tweet)
        #Removes hastag in front of a word """
        tweet = re.sub(r'#([^\s]+)', r'\1', tweet)
        #Replace #word with word
        tweet = re.sub(r'#([^\s]+)', r'\1', tweet)
        #lemma
        from textblob import Word
        tweet =" ".join([Word(word).lemmatize() for word in tweet.split()])
        #stemmer
        st = PorterStemmer()
        tweet=" ".join([st.stem(word) for word in tweet.split()])
        #Removes emoticons from text
        tweet = re.sub(':\)|;\)|:-\)|\(-:|:-D|=D|:P|xD|X-p|\^\^|:-*|\^\.\^|\^\-\^|\^\_\^|\,-\)|\)-:|:\'\(|:\(|:-\(|:\S|T\.T|\.\_\.|:<|:-\S|:-<|\*\-\*|:O|=O|=\-O|O\.o|XO|O\_O|:-\@|=/|:/|X\-\(|>\.<|>=\(|D:', '', tweet)
        #trim
        tweet = tweet.strip('\'"')
        row = tweet
        return row

#pre_process_query(query1='Dhoni has won the match against punjab team, and now he is leaving the stadium with his team')
#Download pre-trained google word2vec model and save it into root path
!wget -P /root/input/ -c "https://s3.amazonaws.com/dl4j-distribution/GoogleNews-vectors-negative300.bin.gz"
import warnings
warnings.filterwarnings(action='ignore', category=UserWarning, module='gensim')
import warnings
warnings.filterwarnings('ignore')
import gensim
import numpy as np
import scipy
from nltk.tokenize.toktok import ToktokTokenizer
import re
import nltk
nltk.download('punkt')

tokenizer = ToktokTokenizer()
stopword_list = nltk.corpus.stopwords.words('english')

EMBEDDING_FILE = '/root/input/GoogleNews-vectors-negative300.bin.gz'
model = gensim.models.KeyedVectors.load_word2vec_format(EMBEDDING_FILE, binary=True, limit=50000)


def remove_stopwords(text):
    text = text.lower()
    text = re.sub(r'[^\w\s]', ' ', text)
    text = text.replace('  ', ' ')
    if text.startswith(" "):
        text = text.lstrip()
    tokens = tokenizer.tokenize(text)
    texts = [word for word in tokens if word not in stopword_list]
    return texts

# Function to get the embedding vector for n dimension, we have used "300"

def get_embedding(word):
    if word in model.vocab:
        return model[word]
    else:
        return np.zeros(300)
    # Getting average vector for each document

def get_vectors(fin):
    out_dict = {}
    for sen in fin:
              average_vector = np.mean(np.array([get_embedding(x) for x in remove_stopwords(sen)]), axis=0)
              dict = {sen: (average_vector)}
              out_dict.update(dict)
    return out_dict


# Function to calculate the similarity between the query vector and document vector
def get_sim(query_embedding, average_vector_doc):
    sim = [(1 - scipy.spatial.distance.cosine(query_embedding, average_vector_doc))]
    return sim


# Rank all the documents based on the similarity to get relevant docs
def Ranked_documents(query, corpse):
    query_words = (
        np.mean(np.array([get_embedding(x) for x in nltk.word_tokenize(query.lower())], dtype=float), axis=0))
    rank = []
    fin = corpse
    dict = get_vectors(fin)
    for k, v in dict.items():
      if not k=='':
        rank.append((k, get_sim(query_words, v)))
    rank = sorted(rank, key=lambda t: t[1], reverse=True)[:3]
    return rank


import warnings
warnings.filterwarnings('ignore')

user_query = input ("Please Enter your Query: ") 
Ranked_documents(user_query,["",
                             "<NAME>, is a Pakistani cricketer and the former captain of the Pakistan national cricket team. Afridi is widely considered as one of the most popular and destructive cricketers. An all-rounder, Afridi bowled leg spin and was recognized for his aggressive batting.",
                             "<NAME> is a Pakistani international cricketer who has represented Pakistan in international cricket since 2015. A right-handed batsman and wicket-keeper, he has scored centuries in all three international formats: Tests, One Day Internationals and Twenty20 Internationals.",
                             "<NAME> is a Pakistani cricketer who plays for the Pakistan national cricket team and for Lahore Qalandars in the Pakistan Super League. On 20 July 2018, he became the first batsman for Pakistan to score a double century in a One Day International match.",
                             "Lahore Qalandars have three big match-winners in the bowling department. The presence of Shaheen Afridi, <NAME> and <NAME> makes them a team to beat. The trio performed well last year and would look to dish out match-winning performances this year as well."])<jupyter_output>Please Enter your Query: Lahore won three matches straight in psl 2022
{'fraction_non_alphanumeric': 0.13009738283627512, 'fraction_numerical': 0.008673158855751673, 'mean_word_length': 3.1548672566371683, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 11, 'https://': 2, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '402052', 'license': 'MIT', 'n_tokens_mistral': 2173, 'n_tokens_neox': 2085, 'n_words': 625, 'path': 'Information_Retrival.ipynb', 'repo_name': 'Sawera557/NLP_Search_engine', 'size': '6603'}