<jupyter_start><jupyter_text>[EDA And Data collection ] :  Optional > Optional<jupyter_code>!wget https://archive.org/download/stackexchange/ai.stackexchange.com.7z<jupyter_output>--2021-11-12 18:18:19--  https://archive.org/download/stackexchange/ai.stackexchange.com.7z
Resolving archive.org (archive.org)... 192.168.3.11
Connecting to archive.org (archive.org)|192.168.3.11|:443... connected.
HTTP request sent, awaiting response... 302 Found
Location: https://ia600107.us.archive.org/27/items/stackexchange/ai.stackexchange.com.7z [following]
--2021-11-12 18:18:20--  https://ia600107.us.archive.org/27/items/stackexchange/ai.stackexchange.com.7z
Resolving ia600107.us.archive.org (ia600107.us.archive.org)... 192.168.3.11
Connecting to ia600107.us.archive.org (ia600107.us.archive.org)|192.168.3.11|:443... connected.
HTTP request sent, awaiting response... 200 OK
Length: 26406301 (25M) [application/x-7z-compressed]
Saving to: ‘ai.stackexchange.com.7z’

ai.stackexchange.co 100%[===================>]  25.18M   929KB/s    in 17s     

2021-11-12 18:18:37 (1.46 MB/s) - ‘ai.stackexchange.com.7z’ saved [26406301/26406301]<jupyter_text>Unziping .7z<jupyter_code>!sudo apt-get install p7zip
!sudp atp-get install p7zip-full
!7za x *.7z
%cd /content/
%pwd
%mkdir dataset
%mv *.xml /content/dataset
pip install pandas_read_xml
import pandas_read_xml as pdx
from pandas_read_xml import flatten, fully_flatten, auto_separate_tables
df = pdx.read_xml("/content/dataset/Comments.xml")
df.head()
# See what "flatten" does
 
df = df.pipe(flatten)
 
df
# See what another "flatten" does
 
df = df.pipe(flatten)
 
df
# See what another "flatten" does
 
df = df.pipe(flatten)
 
df
df.columns
list(df["comments|row|@Text"])
answer_corpus = " ".join(list(df["comments|row|@Text"]))
%cd /content/<jupyter_output>/content<jupyter_text>Setup required enviorment<jupyter_code>!pip install farm-haystack

# Install the latest master of Haystack
!pip install grpcio-tools==1.34.1
!pip install git+https://github.com/deepset-ai/haystack.git
!pip install contractions autocorrect<jupyter_output><empty_output><jupyter_text>Data Preparation<jupyter_code>from haystack.document_stores import ElasticsearchDocumentStore
from haystack.nodes import EmbeddingRetriever
import pandas as pd
import requests


import time as time
import numpy as np
import pandas as pd
import gc

# Text librairies
import re
from bs4 import BeautifulSoup
import nltk
from nltk.tokenize import ToktokTokenizer
from nltk.stem.wordnet import WordNetLemmatizer
from nltk.stem import PorterStemmer
from nltk.corpus import wordnet
from nltk.corpus import stopwords
from nltk.tag.util import untag
import contractions

# import pycontractions 
# Alternative better package for removing contractions
from autocorrect import Speller

# Recommended: Start Elasticsearch using Docker via the Haystack utility function
from haystack.utils import launch_es
launch_es()


import spacy
EN = spacy.load('en_core_web_sm')

df = pd.read_csv("/content/drive/MyDrive/stackoverflow-data/StackOverflow Data/Original_data.csv",nrows=25000)
df.head()<jupyter_output><empty_output><jupyter_text>Testing single records> Que> Answers<jupyter_code>df.shape
df["title"][17]
df["answers"][17]
for i, row in df.iterrows():
  print(row.title)
  break<jupyter_output>What does the /= operator mean in Python?<jupyter_text>**Filtering Records whose score is more than 0**<jupyter_code>df = df[df["score"] >= 0]
df.head()
df.shape
 nltk.download('stopwords')<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.<jupyter_text>**Data Cleaning**<jupyter_code>spell = Speller()
token = ToktokTokenizer()
lemmatizer = WordNetLemmatizer()
stemmer = PorterStemmer()
charac = '!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~0123456789'
stop_words = set(stopwords.words("english"))
adjective_tag_list = set(['JJ','JJR', 'JJS', 'RBR', 'RBS']) # List of Adjective's tag from nltk package
df.columns
%%time

# Parse question and title then return only the text
df['body'] = df['body'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text())
df['title'] = df['title'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text())
df['answers'] = df['answers'].apply(lambda x: BeautifulSoup(x, 'html.parser').get_text())
df.head()
def clean_text(text):
    text = re.sub(r"\'", "'", text) # match all literal apostrophe pattern then replace them by a single whitespace
    text = re.sub(r"\n", " ", text) # match all literal Line Feed (New line) pattern then replace them by a single whitespace
    text = re.sub(r"\xa0", " ", text) # match all literal non-breakable space pattern then replace them by a single whitespace
    text = re.sub('\s+', ' ', text) # match all one or more whitespace then replace them by a single whitespace
    text = text.strip(' ')
    return text
%%time

df['title'] = df['title'].apply(lambda x: clean_text(x)) 
df['body'] = df['body'].apply(lambda x: clean_text(x))
df['answers'] = df['answers'].apply(lambda x: clean_text(x))
df.head()<jupyter_output><empty_output><jupyter_text>Setup Elastic Search<jupyter_code>import os
from subprocess import Popen, PIPE, STDOUT
# # In Colab / No Docker environments: Start Elasticsearch from source
! wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-7.9.2-linux-x86_64.tar.gz -q
! tar -xzf elasticsearch-7.9.2-linux-x86_64.tar.gz
! chown -R daemon:daemon elasticsearch-7.9.2


es_server = Popen(['elasticsearch-7.9.2/bin/elasticsearch'],
                   stdout=PIPE, stderr=STDOUT,
                   preexec_fn=lambda: os.setuid(1)  # as daemon
                  )
# wait until ES has started
! sleep 30<jupyter_output><empty_output><jupyter_text>Setup Document Store<jupyter_code>from haystack.document_stores import ElasticsearchDocumentStore
document_store = ElasticsearchDocumentStore(host="localhost", username="", password="",
                                            index="document",
                                            embedding_field="question_emb",
                                            embedding_dim=384,
                                            excluded_meta_data=["question_emb"])
retriever = EmbeddingRetriever(document_store=document_store, 
                               embedding_model="sentence-transformers/all-MiniLM-L6-v2",
                               use_gpu=True)
df.columns
df.fillna(value="", inplace=True)
df["question"] = df["title"].apply(lambda x: x.strip())
print(df.head())

# Get embeddings for our questions from the FAQs
questions = list(df["title"].values)


df["question_emb"] = retriever.embed_queries(texts=questions)
df = df.rename(columns={"question": "content"})

# Convert Dataframe to list of dicts and index them in our DocumentStore
docs_to_index = df.to_dict(orient="records")
document_store.write_documents(docs_to_index)<jupyter_output>id  ...                                                                         question
0  32490721  ...                                        What does the /= operator mean in Python?
1  16784162  ...  What is a solid example of something that can be done with list comprehensio...
2   2763750  ...                         How to replace only part of the match with python re.sub
3   7931726  ...                                        Are datagrams always received completely?
4   3838329  ...                                       How can I check if two segments intersect?

[5 rows x 7 columns]<jupyter_text>Loading Pipeline<jupyter_code>from haystack.pipelines import FAQPipeline
pipe = FAQPipeline(retriever=retriever)
df["tags"].value_counts().to_dict()
from haystack.utils import print_answers
from pprint import pprint

Q = input("Enter query : ")
K = 3
prediction = pipe.run(query=Q, params={"Retriever": {"top_k": K}})

#res = print_answers(prediction, details="high")


print("Query :",Q)
print("="*50)

for results in range(K):

  result = prediction["answers"][results].to_dict()

  result = {"Query":Q,
            "Answer":result["meta"]["answers"],
            "Score":result["score"],
            "Tags":result["meta"]["tags"]}
  pprint(result)

  print("-"*50)<jupyter_output>Enter query : How to create virtual env in linux ?
{'fraction_non_alphanumeric': 0.1296206225680934, 'fraction_numerical': 0.03489785992217899, 'mean_word_length': 3.8726303317535544, 'pattern_counts': {'":': 7, '<': 36, '<?xml version=': 0, '>': 41, 'https://': 6, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 23, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '564626', 'license': 'CC0-1.0', 'n_tokens_mistral': 2937, 'n_tokens_neox': 2692, 'n_words': 636, 'path': 'Question_and_Answering_Stackoverflow.ipynb', 'repo_name': 'VinayChaudhari1996/Stackoverflow-Question-Answering-', 'size': '8232'}