<jupyter_start><jupyter_text>Objective: The objective of this assignment is “Building a repository of universities of UK”<jupyter_code>#Importing Necessary Libraries
from bs4 import BeautifulSoup as soup
import requests
import lxml
import re
import pandas as pd
from threading import Thread
from elasticsearch import Elasticsearch<jupyter_output><empty_output><jupyter_text>Extracting details from main website<jupyter_code>main_url='https://en.wikipedia.org/wiki/List_of_universities_in_England'
source=requests.get(my_url).text
page_soup=soup(source,'lxml')
containers=page_soup.find('div',{'class':"mw-parser-output"})
rows= containers.table.find_all('tr') 
data=[]
header_name=[]
x=0
type_name=re.split('https://en.wikipedia.org/wiki/',my_url)[1]
# csv_file=open(csv_name[1],'w')

for row in rows:
    x+=1
    if x==1:
        header=row.find_all('th')
        for hedr in header:
            header_name.append(hedr.text[0:-1])
        header_name.append('Url')
        data.append(header_name)
    else:
        cols = row.find_all('td')
        link='https://en.wikipedia.org'+cols[0].a.get('href')
        cols = [ele.text.strip() for ele in cols]
        cols.append(link)
        data.append([ele for ele in cols if ele])

df = pd.DataFrame.from_records(data[1:], columns=data[0])
urls=list(df['Url'])

df<jupyter_output><empty_output><jupyter_text>Extracting details from individual university website<jupyter_code>sub_agg_df = pd.DataFrame(columns=['University','Former names','Detailed_Location','Students','Undergraduates','Postgraduates','Url'])
print(sub_agg_df)

# defining the function for scraping individual website 
def sub_scrapping(url,indx,U_name):
    print(url,indx)
    my_url=url
    try:
        print('Executing for '+my_url)
        source=requests.get(my_url).text
        page_soup=soup(source,'lxml')
        containers=page_soup.find('table',{'class':"infobox vcard"})
        indivual_data={}
        sub_headers=['Former names','Location','Students','Undergraduates','Postgraduates']

        for sh in range(1,6):
            sub_container=containers.find_all('tr')
            for ele in sub_container:
                try:
                    if ele.th.text==sub_headers[sh-1]:
                        indivual_data.update({sub_headers[sh-1]:ele.td.text})
                except:
                    pass
            if len(indivual_data) < sh:
                indivual_data.update({sub_headers[sh-1]:None})


        df_sub=pd.DataFrame([[U_name,indivual_data['Former names'],str(indivual_data['Location']),indivual_data['Students'],indivual_data['Undergraduates'],indivual_data['Postgraduates'],my_url]],columns=['University','Former names','Detailed_Location','Students','Undergraduates','Postgraduates','Url'],index=[indx])
        global sub_agg_df
        sub_agg_df=sub_agg_df.append(df_sub)
    except:
        print('Error in fetching url '+my_url)<jupyter_output>Empty DataFrame
Columns: [University, Former names, Detailed_Location, Students, Undergraduates, Postgraduates, Url]
Index: []<jupyter_text>Creating the Thread for parallel processing.<jupyter_code>threadlist=[]
for x in df.itertuples():
    td= Thread(target=sub_scrapping,args=(x.Url,x.Index,x.University))
    td.start()
    threadlist.append(td)
for b in threadlist:
    b.join()
sub_agg_df<jupyter_output>https://en.wikipedia.org/wiki/Anglo-European_College_of_Chiropractichttps://en.wikipedia.org/wiki/Anglia_Ruskin_University 1
Executing for https://en.wikipedia.org/wiki/Anglia_Ruskin_University
https://en.wikipedia.org/wiki/Arden_University 2
Executing for https://en.wikipedia.org/wiki/Arden_University
 https://en.wikipedia.org/wiki/Arts_University_Bournemouth0 
Executing for https://en.wikipedia.org/wiki/Anglo-European_College_of_Chiropractic
3
Executing for https://en.wikipedia.org/wiki/Arts_University_Bournemouth
https://en.wikipedia.org/wiki/University_of_the_Arts_London 4
Executing for https://en.wikipedia.org/wiki/University_of_the_Arts_London
https://en.wikipedia.org/wiki/Aston_University 5https://en.wikipedia.org/wiki/University_of_Bath
 https://en.wikipedia.org/wiki/Bath_Spa_University 7
Executing for https://en.wikipedia.org/wiki/Bath_Spa_University
Executing for https://en.wikipedia.org/wiki/Aston_University6https://en.wikipedia.org/wiki/University_of_Bedfordshire
Executing [...]<jupyter_text>Final adjustment by joinging the sub table with sub table<jupyter_code>df.drop('Url', axis=1, inplace=True)
final_df=pd.merge(sub_agg_df, df, on='University')
final_df
# Optional: Saving the table in CSV format for analysis purspose in excel:
final_df.to_csv(type_name+'.csv')
es= Elasticsearch('http://localhost:9200')
es.indices.delete(index='universities')
es.indices.create(index='universities',ignore=400)
for x in final_df.iterrows():
    es.index(index='universities',doc_type=type_name,id=x[0] ,body=dict(x[1]))
print('!!!!!!!!----Data Transfer Completed-----!!!!!!!')<jupyter_output>C:\Users\akhil\anaconda3\envs\webscrapping\lib\site-packages\ipykernel_launcher.py:5: ElasticsearchDeprecationWarning: [types removal] Specifying types in document index requests is deprecated, use the typeless endpoints instead (/{index}/_doc/{id}, /{index}/_doc, or /{index}/_create/{id}).
  """<jupyter_text>Check one query whether the documents are inserted or not.<jupyter_code>es= Elasticsearch('http://localhost:9200')
res=es.search(index='universities',body={"from":0, "size":1,"query":{"match":{"University":"Harper Adams University"}}})
res<jupyter_output><empty_output><jupyter_text>Optimize search which includes both current name and former names of university:<jupyter_code>res=es.search(index="universities",body={"from":0, "size":2, "query":{"bool":{"must":{"match":{"University":"Harper Adams University"}},"must":{"match":{"Former names":"None"}}}}})           
res<jupyter_output><empty_output><jupyter_text>Since the parse data need more cleaning before it pass to ElasticSearch engine. The schema should be in following format in production ready environment:<jupyter_code>{
    "settings":{
        "analysis":{
            "analyzer":{
                "my_analyzer":{
                    "type":"keyword",
                }
            }
        }
    }

    "mappings":{
        "doc":{
            "dynamic": "strict",
            "properties":{
                "University":{
                    "type":"text",
                    "fields":{
                        "keyword":{
                            "type":"keyword"
                        }
                    },
                    "analyzer":"my_analyzer"
                },
                "Former names":{
                    "type":"text",
                    "fields":{
                        "keyword":{
                            "type":"keyword"
                        }
                    },
                    "analyzer":"my_analyzer"
                },
                "Detailed_Location":{
                    "type":"text",
                    "fields":{
                        "keyword":{
                            "type":"keyword"
                        }
                    },
                    "analyzer":"my_analyzer"
                },
                "Students":{
                    "type":"integer"
                },
                "Undergraduates":{
                    "type":"integer"
                },
                "Postgraduates":{
                    "type":"integer"
                },
                "Url":{
                    "type":"text",
                    "fields":{
                        "keyword":{
                            "type":"keyword"
                        }
                    },
                    "analyzer":"my_analyzer"
                },
                "Location":{
                    "type":"text",
                    "fields":{
                        "keyword":{
                            "type":"keyword"
                        }
                    },
                    "analyzer":"my_analyzer"
                },
                "Established":{
                    "type":"integer"
                },
                "Number of students":{
                    "type":"integer"
                },
                "Tuition fee":{
                    "type":"integer"
                },
                "Degree powers":{
                    "type":"text",
                    "fields":{
                        "keyword":{
                            "type":"keyword"
                        }
                    },
                    "analyzer":"my_analyzer"
                }
            }
        }
    }
}<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1344197873324087, 'fraction_numerical': 0.005085529357374018, 'mean_word_length': 2.2751703255109765, 'pattern_counts': {'":': 72, '<': 31, '<?xml version=': 0, '>': 30, 'https://': 19, 'lorem ipsum': 0, 'www.': 0, 'xml': 3}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '164524', 'license': 'MIT', 'n_tokens_mistral': 2652, 'n_tokens_neox': 2457, 'n_words': 399, 'path': 'ScrapperProject.ipynb', 'repo_name': 'akhildudhe/Python_WebScrapper_With_Elastic_Search', 'size': '8652'}