<jupyter_start><jupyter_text>Config<jupyter_code>import requests
import urllib
from bs4 import BeautifulSoup
import bs4
import re
import scholarly
import pandas as pd
import multiprocessing as mp
from joblib import Parallel, delayed<jupyter_output><empty_output><jupyter_text>Scraping Faculty Names<jupyter_code>eco_faculy=pd.read_excel('Data/Faculty_names.xlsx',index_col=0)
## Scraping the faculty name did not work out because of the wierd html tags of facultis in the cology directory
url='https://www.ecology.uga.edu/directory/#faculty'
Page=requests.get(url)
soup=BeautifulSoup(Page.text,'html.parser')
eco_faculty_soup=soup.find_all('div',{'class':'portal-item bio-wrap mix courtesy-faculty g '})
faculty_names_tag=eco_faculty_soup.find_all('span',{'class':'title'})

n_faculty=len(faculty_names_tag)

faculty_names=[]

for i in range(n_faculty):
    
    academic_title=faculty_names_tag[i].parent.parent.p.text
    full_name=faculty_names_tag[i].parent.parent.h5.text
    is_eco=False
    if 'ecology' in faculty_names_tag[0].parent.parent.a['href']:
        is_eco=True
    
    if ('Professor' in academic_title) and (is_eco==True):
        faculty_names.append(full_name)

faculty_names=list(set(faculty_names))
eco_faculy['GS_link']=np.nan
eco_faculy['academic_title']=np.nan
eco_faculy['GS_keywords']=np.nan
for faculty_name in eco_faculy.index:
    search_query = scholarly.search_author(faculty_name+', uga')
    query_result = next(search_query,'NA')
    if query_result!='NA':
        author = query_result.fill()
        eco_faculy.loc[faculty_name,'GS_link']=author.id
        eco_faculy.loc[faculty_name,'academic_title']=author.affiliation
        eco_faculy.loc[faculty_name,'GS_keywords']=','.join(author.interests)
        print(faculty_name,author.affiliation,'\n\n')
eco_faculy.to_pickle('Data/eco_faculy')
eco_faculy.to_excel('Data/eco_faculy.xlsx')<jupyter_output><empty_output><jupyter_text>Scraping abstract: Faculty with Public Google Scholar Profile<jupyter_code>eco_faculty=pd.read_pickle('Data/eco_faculty')
eco_faculty.index=eco_faculty.index.str.replace('.',' ')
def PubsAbs(faculty_name,df_eco_faculty):
    print(faculty_name)
    pub_df=pd.DataFrame(columns=['title','abstract','year'])
    
    search_query = scholarly.search_author(faculty_name+', uga')
    query_result = next(search_query,'NA')
    author = query_result.fill()
    
    for i,pub in enumerate(author.publications):
        print(i)
        pub_meta=pub.fill()
        pub_df.loc[i,'title']=pub_meta.bib['title']
        
        if 'abstract' in pub_meta.bib.keys():
            if type(pub_meta.bib['abstract'])==bs4.element.Tag:
                abs_txt=pub_meta.bib['abstract'].text
            else:
                abs_txt=pub_meta.bib['abstract'] 
            abs_txt=abs_txt.replace('\xa0…','')
            pub_df.loc[i,'abstract']=abs_txt

        if ('year' in pub.bib.keys()):
            pub_df.loc[i,'year']=pub.bib['year']
    pub_df.to_pickle('Faculty pubs/'+faculty_name)
    return 0
n_cores=mp.cpu_count()
pool = mp.Pool(processes=n_cores-2)
remaining_faculty=eco_faculty[eco_faculty.GS_link.notnull()].index
Parallel(n_jobs=n_cores)(delayed(PubsAbs)(faculty_name,eco_faculty) for faculty_name in remaining_faculty)<jupyter_output><empty_output><jupyter_text>Scraping abstract: Faculty with Private Google Scholar Profile<jupyter_code>eco_faculty[eco_faculty.GS_link.isnull()].index
pd.read_pickle('Faculty pubs/Elizabeth King')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13052208835341367, 'fraction_numerical': 0.0025817555938037868, 'mean_word_length': 5.26032315978456, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 17, 'https://': 1, 'lorem ipsum': 0, 'www.': 1, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '637880', 'license': 'MIT', 'n_tokens_mistral': 1316, 'n_tokens_neox': 1280, 'n_words': 167, 'path': 'scraping_GS.ipynb', 'repo_name': 'ArashSaeidpour/Hierarchical-Clustering-of-Odum-School-of-Ecology-Faculty-based-on-their-publications', 'size': '3486'}