<jupyter_start><jupyter_text>Getting names and email id:<jupyter_code>from bs4 import BeautifulSoup
import urllib.request
import pandas as pd
import re
df=pd.DataFrame(columns=['Name','Email','Department'])<jupyter_output><empty_output><jupyter_text>For applied mechanics<jupyter_code>source = urllib.request.urlopen('https://am.iitd.ac.in/?q=node/24').read()
soup=BeautifulSoup(source,'lxml')
for table in soup.find_all('table'):
    names=table.find('td',class_="rwspeopletitle")
    #entries=table.find_all('td')
    txt = table.text
    txt=txt.replace('[at]','@')
    email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", txt)
    if not len(email):
        email=re.findall("\w+@\w+\.{1}\w+", txt) 
    df.loc[len(df.index)] = [names.text,email[0],"Department of Applied Mechanics"]<jupyter_output><empty_output><jupyter_text>for Biochemical Engineering and Biotechnology<jupyter_code>source = urllib.request.urlopen('https://beb.iitd.ac.in/faculty.html').read()
soup=BeautifulSoup(source,'lxml')
source = urllib.request.urlopen('https://beb.iitd.ac.in/faculty.html').read()
soup=BeautifulSoup(source,'lxml')
try:
    for card in soup.find_all('div',class_="card"):
        l=card.a.get('href')
        link = urllib.request.urlopen('https://beb.iitd.ac.in/'+l).read()
        page=BeautifulSoup(link,'lxml')
        name=page.find('font',class_='name')
        email=page.p.a
        df.loc[len(df.index)] = [names.text,email.text,"Biochemical Engineering and Biotechnology"]
except:
    pass<jupyter_output><empty_output><jupyter_text>for chemical engineering<jupyter_code>source = urllib.request.urlopen('http://chemical.iitd.ac.in/people/').read()
soup=BeautifulSoup(source,'lxml')
for faculty in soup.find_all('div',class_='each'):
    name=faculty.find('div',class_="faculty_responsive_val ml5").text
    name=name.strip()
    email=faculty.find('div',class_="Contact_div table_middle").a.text
    df.loc[len(df.index)] = [name,email,"Chemical Engineering "]<jupyter_output><empty_output><jupyter_text>for chemistry<jupyter_code>source = urllib.request.urlopen('https://chemistry.iitd.ac.in/faculty.html').read()
soup=BeautifulSoup(source,'lxml')
name="None"
try:
    for people in soup.find_all('div',class_="row"):
        if people.find('div',class_="col-sm-7"):
            if people.find('div',class_="col-sm-7").h3.text.strip()==name:
                continue
            name= people.find('div',class_="col-sm-7").h3.text.strip()
            txt=people.text
            txt=txt.replace('(AT)','@')
            email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", txt)
            df.loc[len(df.index)] = [name,email[0],"Department of Chemistry"]
except:
    pass<jupyter_output><empty_output><jupyter_text>for civil<jupyter_code>source = urllib.request.urlopen('https://civil.iitd.ac.in/index.php?lmenuid=faculty').read()
soup=BeautifulSoup(source,'lxml')
for td in soup.find_all('td'):
    if td.b:
        email=td.text
        email=email.replace('[at]','@')
        email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", email)
        #print(td.b.text,email[0])
        df.loc[len(df.index)] = [td.b.text,email[0],"Civil Engg, IIT Delhi"]<jupyter_output><empty_output><jupyter_text>for computer science and engineering<jupyter_code>source = urllib.request.urlopen('http://www.cse.iitd.ac.in/index.php/2011-12-29-23-14-30/faculty').read()
soup=BeautifulSoup(source,'lxml')

for tr in soup.find_all('tr'):
    if tr.a:
        email=tr.text
        email=email.replace(' AT ','@')
        email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", email)
        if not email:
            email=re.findall("\w+@\w+\.{1}\w+",tr.text.replace(' AT ','@'))
        if not email:
            continue
        df.loc[len(df.index)] = [tr.a.text,email[0],"Department of Computer Science and Engineering"]
        #print(tr.a.text,email[0])<jupyter_output><empty_output><jupyter_text>for electrical<jupyter_code>source = urllib.request.urlopen('https://ee.iitd.ac.in/people/faculty.html').read()
soup=BeautifulSoup(source,'lxml')
for tags in soup.find_all('td'):
    if tags.span:
        txt=tags.text
        txt=txt.replace('[AT]','@')
        txt=txt.replace('[at]','@')
        email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", txt)
        if not email:
            email=['none']
            continue
        df.loc[len(df.index)] = [tags.strong.text,email[0],"Department of Electrical Engineering"]<jupyter_output><empty_output><jupyter_text>for Management<jupyter_code>source = urllib.request.urlopen('https://dms.iitd.ac.in/all-faculty-members/').read()
soup=BeautifulSoup(source,'lxml')
for people in soup.find_all('div',class_="item-content"):
    name=people.h3.text
    email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+",str(people))
    if not email:
        email=re.findall("\w+@\w+\.{1}\w+",str(people))
    if not email:
        email=["none"]
    df.loc[len(df.index)] = [name,email[0],"Department of Management Studies"]<jupyter_output><empty_output><jupyter_text>For Mathematics<jupyter_code>source = urllib.request.urlopen('https://maths.iitd.ac.in/drupal/faculty').read()
soup=BeautifulSoup(source,'lxml')
for people in soup.find_all('tr'):
    if people.p:
        if not people.a:
            continue
        name=people.a.text
        txt=people.text
        txt=txt.replace('[at]','@')
        email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", txt)
        if not email:
            email=re.findall("\w+@\w+\.{1}\w+",txt)
        df.loc[len(df.index)] = [name,email[0],"Department of Mathematics"]<jupyter_output><empty_output><jupyter_text>For Mechanics<jupyter_code>source = urllib.request.urlopen('https://mech.iitd.ac.in/faculty').read()
soup=BeautifulSoup(source,'lxml')
for t in soup.find_all('address'):
    email=t.text.replace('[at]','@')
    t=t.text.split('\n')
    name=t[1]
    email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", email)
    if not email:
        continue
    df.loc[len(df.index)] = [name,email[0],"Department of Mechanical Engineering"]<jupyter_output><empty_output><jupyter_text>For physics<jupyter_code>source = urllib.request.urlopen('https://physics.iitd.ac.in/faculty').read()
soup=BeautifulSoup(source,'lxml')
for faculty in soup.find_all('div',class_="team-details ar-team-ht"):
    email=faculty.find('span',class_="spamspan").text
    email=email.replace('[dot]','.')
    email=email.replace('[at]','@')
    name=faculty.h4.text
    df.loc[len(df.index)] = [name,email,"Department of Physics"]<jupyter_output><empty_output><jupyter_text>For textile and fibre<jupyter_code>source = urllib.request.urlopen('https://textile.iitd.ac.in/faculty.php').read()
soup=BeautifulSoup(source,'lxml')
for faculty in soup.find_all('div',class_="faculty-info"):
    link=faculty.a.get('href')
    try:
        conpage=urllib.request.urlopen('https://textile.iitd.ac.in/'+link+"#contact-details").read()
    except:
        continue
    conpage_html=BeautifulSoup(conpage,'lxml')
    em=conpage_html.find('ul',class_="faculty-contact").a.text
    em=em.replace('[at]','@')
    em=em.replace('[AT]','@')
    email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", em)
    if not email:
        email=re.findall("\w+@\w+\.{1}\w+[.iitd.ernet.in]+", em)
    if not email:
        email=["None"]
    name=conpage_html.h2.text
    df.loc[len(df.index)] = [name,email[0],"Department of Textile Technology"]<jupyter_output><empty_output><jupyter_text>For Design<jupyter_code>source = urllib.request.urlopen('https://design.iitd.ac.in/faculty-contacts.html').read()
soup=BeautifulSoup(source,'lxml')
name="None"
for people in soup.find_all('div',class_="clearfix grpelem"):
    if people.find('p',class_="General"):
        if name==people.find('p',class_="General").text:
            continue
        txt=people.text
        email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", txt)
        if not len(email):
            email=re.findall("\w+@\w+\.{1}\w+", txt)
        name=people.find('p',class_="General").text
        df.loc[len(df.index)] = [name,email[0],"Department of Design"]<jupyter_output><empty_output><jupyter_text>For Humanities<jupyter_code>source = urllib.request.urlopen('https://hss.iitd.ac.in/faculty').read()
soup=BeautifulSoup(source,'lxml')
name="None"
for people in soup.find_all('div'):
        if people.find('div',class_="views-field views-field-title"):
            txt=people.text
            txt=txt.replace('[at]','@')
            txt=txt.replace(' [dot] ','.')
            email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", txt)
            if not email:
                txt=txt.replace(' @ ','@')
                email=re.findall("\w+@\w+\.{1}\w+[.iitd.ac.in]+", txt)
            if not email:
                email=["None"]
            if name==people.find('div',class_="views-field views-field-title").text:
                continue
            name=people.find('div',class_="views-field views-field-title").text
            df.loc[len(df.index)] = [name,email[0],"Humanities & Social Sciences"]
df.tail()
df.to_json('Faculty_info.json',orient='table',index=False)
pdf=pd.read_json('Faculty_info.json',orient='table')
pdf
df.to_csv('Faculty_info.csv',index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.20345279720279721, 'fraction_numerical': 0.006337412587412587, 'mean_word_length': 3.990730643402399, 'pattern_counts': {'":': 0, '<': 61, '<?xml version=': 0, '>': 61, 'https://': 15, 'lorem ipsum': 0, 'www.': 1, 'xml': 17}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '27877', 'license': 'MIT', 'n_tokens_mistral': 3548, 'n_tokens_neox': 3484, 'n_words': 378, 'path': 'IITD-Krishnava/.ipynb_checkpoints/Data_collection_code-checkpoint.ipynb', 'repo_name': 'CodeTrophs/PaperTags', 'size': '9152'}