<jupyter_start><jupyter_text>Table of Contents1&nbsp;&nbsp;Importing libraries2&nbsp;&nbsp;Function definitions3&nbsp;&nbsp;Scraping data science jobs from Glassdoor  Importing libraries<jupyter_code>from selenium.webdriver.common.action_chains import ActionChains
from selenium.common.exceptions import NoSuchElementException, ElementClickInterceptedException
from selenium import webdriver
import time
import pandas as pd
from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.common.exceptions import TimeoutException, NoSuchElementException
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.common.action_chains import ActionChains
from fake_useragent import UserAgent
from urllib.request import urlopen, Request
from urllib.parse import urlparse
from bs4 import BeautifulSoup as soup




from IPython.core.display import display, HTML
display(HTML("<style>.container { width:100% !important; }</style>"))
display(HTML("<style>.output_result { max-width:100% !important; }</style>"))

#display(HTML('<style>.prompt{width: 0px; min-width: 0px; visibility: collapse}</style>'))

import warnings
warnings.filterwarnings('ignore')
warnings.filterwarnings("ignore", category=UserWarning, module='bs4')

pd.options.display.max_rows = 999<jupyter_output><empty_output><jupyter_text>Function definitions<jupyter_code>def requestAndParse(url):
    try:
        # define headers to be provided for request authentication
        headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) ' 
                        'AppleWebKit/537.11 (KHTML, like Gecko) '
                        'Chrome/23.0.1271.64 Safari/537.11',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Accept-Encoding': 'none',
            'Accept-Language': 'en-US,en;q=0.8',
            'Connection': 'keep-alive'}
        request_obj = Request(url = url, headers = headers)
        opened_url = urlopen(request_obj)
        page_html = opened_url.read()
        opened_url.close()
        page_soup = soup(page_html, "html.parser")
        return page_soup, requested_url

    except Exception as e:
        print(e)
#Initializes driver
def initialize_driver(path):
    #Initializing the webdriver
    options = webdriver.ChromeOptions()
    
    ua = UserAgent()
    userAgent = ua.random
    options.add_argument(f'user-agent={userAgent}')
    
    #makes the driver more 'undetectale'
    options.add_argument("--disable-blink-features"), options.add_argument("--disable-blink-features=AutomationControlled")
    options.add_experimental_option("excludeSwitches", ["enable-automation"])
    options.add_experimental_option('useAutomationExtension', False)
    
    #Change the path to where chromedriver is in your home folder.
    driver = webdriver.Chrome(executable_path=path, options=options)
    driver.maximize_window()

    return driver

#Brings job ids from job position and location search
def job_ids(driver, keyword, location,pagelimit=33):
    eids = []
    jids = []
    pagenum_limit = 33
    pagenum = 0
    
    try:
        # fill in search bar keyword field
        position_field = driver.find_element_by_xpath("//*[@id='sc.keyword']")
        position_field.send_keys(Keys.COMMAND+'a')
        position_field.send_keys(Keys.DELETE)
        position_field.send_keys(keyword)
        # fill in search bar location field
        location_field = driver.find_element_by_xpath("//*[@id='sc.<EMAIL>']")
        location_field.send_keys(Keys.COMMAND+'a')
        location_field.send_keys(Keys.DELETE)
        location_field.send_keys(location)

        # click search button and then view all jobs
        time.sleep(2)
        driver.find_element_by_xpath("//*[@type='submit']").click()
        
        time.sleep(3)
        driver.find_element_by_xpath("//a[@data-test='jobs-location-see-all-link']").click()
               
        while int(pagenum) <= pagelimit:
            
            
            # close a random popup if it shows up
            try:
                driver.find_element_by_xpath("//*[@id='JAModal']/div/div[2]/span").click()
            except NoSuchElementException:
                pass
            
            try:
                driver.find_element_by_css_selector('[alt="Close"]').click() #clicking to the X.
            except Exception:
                pass

            time.sleep(1)

            eids += [x.get_attribute('data-brandviews').split(':')[2][4:] for x in driver.find_elements_by_css_selector("li.react-job-listing")]
            jids += [x.get_attribute('data-id') for x in driver.find_elements_by_css_selector("li.react-job-listing")]


            # close a random popup if it shows up
            try:
                driver.find_element_by_xpath("//*[@id='JAModal']/div/div[2]/span").click()
            except NoSuchElementException:
                pass
            
            pages = driver.find_element_by_xpath("//div[@data-test='ResultsFooter']").text.split()
            pagenum = pages[1]
            if(pagenum == pages[3]):
                break
                
            # go to next page
            try:
                driver.find_element_by_xpath('.//a[@data-test="pagination-next"]').click()

            except NoSuchElementException:
                pass
        return eids, jids
        
        
    except NoSuchElementException:
        pass
def parse_url(url):
    try:
        # define headers to be provided for request authentication
        headers = {'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64) ' 
                        'AppleWebKit/537.11 (KHTML, like Gecko) '
                        'Chrome/23.0.1271.64 Safari/537.11',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.3',
            'Accept-Encoding': 'none',
            'Accept-Language': 'en-US,en;q=0.8',
            'Connection': 'keep-alive'}
        request_obj = Request(url = url, headers = headers)
        opened_url = urlopen(request_obj)
        page_html = opened_url.read()
        opened_url.close()
        page_soup = soup(page_html, "html.parser")
                
        if type(page_soup.find("div", class_="css-ur1szg e11nt52q0").find("span", class_="css-1pmc6te e11nt52q4")) == type(None):
            rating=''
            company=page_soup.find("div", class_="css-16nw49e e11nt52q1").getText()

        else:
            rating = page_soup.find("div", class_="css-ur1szg e11nt52q0").find("span", class_="css-1pmc6te e11nt52q4").getText()
            company = page_soup.find("div", class_="css-16nw49e e11nt52q1").getText().replace(rating,'')
            rating = rating[:-1]

        position = page_soup.find("div", class_="css-17x2pwl e11nt52q6").getText()
        jobloc = page_soup.find("div", class_="css-1v5elnn e11nt52q2").getText()
        
        if type(page_soup.find("span", class_="small css-10zcshf e1v3ed7e1")) == type(None):
            salary=''
        else:
            salary = page_soup.find("span", class_="small css-10zcshf e1v3ed7e1").getText()#.split(':')[1].strip()
        desc = ' '.join([para.get_text() for para in page_soup.find("div", {'id':"JobDescriptionContainer"}).find_all(['li','p'])])
        
    
    except Exception as e:
        print(e)
        rating=''
        company=''
        position=''
        jobloc=''
        salary=''
        desc=''
    return position, desc, salary, jobloc, company, rating<jupyter_output><empty_output><jupyter_text>Scraping data science jobs from Glassdoor<jupyter_code>driver = initialize_driver("/usr/local/bin/chromedriver")
url = "https://www.glassdoor.com/blog/tag/job-search/"
driver.get(url)

locations = ['remote', 'boulder, co', 'detroit, mi', 'saint louis, mo', 'new orleans, la', 'las vegas, nv', 'new mexico', 
             'salt lake city, ut', 'cincinnati, oh', 'bentonville, ar', 'minneapolis, mn', 'madison, wi', 'sacramento, ca',
             'new jersey', 'houston, tx', 'san antonio, tx', 'san diego, ca', 'fort worth, tx', 'jacksonville, fl', 
             'columbus, oh', 'north carolina', 'indianapolis, in', 'washington, dc']

for location in locations:
    print('Getting Job IDs for ', location)
    eids, jids = job_ids(driver,'data science jobs', location)
    time.sleep(1)
    pd.DataFrame({'eids':eids, 'jids':jids}).to_csv("_".join(keyword.split()+location.split(', '))+'.csv')
  
keyword='data scientist'
jobdict = {'position':[], 'desc':[], 'salary':[], 'jobloc':[], 'company':[], 'rating':[]}

for location in locations:
    counter = 0
    print("Scraping jobs from ", location)
    df = pd.read_csv("_".join(keyword.split()+location.split(', '))+'.csv').iloc[:,1:]
    df = df[~df.jids.duplicated()]
    for urlid in df.jids:
        counter += 1
        jurl = 'http://www.glassdoor.com/job-listing/?jl='+str(urlid)
        res = parse_url(jurl)
        jobdict['position'].append(res[0])
        jobdict['desc'].append(res[1])
        jobdict['salary'].append(res[2])
        jobdict['jobloc'].append(res[3])
        jobdict['company'].append(res[4])
        jobdict['rating'].append(res[5])
        if counter%100 == 0:
            print(counter, '/', len(df.jids))

pd.DataFrame(jobdict).to_csv('glassdoor_data_science_jobs.csv')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1420323325635104, 'fraction_numerical': 0.02131009867730422, 'mean_word_length': 2.933526011560694, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 20, 'https://': 1, 'lorem ipsum': 0, 'www.': 2, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '413958', 'license': 'MIT', 'n_tokens_mistral': 3158, 'n_tokens_neox': 2993, 'n_words': 574, 'path': 'notebooks/1.0-MS-GlassdoorScraper.ipynb', 'repo_name': 'mervesa/DS_jobposting_topic', 'size': '9527'}