<jupyter_start><jupyter_text>The logic is something like this:1. Identify the types of jobs we want to get the job postings for. Let us say one of them is "backend engineer". 2. Use Selenium Webdriver to load the page http://www.indeed.com/m/jobs?q=backend+engineer (You can add other query parameters here like location, which we are ignoring here).Get the html and use BeautifulSoup to parse the content. This main search page tells us 3 important things - (a) The first 10 jobs for this search and their URLs, (b) The next to the next page of 10 jobs and (c) How many pages we need to click through to load all the jobs for this search.3. Write code to parse out the Job Urls in the first page. Every URL will be of this format: https://www.indeed.com/m/viewjob?jk=10bca72276277b22 . Load this page and get job title, location, salary, company name, summary and write one row per job into a CSV.4. Use selenium to click the next link to go to the next page and do exactly what we did in (3). Do these N times where N is the number of pages.5. Identify the next job type and repeat from (2).6. Upload all these CSVs into a S3 bucket.<jupyter_code># This is a very simple example of how to parse stuff in the main Search page.
start_url = "http://www.indeed.com/m/jobs?q=backend+engineer"
page = requests.get(start_url)
start_soup = BeautifulSoup(page.text, "html.parser")
print(start_soup.title.text)
# This is a very simple example of how to parse stuff out of an individual job page.
test_url="https://www.indeed.com/m/viewjob?jk=10bca72276277b22"
test_job_link_page = requests.get(test_url)
test_job_link_soup = BeautifulSoup(test_job_link_page.text, "html.parser")
#print(test_job_link_soup.body.p.text)
print('job title:', test_job_link_soup.body.p.b.text.strip())
print('company name:', test_job_link_soup.body.p.b.next_sibling.next_sibling.string.strip())
print('location:', test_job_link_soup.body.p.span.text.strip())
#print('summary:', test_job_link_soup.find(name="div", attrs={"id":"desc"}).text)
# Given a soup object, parse out all the job urls.
def extract_job_links(soup): 
  job_links = []
  for h in soup.find_all(name="h2", attrs={"class":"jobTitle"}):
      for a in h.find_all(name="a", attrs={"rel":"nofollow"}):
        job_links.append(a["href"])
  return(job_links)
# Given a list of job urls (links), parse out relevant content for all the job pages and store in a dataframe
def extract_job_listings(job_links):
    job_link_base_url="https://www.indeed.com/m/{}"
    job_listings=[]
    for job_link in job_links:
        j = random.randint(1000,2200)/1000.0
        time.sleep(j) #waits for a random time so that the website don't consider you as a bot
        job_link_url = job_link_base_url.format(job_link)
        #print('job_link_url:', job_link_url)
        job_link_page = requests.get(job_link_url)
        job_link_soup = BeautifulSoup(job_link_page.text, "html.parser")
        #print('job_link_text:', job_link_soup.text)
        #job_listings_df.loc[count] = extract_job_listing(job_link_url, job_link_soup)
        job_listings.append(extract_job_listing(job_link_url, job_link_soup))
    
    
    columns = ["job_url", "job_title", "company_name", "location", "summary", "salary"]
    job_listings_df = pd.DataFrame(job_listings, columns=columns)
    return job_listings_df

# Given a single job listing url and the corresponding page, parse out the relevant content to create an entry 
def extract_job_listing(job_link_url, job_link_soup):
    job_listing = []
    job_listing.append(job_link_url)
    job_listing.append(job_link_soup.body.p.b.text.strip())
    job_listing.append(job_link_soup.body.p.b.next_sibling.next_sibling.string.strip())
    job_listing.append(job_link_soup.body.p.span.text.strip())
    job_listing.append(job_link_soup.find(name="div", attrs={"id":"desc"}).text)
    job_listing.append("Not_Found")
    return job_listing
    
    #print(job_listing)        
# Given a single page with many listings, go to the individual job pages and store all the content to CSV
def parse_job_listings_to_csv(soup, fileName):
    job_links = extract_job_links(soup)
    job_posts = extract_job_listings(job_links)
    job_posts.to_csv(fileName, encoding='utf-8', index=False)
# A simple example to show how to use Selenium to go through the next links.
next_page_url_pattern="https://www.indeed.com/m/{}"
driver = webdriver.Chrome('/Users/Raghu/Downloads/chromedriver')
start_url = "http://www.indeed.com/m/jobs?q=data+analyst"
driver.set_page_load_timeout(15)
driver.get(start_url)
start_soup = BeautifulSoup(driver.page_source, "html.parser")

print('first page jobs:')
print(extract_job_links(start_soup))
#print start_soup.find(name='a', text='Next')
next_link=driver.find_elements_by_xpath("//a[text()='Next']")[0]
#print next_link
next_link.click()
# Use Selenium to go to do pagination - Click the next links (for now limit to only 5 next links)
driver = webdriver.Chrome('/Users/Raghu/Downloads/chromedriver')
start_url = "http://www.indeed.com/m/jobs?q=frontend+engineer"
driver.set_page_load_timeout(15)
driver.get(start_url)
start_soup = BeautifulSoup(driver.page_source, "html.parser")

parse_job_listings_to_csv(start_soup, "job_postings_0.csv")

for i in range(1,5):
    print('loading {} page'.format(str(i)))
    j = random.randint(1000,3300)/1000.0
    time.sleep(j) #waits for a random time so that the website don't consider you as a bot
    next_page_url = driver.find_elements_by_xpath("//a[text()='Next']")[0]
    page_loaded = True
    try:
        next_page_url.click()
    except TimeoutException:
        get_info = False
        driver.close()
    if page_loaded:
        soup=BeautifulSoup(driver.page_source)
        parse_job_listings_to_csv(soup, "job_postings_{}.csv".format(str(i)))

driver.close()<jupyter_output>loading 1 page
loading 2 page
loading 3 page
loading 4 page
{'fraction_non_alphanumeric': 0.11217183770883055, 'fraction_numerical': 0.013637913399249914, 'mean_word_length': 4.72390243902439, 'pattern_counts': {'":': 4, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 4, 'lorem ipsum': 0, 'www.': 8, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '467456', 'license': 'MIT', 'n_tokens_mistral': 2061, 'n_tokens_neox': 1991, 'n_words': 538, 'path': 'sandbox/indeed-scraping/indeed-scraping-for-job-type.ipynb', 'repo_name': 'shiftorg/skills', 'size': '5866'}