<jupyter_start><jupyter_text>Indeed Web ScraperThis Notebook is for educational purposes. Indeed's Terms of Service does not allow "use of any automated system or software, whether operated by a third party or otherwise, to extract data from the Site (such as screen scraping or crawling)"Use of this notebook would be a TOS violation<jupyter_code>import requests
from bs4 import BeautifulSoup
from datetime import datetime
import pandas as pd
import copy
def get_url(job_position):
    """Function to search a position on the Indeed webpage"""
    template = 'https://www.indeed.com/jobs?q={}&l'
    url = template.format(job_position)

    return url
def response_func(url):
    """Function returns a get request for html of an Indeed webpage"""
    response = requests.get(url, headers={'User-Agent': 'test'})

    return response
def get_cards(response):
    """ Indeed search results are divided up by cards.
        This function returns the cards get_job_cards for each job title'
    """
    soup = BeautifulSoup(response.text, 'html.parser')
    cards = soup.find_all('td', {'class': 'resultContent'})

    return cards
def get_job_title(card):
    """Function returns the job title"""
    job_title = card.find('h2', {'class': 'jobTitle'}).text
    if 'new' in job_title:
        job_title = job_title.replace('new', "")

    return job_title
def get_job_url(card):
    """Function returns the job posting url"""
    try:
        job_url = 'https://www.indeed.com' + card.h2.a.get('href')
        return job_url
    except AttributeError:
        return 'https://www.indeed.com/404'
def get_company_name(card):
    """Function returns the company name who posted the job"""
    company = card.find('span', 'companyName').text
    return company
def get_location(card):
    """Function returns the job location"""
    location = card.find('div', 'companyLocation').text

    return location
def get_job_description(job_url):
    response = requests.get(job_url, headers={'User-Agent': 'test'})
    soup = BeautifulSoup(response.text, 'html.parser')
    try:
        description = soup.find('div', 'jobsearch-jobDescriptionText').text.strip()
        return description
    except AttributeError:
        return "No job description"
def get_job_records(job_position):
    """
    Function returns an array of the following
    * Company Name
    * Job Title
    * Location
    * Descriptions
    * Job Url
    """

    # Retrieves the data on the first page of the search results
    records = []
    url = get_url(job_position)
    response = response_func(url)
    while True:
        cards = get_cards(response)
        for card in cards:
          job_title = get_job_title(card)
          company_name = get_company_name(card)
          job_location = get_location(card)
          job_url = get_job_url(card)
          job_description = get_job_description(job_url)
          today = datetime.today().strftime('%Y-%m-%d')
          records.append([today,
                          job_title,
                          company_name,
                          job_location,
                          job_description,
                          job_url])
    # Goes through each page of the search results
        try:
          response = requests.get(url)
          soup = BeautifulSoup(response.text, 'html.parser')
          url = 'https://www.indeed.com' + soup.find(
                            'a',
                            {'aria=label': 'Next'}
                            ).get('href')
        except AttributeError:
          break
    return records
search_terms = [
    "data scientist",
    "machine learning engineer",
    "data engineer",
    "web developer",
    "frontend developer",
    "backend developer",
    "devops",
    "software engineer",
]
for position in search_terms:
  try:
    scraped_list = get_job_records(position)
    indeed_jobs = pd.DataFrame(scraped_list,
                              columns=[ 'DateOfScrape',
                                        'JobTitle',
                                        'Company',
                                        'Location',
                                        'Description',
                                        'JobUrl'])
    saving_jobs = copy.deepcopy(indeed_jobs)
    saving_jobs = saving_jobs.drop_duplicates(['Description']).reset_index(drop=True)
    saving_jobs = saving_jobs[saving_jobs['Description'].notna()]
    saving_jobs['Description'] = saving_jobs['Description'].apply(lambda x: x.replace('\n', ' '))
    if len(saving_jobs)==1:
      continue

    saving_jobs.set_index(['DateOfScrape'], inplace=True)
    position = position.replace(' ', '_')
    display(saving_jobs)
    saving_jobs.to_csv(f'{position}_jobs.csv', encoding='utf-8', index=False)
    print(f'Successfully saved {position} jobs!")')
  except AttributeError:
    print(f'Failed to save {position} jobs check errlogs')
    continue<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10618932038834951, 'fraction_numerical': 0.0016181229773462784, 'mean_word_length': 2.4774964838255977, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '533882', 'license': 'MIT', 'n_tokens_mistral': 1378, 'n_tokens_neox': 1342, 'n_words': 369, 'path': 'notebooks/Indeed_web_scraper/Indeed_web_scraper.ipynb', 'repo_name': 'Lambda-School-Labs/underdog-devs-ds-a', 'size': '4944'}